diff --git a/deploy/install_decode_watchdog_launchd.sh b/deploy/install_decode_watchdog_launchd.sh new file mode 100755 index 00000000..0b7699ce --- /dev/null +++ b/deploy/install_decode_watchdog_launchd.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Install the external Primary decode watchdog as a per-user LaunchAgent. +set -euo pipefail + +: "${KAKEYA_RUNTIME_REPO:?set KAKEYA_RUNTIME_REPO}" +: "${KAKEYA_RUNTIME_PYTHON:?set KAKEYA_RUNTIME_PYTHON}" + +WATCHDOG_LABEL="${KAKEYA_WATCHDOG_LABEL:-ai.kakeya.decode-watchdog}" +RUNTIME_LABEL="${KAKEYA_RUNTIME_LABEL:-ai.kakeya.grpc-runtime-prefill}" +STALL_SECONDS="${KAKEYA_DECODE_STALL_SECONDS:-120}" +INTERVAL_SECONDS="${KAKEYA_WATCHDOG_INTERVAL_SECONDS:-30}" +LIVENESS_FILE="${KAKEYA_DECODE_LIVENESS_FILE:-$HOME/.kakeya/primary-decode-liveness.json}" +STATE_FILE="${KAKEYA_WATCHDOG_STATE_FILE:-$HOME/.kakeya/decode-watchdog-state.json}" +UNHEALTHY_FILE="${KAKEYA_RUNTIME_UNHEALTHY_FILE:-$HOME/.kakeya/primary-runtime-unhealthy.json}" +PLIST="$HOME/Library/LaunchAgents/$WATCHDOG_LABEL.plist" +LOG_FILE="$HOME/.kakeya/decode-watchdog.log" + +mkdir -p "$(dirname "$PLIST")" "$(dirname "$LOG_FILE")" +cat > "$PLIST" < + + + Label$WATCHDOG_LABEL + ProgramArguments + $KAKEYA_RUNTIME_PYTHON + $KAKEYA_RUNTIME_REPO/scripts/decode_watchdog.py + --liveness-file$LIVENESS_FILE + --state-file$STATE_FILE + --unhealthy-file$UNHEALTHY_FILE + --runtime-label$RUNTIME_LABEL + --stall-seconds$STALL_SECONDS + + StartInterval$INTERVAL_SECONDS + RunAtLoad + ProcessTypeBackground + StandardOutPath$LOG_FILE + StandardErrorPath$LOG_FILE + +EOF + +chmod 644 "$PLIST" +DOMAIN="gui/$(id -u)" +launchctl bootout "$DOMAIN/$WATCHDOG_LABEL" 2>/dev/null || true +launchctl bootstrap "$DOMAIN" "$PLIST" +launchctl kickstart -k "$DOMAIN/$WATCHDOG_LABEL" +echo "installed $WATCHDOG_LABEL -> $PLIST" diff --git a/docs/adr/0017-prefill-compute-worker-orchestration.md b/docs/adr/0017-prefill-compute-worker-orchestration.md index 533913bd..b3fa88b7 100644 --- a/docs/adr/0017-prefill-compute-worker-orchestration.md +++ b/docs/adr/0017-prefill-compute-worker-orchestration.md @@ -31,6 +31,42 @@ Kakeya adopts three explicit fleet roles: user decode. 3. **PREFILL_CACHE:** loads no model; spends RAM on immutable snapshots only. +### Local decode process boundary + +Primary decode may run behind the feature flag `--decode-worker`. In this +profile the gRPC/router process does not construct an MLX verifier and does not +load model weights. It starts one child process over a mode-0600 Unix-domain +socket; that child owns the single loaded model and all session K/V adapters. +The transport is protocol-v1 length-prefixed JSON plus an optional opaque +binary payload. It never uses pickle and is not exposed on TCP. + +The local protocol has six operations: + +- `Init(session_id, token_ids?)` creates/replaces isolated session K/V; +- `ImportSnapshot(session_id, compatibility, payload)` imports the existing + allens portable snapshot format after exact compatibility validation; +- `Append(session_id, token_ids)` commits an all-accepted block; +- `GenerateStep(session_id)` atomically selects and commits one greedy token; +- `Close(session_id)` releases that session's K/V; and +- `Health()` returns protocol, process, model geometry, session and MLX-memory + state. + +The router serializes requests because the MVP MLX execution stream is shared. +No per-token K/V snapshot crosses IPC. The router retains a proof checkpoint: +the last imported immutable allens snapshot plus only tokens acknowledged +after that boundary (or full acknowledged history when no snapshot exists). +An IPC EOF, child exit, or operation timeout hard-kills the child, starts a new +one, restores that checkpoint, and retries the unacknowledged current +operation once. Because checkpoint state advances only after a correlated +response and `GenerateStep` is atomic in the child, an ambiguous crash cannot +duplicate a committed token. Client cancellation hard-kills an in-flight +worker forward; the next request follows the same restore path. + +This process boundary is initially opt-in. The in-process MLX path remains the +rollback path until the Mac acceptance gates pass. It is not a remote decode +service and does not change the rule that no fleet RPC occurs in the token +loop. + ### Request flow For a cold append, the primary: @@ -118,6 +154,10 @@ Blocking tests cover: - real MLX local-prefill vs remote-prefill-import continuation logits and argmax equivalence; - zero prefill/cache RPCs during `Generate`. +- protocol correlation/version/size validation for local decode IPC; +- in-process vs worker greedy-token parity; +- cancellation hard-kill and child crash/timeout recovery from both + full-history and allens-snapshot-plus-proof checkpoints. ## Consequences @@ -136,6 +176,10 @@ Costs: primary; - snapshots still duplicate bounded state at checkpoint boundaries; - MLX worker execution is serialized per loaded verifier in the MVP. +- worker-mode locally computed prefill checkpoints are not republished by the + router, because export is deliberately absent from the token-loop protocol; + allens-produced snapshots remain importable and are the preferred durable + recovery boundary. ## Legacy paths diff --git a/docs/ops/primary-decode-hardening.md b/docs/ops/primary-decode-hardening.md new file mode 100644 index 00000000..e89d6f5d --- /dev/null +++ b/docs/ops/primary-decode-hardening.md @@ -0,0 +1,79 @@ +# Primary decode hardening + +The Primary runtime publishes read-only diagnostics on +`http://127.0.0.1:8091` by default: + +- `GET /healthz` combines decode liveness and unified-memory health. +- `GET /v1/runtime/liveness` reports phase, session, token index, update + timestamp, and PID. +- `GET /v1/runtime/memory` reports MLX active/cache/peak bytes, process RSS, + active sessions, and live KV bytes. + +The default memory policy warns at 18 GiB, stops admitting new sessions at +20 GiB, and marks the runtime unhealthy at 21.5 GiB. When the final active +session is removed under memory pressure, cleanup runs in this order: +verifier reset, Python garbage collection, then `mlx.core.clear_cache()`. +Thresholds and the diagnostics port are configurable with the +`--memory-*-gb` and `--runtime-health-*` server flags. + +All removal paths (explicit close, client cancellation, LRU, TTL, INV-1, and +INV-2) pass through the `SessionStore` removal hook. This releases the slab +and removes any per-session verifier binding exactly once. + +## Decode watchdog + +Install the independent per-user LaunchAgent: + +```bash +KAKEYA_RUNTIME_REPO=/path/to/repo \ +KAKEYA_RUNTIME_PYTHON=/path/to/python \ +KAKEYA_RUNTIME_LABEL=ai.kakeya.grpc-runtime-prefill \ +deploy/install_decode_watchdog_launchd.sh +``` + +Every 30 seconds the watchdog reads +`~/.kakeya/primary-decode-liveness.json`. It restarts the configured Primary +LaunchAgent only after observing the same decode token stale for at least +120 seconds twice in succession. The watchdog also recycles a runtime that +has written the 21.5 GiB unhealthy marker. Runtime startup clears a stale +marker. + +`Session.generate(inter_token_timeout_s=...)` provides a client-side +notification deadline. It cancels the stream and raises +`InterTokenTimeoutError`; it does not attempt process recovery. Hard recovery +remains the external watchdog's responsibility. + +## Isolated MLX decode worker + +Enable the opt-in process boundary by adding these flags to the Primary +runtime command: + +```bash +--backend mlx \ +--decode-worker \ +--decode-worker-timeout-s 120 \ +--decode-worker-startup-timeout-s 180 +``` + +The router then loads no MLX model. A private mode-0600 UDS child owns the +model and per-session K/V. Keep `--decode-worker-socket` unset for an +automatically unique temporary path; set it only when launchd supervision or +socket diagnostics require a stable path. + +Migration procedure: + +1. deploy with the flag absent and confirm the existing in-process smoke; +2. enable `--decode-worker` on one Primary and verify `/healthz`, append, + streaming generation, cancellation, and one injected child kill; +3. confirm the child PID changes, the current turn resumes from the imported + allens snapshot plus acknowledged proof checkpoint, and the router RSS does + not contain a second model; +4. retain flag removal as rollback until the Mac parity, 100-session, + fault-injection, latency, and four-hour acceptance suite has passed. + +Worker mode can import allens portable snapshots but deliberately does not +export a K/V snapshot per token. Locally computed prefill boundaries therefore +are not republished through the router in this release. A cancellation kills +the in-flight child forward; the affected session is still closed by the gRPC +cancellation contract, while other retained session checkpoints are restored +lazily if later used. diff --git a/docs/primary_decode_acceptance.md b/docs/primary_decode_acceptance.md new file mode 100644 index 00000000..37f362d5 --- /dev/null +++ b/docs/primary_decode_acceptance.md @@ -0,0 +1,112 @@ +# Primary decode Mac acceptance + +`scripts/bench_agentic/primary_decode_acceptance.py` is the blocking Mac +acceptance runner for the isolated Primary decode worker. It emits JSON +conforming to `schemas/primary_decode_acceptance.schema.json` and JUnit XML. +The full mode includes the four-hour mixed-prompt run; it is only started by +an explicit `--mode all` or `--mode endurance`. + +Example full command: + +```bash +PYTHONPATH=.:sdks/python python3 \ + scripts/bench_agentic/primary_decode_acceptance.py \ + --mode all \ + --grpc-address 127.0.0.1:50051 \ + --tokenizer-id Qwen/Qwen3-0.6B \ + --worker-control-command "/path/to/decode-worker-acceptance-adapter" \ + --latency-baseline results/platform-tests/bench_mlx_verifier_1779507043.json \ + --output results/platform-tests/primary_decode_acceptance.json \ + --junit-output results/platform-tests/primary_decode_acceptance.xml +``` + +Start the runtime with its test-only, mode-0600 UDS enabled: + +```bash +PYTHONPATH=.:sdks/python python3 scripts/start_grpc_runtime_server.py \ + --backend mlx \ + --verifier-id Qwen/Qwen3-0.6B \ + --bind 127.0.0.1:50051 \ + --capacity 128 --sink 4 --window 64 \ + --decode-worker \ + --decode-worker-timeout-s 110 \ + --decode-worker-acceptance-socket /tmp/kakeya-decode-acceptance.sock +``` + +The corresponding adapter command is: + +```bash +PYTHONPATH=. python3 \ + scripts/bench_agentic/decode_worker_acceptance_adapter.py \ + --socket /tmp/kakeya-decode-acceptance.sock +``` + +Run `footprint`, `disconnect`, `hang`, `kv-restore`, and `latency` separately +for pre-CI/hardware checks. Reserve `--mode all` for the blocking release run: +it deliberately includes the four-hour endurance workload. + +Short development runs can override `--session-count` and +`--endurance-duration-s`, but the gates remain fixed at 100 sessions and +14,400 seconds. A shortened run therefore emits useful diagnostics while +remaining failed and cannot be mistaken for release evidence. + +## Decode-worker adapter contract + +The runtime provides +`scripts/bench_agentic/decode_worker_acceptance_adapter.py`, passed through +`--worker-control-command`. The harness starts it once per operation, writes +one JSON object to stdin, and expects one JSON object on stdout: + +```json +{ + "schema_version": 1, + "operation": "snapshot", + "payload": {} +} +``` + +```json +{ + "schema_version": 1, + "operation": "snapshot", + "ok": true, + "data": {} +} +``` + +Nonzero exit status, malformed JSON, `ok: false`, or mismatched operation is +reported as a gate error. The adapter must be test-only/local-only and must +not expose fault injection on a network listener. + +Required operations: + +- `snapshot`: returns integer `runtime_pid`, `worker_pid`, + `worker_restart_count`, `process_footprint_bytes`, `active_sessions`, and + `active_generations`. The footprint must cover the runtime plus owned decode + worker, not only the router process. +- `inject_hang`: accepts payload `phase: "next_forward"` and + `expected_worker_pid`; atomically arms exactly the next MLX forward in that + worker and returns `accepted: true`. The injected forward must remain hung + until the normal watchdog/recycle path kills the worker. +- `kv_restore_parity`: accepts `prompt_token_ids` and performs a normal + decode, worker recycle, restore from the persisted Allens KV snapshot plus + proof checkpoint, and repeated decode. It returns + `baseline_first_token_id`, `restored_first_token_id`, + `baseline_logits_sha256`, `restored_logits_sha256`, and the literal + `restore_source: "allens_kv+proof_checkpoint"`. SHA-256 is over a + canonical, dtype-preserving byte representation of the last-token logits. + +The gRPC/router branch must additionally guarantee that closing the client +channel cancels the in-flight Generate and removes its session. The harness +starts timing only after `snapshot.active_generations` becomes nonzero, then +requires both active counters to become zero within five seconds. + +## Existing benchmark reuse + +Mixed-prompt endurance records are aggregated by +`inference_engine.bench.session_long_run.aggregate_run`. Latency baseline +input accepts either a previous acceptance report's top-level `latency` +summary or the existing `bench_mlx_verifier.py` JSON. For the latter, the +reported MLX generation mean per token is used as the reference for both +p50 and p95 and is identified as such in `baseline_source`; a prior +acceptance report is preferred once available. diff --git a/inference_engine/backends/mlx/decode_worker.py b/inference_engine/backends/mlx/decode_worker.py new file mode 100644 index 00000000..741974b9 --- /dev/null +++ b/inference_engine/backends/mlx/decode_worker.py @@ -0,0 +1,940 @@ +"""Feature-flagged MLX decode process with versioned local UDS IPC. + +The router imports this module without importing MLX. The child process loads +the model once, owns all session K/V, and executes one request at a time. A +request is a length-prefixed JSON header plus an optional opaque binary payload +(used for allens-compatible K/V snapshots); pickle is never used on the wire. + +Protocol v1 operations: + +``Init`` creates/replaces a session and optionally prefills tokens; +``ImportSnapshot`` restores a portable prefill checkpoint; ``Append`` commits +accepted tokens; ``GenerateStep`` atomically chooses and commits one greedy +token; ``Close`` drops session K/V; and ``Health`` reports process/model state. +""" + +from __future__ import annotations + +import importlib +import hashlib +import json +import multiprocessing +import os +import select +import signal +import socket +import struct +import tempfile +import threading +import time +import uuid +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional + +from inference_engine.session.coordinator import OperationCancelledError + +PROTOCOL_VERSION = 1 +MAX_HEADER_BYTES = 1 << 20 +MAX_PAYLOAD_BYTES = 2 << 30 + + +class DecodeWorkerError(RuntimeError): + """Base class for decode-worker failures.""" + + +class DecodeWorkerUnavailable(DecodeWorkerError): + """The worker exited, hung, or broke its local transport.""" + + +class DecodeWorkerRemoteError(DecodeWorkerError): + """The worker completed a request with an application error.""" + + def __init__(self, error_type: str, message: str) -> None: + super().__init__(f"{error_type}: {message}") + self.error_type = error_type + self.message = message + + +@dataclass(frozen=True) +class DecodeWorkerConfig: + model_id: str + sink_size: int = 4 + window_size: int = 64 + request_timeout_s: float = 120.0 + startup_timeout_s: float = 180.0 + socket_path: str = "" + verifier_factory: str = ( + "inference_engine.backends.mlx.decode_worker:_build_mlx_verifier" + ) + + +@dataclass +class ProofCheckpoint: + """Router-owned restart source. + + ``snapshot`` is an immutable allens-compatible prefill snapshot. Tokens + committed after that boundary are retained in ``replay_token_ids``. When + no snapshot exists, the replay list is the complete committed history. + The current request is added only after its response arrives, making a + retry after an ambiguous hard kill idempotent. + """ + + snapshot: bytes | None = None + compatibility: dict[str, Any] | None = None + replay_token_ids: list[int] = field(default_factory=list) + initialized: bool = False + + +def _recv_exact(sock: socket.socket, size: int) -> bytes: + chunks: list[bytes] = [] + remaining = size + while remaining: + chunk = sock.recv(remaining) + if not chunk: + raise EOFError("decode worker closed the socket") + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def _send_frame( + sock: socket.socket, + header: dict[str, Any], + payload: bytes = b"", +) -> None: + body = bytes(payload) + encoded = json.dumps( + {**header, "payload_length": len(body)}, + sort_keys=True, + separators=(",", ":"), + ).encode() + if len(encoded) > MAX_HEADER_BYTES or len(body) > MAX_PAYLOAD_BYTES: + raise ValueError("decode worker frame exceeds protocol limits") + sock.sendall(struct.pack("!I", len(encoded)) + encoded + body) + + +def _recv_frame(sock: socket.socket) -> tuple[dict[str, Any], bytes]: + header_size = struct.unpack("!I", _recv_exact(sock, 4))[0] + if header_size <= 0 or header_size > MAX_HEADER_BYTES: + raise ValueError("invalid decode worker header length") + header = json.loads(_recv_exact(sock, header_size)) + payload_size = int(header.pop("payload_length", 0)) + if payload_size < 0 or payload_size > MAX_PAYLOAD_BYTES: + raise ValueError("invalid decode worker payload length") + return header, _recv_exact(sock, payload_size) + + +def _resolve(path: str) -> Callable[..., Any]: + module_name, symbol = path.split(":", 1) + return getattr(importlib.import_module(module_name), symbol) + + +def _build_mlx_verifier(config: dict[str, Any]): + """Child-only default factory; importing it is what imports MLX.""" + import torch + from kv_cache_proposer.verifier import VerifierConfig + from inference_engine.backends.mlx.verifier import MLXSinkWindowVerifier + + return MLXSinkWindowVerifier(VerifierConfig( + model_id=str(config["model_id"]), + dtype=torch.bfloat16, + device="cpu", + sink_size=int(config["sink_size"]), + window_size=int(config["window_size"]), + )) + + +def _model_dimensions(verifier: Any) -> tuple[int, int, int]: + declared = getattr(verifier, "dimensions", None) + if declared is not None: + return tuple(int(value) for value in declared) + try: + cfg = getattr(verifier.model, "config", None) or getattr( + verifier.model, "args", None + ) + cfg = getattr(cfg, "text_config", None) or cfg + return ( + int(getattr(cfg, "num_hidden_layers")), + int( + getattr(cfg, "num_key_value_heads", None) + or getattr(cfg, "num_attention_heads") + ), + int( + getattr(cfg, "head_dim", None) + or (cfg.hidden_size // cfg.num_attention_heads) + ), + ) + except (AttributeError, TypeError): + from inference_engine.backends.mlx.cross_model_dlm_verifier import ( + per_layer_kv_geometry, + resolve_mlx_text_model, + ) + geometry = per_layer_kv_geometry(resolve_mlx_text_model(verifier.model)) + if not geometry: + raise + return ( + len(geometry), + max(item[0] for item in geometry), + max(item[1] for item in geometry), + ) + + +def _state(verifier: Any) -> dict[str, Any]: + cached = [int(token) for token in verifier.cached_token_sequence] + kv_bytes = getattr(verifier, "live_kv_bytes", lambda: 0)() + return { + "cached_token_ids": cached, + "k_seq_length": len(cached), + "next_global_position": int(verifier.next_global_position), + "kv_live_bytes": int(kv_bytes), + } + + +def _canonical_logits_sha256(verifier: Any) -> str: + """Hash the current last-token logits without losing dtype metadata.""" + custom = getattr(verifier, "logits_sha256", None) + if custom is not None: + return str(custom()) + logits = getattr(verifier, "_next_token_logits_mx", None) + if logits is None: + logits = verifier.next_token_logits + if logits is None: + raise RuntimeError("decode worker session has no next-token logits") + from inference_engine.distributed.tensor_codec import ( + mlx_to_wire, + to_proto_fields, + torch_to_wire, + ) + + wire = ( + torch_to_wire(logits) + if hasattr(logits, "detach") + else mlx_to_wire(logits) + ) + dtype, shape, data = to_proto_fields(wire) + metadata = json.dumps( + {"dtype": dtype, "shape": [int(value) for value in shape]}, + sort_keys=True, + separators=(",", ":"), + ).encode() + return hashlib.sha256(metadata + b"\0" + data).hexdigest() + + +class _WorkerRuntime: + def __init__(self, config: dict[str, Any], factory_path: str) -> None: + self.template = _resolve(factory_path)(config) + self.sessions: dict[str, Any] = {} + self.started_at = time.time() + self._hang_next_forward = False + try: + self.dimensions = _model_dimensions(self.template) + except (AttributeError, TypeError): + # Test factories and future adapters may expose dimensions directly. + self.dimensions = tuple( + int(value) for value in getattr( + self.template, "dimensions", (1, 1, 1) + ) + ) + + def _new_session(self): + spawn = getattr(self.template, "spawn", None) + if spawn is None: + raise RuntimeError("decode worker verifier must implement spawn()") + return spawn() + + def _before_forward(self) -> None: + if not self._hang_next_forward: + return + self._hang_next_forward = False + while True: + time.sleep(60.0) + + def dispatch( + self, + operation: str, + session_id: str, + arguments: dict[str, Any], + payload: bytes, + ) -> dict[str, Any] | tuple[dict[str, Any], bytes]: + if operation == "Health": + memory = (0, 0, 0) + try: + import mlx.core as mx + memory = ( + int(mx.get_active_memory()), + int(mx.get_cache_memory()), + int(mx.get_peak_memory()), + ) + except (ImportError, AttributeError, RuntimeError): + pass + return { + "pid": os.getpid(), + "protocol_version": PROTOCOL_VERSION, + "session_count": len(self.sessions), + "uptime_seconds": time.time() - self.started_at, + "num_layers": self.dimensions[0], + "num_kv_heads": self.dimensions[1], + "head_dim": self.dimensions[2], + "weight_bytes": int( + getattr(getattr(self.template, "stats", None), "weight_bytes", 0) + ), + "mlx_active_bytes": memory[0], + "mlx_cache_bytes": memory[1], + "mlx_peak_bytes": memory[2], + } + if operation == "InjectHang": + expected_pid = int(arguments["expected_worker_pid"]) + if expected_pid != os.getpid(): + return {"accepted": False, "worker_pid": os.getpid()} + if self._hang_next_forward: + return {"accepted": False, "worker_pid": os.getpid()} + self._hang_next_forward = True + return {"accepted": True, "worker_pid": os.getpid()} + if not session_id: + raise ValueError(f"{operation} requires session_id") + if operation == "Init": + verifier = self._new_session() + self.sessions[session_id] = verifier + tokens = [int(token) for token in arguments.get("token_ids", ())] + if tokens: + self._before_forward() + verifier.prefill(tokens) + return _state(verifier) + if operation == "Close": + existed = self.sessions.pop(session_id, None) is not None + return {"closed": existed} + try: + verifier = self.sessions[session_id] + except KeyError as exc: + raise KeyError(f"unknown decode session {session_id!r}") from exc + if operation == "ImportSnapshot": + custom_import = getattr(verifier, "import_snapshot", None) + if custom_import is not None: + custom_import(payload, arguments["compatibility"]) + return _state(verifier) + from inference_engine.backends.mlx.prefill_snapshot import ( + import_mlx_prefill_snapshot, + ) + from inference_engine.distributed.capability import CacheCompatibility + + verifier.reset() + imported = import_mlx_prefill_snapshot( + payload, + verifier.cache, + compatibility=CacheCompatibility(**arguments["compatibility"]), + ) + if imported.next_token_logits is None: + raise ValueError("decode snapshot is missing continuation logits") + verifier.cached_token_sequence = list(imported.cached_token_ids) + verifier.next_global_position = int(imported.token_count) + verifier.cache_logical_size = len(imported.cached_token_ids) + if hasattr(verifier, "_set_next_token_logits_mx") and not hasattr( + imported.next_token_logits, "detach" + ): + verifier._set_next_token_logits_mx(imported.next_token_logits) + else: + verifier.next_token_logits = imported.next_token_logits + return { + **_state(verifier), + "token_count": imported.token_count, + "block_hash": imported.block_hash.hex(), + } + if operation == "ExportSnapshot": + compatibility = dict(arguments["compatibility"]) + custom_export = getattr(verifier, "export_snapshot", None) + if custom_export is not None: + snapshot = bytes(custom_export(compatibility)) + else: + from inference_engine.backends.mlx.prefill_snapshot import ( + export_mlx_prefill_snapshot, + ) + from inference_engine.distributed.capability import CacheCompatibility + + next_token_logits = getattr( + verifier, "_next_token_logits_mx", None + ) + if next_token_logits is None: + next_token_logits = verifier.next_token_logits + snapshot = export_mlx_prefill_snapshot( + verifier.cache, + token_count=int(verifier.next_global_position), + cached_token_ids=verifier.cached_token_sequence, + compatibility=CacheCompatibility(**compatibility), + next_token_logits=next_token_logits, + ) + return ( + { + **_state(verifier), + "first_token_id": int(verifier.greedy_next_token_id()), + "logits_sha256": _canonical_logits_sha256(verifier), + }, + snapshot, + ) + if operation == "Append": + tokens = [int(token) for token in arguments["token_ids"]] + if not tokens: + raise ValueError("Append token_ids must be non-empty") + self._before_forward() + verifier.append_accepted_tokens(tokens) + return _state(verifier) + if operation == "GenerateStep": + token_id = int(verifier.greedy_next_token_id()) + input_logits_sha256 = _canonical_logits_sha256(verifier) + self._before_forward() + verifier.append_accepted_tokens([token_id]) + return { + **_state(verifier), + "token_id": token_id, + "input_logits_sha256": input_logits_sha256, + } + raise ValueError(f"unknown decode worker operation {operation!r}") + + +def _serve_worker( + config: dict[str, Any], + socket_path: str, + factory_path: str, +) -> None: + runtime = _WorkerRuntime(config, factory_path) + path = Path(socket_path) + path.unlink(missing_ok=True) + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + server.bind(socket_path) + os.chmod(socket_path, 0o600) + server.listen(1) + while True: + connection, _ = server.accept() + with connection: + while True: + try: + request, payload = _recv_frame(connection) + except EOFError: + break + request_id = str(request.get("request_id", "")) + try: + if int(request.get("version", -1)) != PROTOCOL_VERSION: + raise ValueError("decode worker protocol version mismatch") + result = runtime.dispatch( + str(request["operation"]), + str(request.get("session_id", "")), + dict(request.get("arguments", {})), + payload, + ) + response_payload = b"" + if isinstance(result, tuple): + result, response_payload = result + _send_frame(connection, { + "version": PROTOCOL_VERSION, + "request_id": request_id, + "ok": True, + "result": result, + }, response_payload) + except BaseException as exc: + _send_frame(connection, { + "version": PROTOCOL_VERSION, + "request_id": request_id, + "ok": False, + "error_type": type(exc).__name__, + "error": str(exc), + }) + finally: + server.close() + path.unlink(missing_ok=True) + + +class DecodeWorkerClient: + """Serialized router-side process owner and recovery coordinator.""" + + def __init__(self, config: DecodeWorkerConfig) -> None: + if config.request_timeout_s <= 0 or config.startup_timeout_s <= 0: + raise ValueError("decode worker timeouts must be > 0") + self.config = config + requested_socket = config.socket_path or str( + Path(tempfile.gettempdir()) / f"kakeya-decode-{uuid.uuid4().hex}.sock" + ) + # Darwin's sockaddr_un path is only 104 bytes. Pytest temp roots and + # launchd working directories can exceed it, so deterministically + # shorten an explicit path while preserving per-path uniqueness. + self.socket_path = ( + requested_socket + if len(os.fsencode(requested_socket)) < 100 + else str( + Path(tempfile.gettempdir()) + / ( + "kdw-" + + hashlib.sha256(os.fsencode(requested_socket)).hexdigest()[:24] + + ".sock" + ) + ) + ) + self._process: multiprocessing.Process | None = None + self._socket: socket.socket | None = None + self._lock = threading.RLock() + self._checkpoints: dict[str, ProofCheckpoint] = {} + self._proxies: dict[str, DecodeWorkerSession] = {} + self._generation = 0 + self._restored_generation: dict[str, int] = {} + self.restart_count = 0 + self._health: dict[str, Any] = {} + self.start() + + @property + def pid(self) -> int | None: + return self._process.pid if self._process is not None else None + + @property + def dimensions(self) -> tuple[int, int, int]: + return ( + int(self._health["num_layers"]), + int(self._health["num_kv_heads"]), + int(self._health["head_dim"]), + ) + + def start(self) -> None: + with self._lock: + self._stop_locked() + Path(self.socket_path).unlink(missing_ok=True) + ctx = multiprocessing.get_context("spawn") + payload = { + "model_id": self.config.model_id, + "sink_size": self.config.sink_size, + "window_size": self.config.window_size, + } + self._process = ctx.Process( + target=_serve_worker, + args=(payload, self.socket_path, self.config.verifier_factory), + name="kakeya-mlx-decode-worker", + daemon=True, + ) + self._process.start() + deadline = time.monotonic() + self.config.startup_timeout_s + last_error: BaseException | None = None + while time.monotonic() < deadline: + if not self._process.is_alive(): + raise DecodeWorkerUnavailable( + f"decode worker exited during startup " + f"(exitcode={self._process.exitcode})" + ) + try: + connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connection.connect(self.socket_path) + self._socket = connection + self._health = self._request_locked( + "Health", "", {}, b"", timeout_s=5.0 + ) + self._generation += 1 + return + except (FileNotFoundError, ConnectionRefusedError, OSError) as exc: + last_error = exc + time.sleep(0.02) + self._stop_locked() + raise DecodeWorkerUnavailable( + f"decode worker startup timed out: {last_error}" + ) + + def health(self) -> dict[str, Any]: + with self._lock: + self._health = self._request_locked("Health", "", {}, b"") + return dict(self._health) + + def inject_hang(self, expected_worker_pid: int) -> dict[str, Any]: + with self._lock: + return self._request_locked( + "InjectHang", + "", + {"expected_worker_pid": int(expected_worker_pid)}, + b"", + ) + + def acceptance_snapshot( + self, + *, + runtime_pid: int, + process_footprint_bytes: int, + active_sessions: int, + active_generations: int, + ) -> dict[str, Any]: + return { + "runtime_pid": int(runtime_pid), + "worker_pid": int(self.pid or 0), + "worker_restart_count": int(self.restart_count), + "process_footprint_bytes": int(process_footprint_bytes), + "active_sessions": int(active_sessions), + "active_generations": int(active_generations), + } + + def kv_restore_parity(self, prompt_token_ids: list[int]) -> dict[str, Any]: + """Exercise an Allens snapshot plus router proof-checkpoint restore.""" + tokens = [int(token) for token in prompt_token_ids] + if not tokens: + raise ValueError("kv_restore_parity requires prompt_token_ids") + from inference_engine.distributed.capability import CacheCompatibility + + compatibility = CacheCompatibility( + model_id=self.config.model_id, + sink_size=self.config.sink_size, + window_size=self.config.window_size, + ) + compat = asdict(compatibility) + session_id = f"acceptance-{uuid.uuid4().hex}" + session = self.get(session_id) + try: + session.prefill(tokens) + with self._lock: + baseline, snapshot = self._request_with_payload_locked( + "ExportSnapshot", + session_id, + {"compatibility": compat}, + b"", + ) + checkpoint = self._checkpoints[session_id] + checkpoint.snapshot = snapshot + checkpoint.compatibility = compat + checkpoint.replay_token_ids = [] + checkpoint.initialized = True + self._restored_generation[session_id] = self._generation + self.hard_kill() + restored = self._session_request( + session_id, + "GenerateStep", + {}, + ) + return { + "baseline_first_token_id": int(baseline["first_token_id"]), + "restored_first_token_id": int(restored["token_id"]), + "baseline_logits_sha256": str(baseline["logits_sha256"]), + "restored_logits_sha256": str( + restored["input_logits_sha256"] + ), + "restore_source": "allens_kv+proof_checkpoint", + } + finally: + self.close_session(session_id) + + def get(self, session_id: str) -> "DecodeWorkerSession": + with self._lock: + proxy = self._proxies.get(session_id) + if proxy is None: + proxy = DecodeWorkerSession(self, session_id) + self._proxies[session_id] = proxy + self._checkpoints.setdefault(session_id, ProofCheckpoint()) + return proxy + + session = get + + def close_session(self, session_id: str) -> None: + with self._lock: + try: + self._request_locked("Close", session_id, {}, b"", timeout_s=2.0) + except DecodeWorkerError: + pass + self._checkpoints.pop(session_id, None) + self._proxies.pop(session_id, None) + self._restored_generation.pop(session_id, None) + + def k_seq_length(self, session: Any) -> int: + return self.get(session.session_id).k_seq_length(session) + + def kv_live_bytes(self, session: Any) -> int: + return self.get(session.session_id).kv_live_bytes(session) + + def memory_bytes(self) -> tuple[int, int, int]: + health = self.health() + return ( + int(health.get("mlx_active_bytes", 0)), + int(health.get("mlx_cache_bytes", 0)), + int(health.get("mlx_peak_bytes", 0)), + ) + + def hard_kill(self) -> None: + with self._lock: + self._stop_locked(hard=True) + + def close(self) -> None: + with self._lock: + self._stop_locked() + Path(self.socket_path).unlink(missing_ok=True) + + def _stop_locked(self, *, hard: bool = False) -> None: + if self._socket is not None: + try: + self._socket.close() + except OSError: + pass + self._socket = None + process = self._process + self._process = None + if process is not None and process.is_alive(): + if hard and process.pid: + os.kill(process.pid, signal.SIGKILL) + else: + process.terminate() + process.join(timeout=2.0) + if process.is_alive() and process.pid: + os.kill(process.pid, signal.SIGKILL) + process.join(timeout=1.0) + + def _request_locked( + self, + operation: str, + session_id: str, + arguments: dict[str, Any], + payload: bytes, + *, + timeout_s: float | None = None, + cancel_event: threading.Event | None = None, + ) -> dict[str, Any]: + result, _ = self._request_with_payload_locked( + operation, + session_id, + arguments, + payload, + timeout_s=timeout_s, + cancel_event=cancel_event, + ) + return result + + def _request_with_payload_locked( + self, + operation: str, + session_id: str, + arguments: dict[str, Any], + payload: bytes, + *, + timeout_s: float | None = None, + cancel_event: threading.Event | None = None, + ) -> tuple[dict[str, Any], bytes]: + if self._socket is None: + raise DecodeWorkerUnavailable("decode worker is not connected") + request_id = uuid.uuid4().hex + try: + _send_frame(self._socket, { + "version": PROTOCOL_VERSION, + "request_id": request_id, + "operation": operation, + "session_id": session_id, + "arguments": arguments, + }, payload) + deadline = time.monotonic() + ( + self.config.request_timeout_s if timeout_s is None else timeout_s + ) + while True: + if cancel_event is not None and cancel_event.is_set(): + self._stop_locked(hard=True) + raise OperationCancelledError( + f"decode worker {operation} cancelled" + ) + remaining = deadline - time.monotonic() + if remaining <= 0: + self._stop_locked(hard=True) + raise DecodeWorkerUnavailable( + f"decode worker {operation} timed out" + ) + ready, _, _ = select.select( + [self._socket], [], [], min(0.05, remaining) + ) + if ready: + break + response, response_payload = _recv_frame(self._socket) + except OperationCancelledError: + raise + except (OSError, EOFError, ValueError, json.JSONDecodeError) as exc: + self._stop_locked(hard=True) + raise DecodeWorkerUnavailable(str(exc)) from exc + if ( + response.get("version") != PROTOCOL_VERSION + or response.get("request_id") != request_id + ): + self._stop_locked(hard=True) + raise DecodeWorkerUnavailable("decode worker response correlation failed") + if not response.get("ok"): + error_type = str(response.get("error_type", "RemoteError")) + message = str(response.get("error", "")) + if error_type == "ValueError": + raise ValueError(message) + raise DecodeWorkerRemoteError(error_type, message) + return dict(response.get("result", {})), response_payload + + def _session_request( + self, + session_id: str, + operation: str, + arguments: dict[str, Any], + payload: bytes = b"", + *, + cancel_event: threading.Event | None = None, + ) -> dict[str, Any]: + with self._lock: + checkpoint = self._checkpoints.get(session_id) + if ( + operation not in {"Init", "ImportSnapshot", "Close"} + and checkpoint is not None + and checkpoint.initialized + and self._restored_generation.get(session_id) != self._generation + ): + self._restore_locked(session_id) + for attempt in range(2): + try: + return self._request_locked( + operation, + session_id, + arguments, + payload, + cancel_event=cancel_event, + ) + except OperationCancelledError: + raise + except DecodeWorkerUnavailable: + if attempt: + raise + self.restart_count += 1 + self.start() + self._restore_locked(session_id) + raise AssertionError("unreachable") + + def _restore_locked(self, session_id: str) -> None: + checkpoint = self._checkpoints.get(session_id) + if checkpoint is None or not checkpoint.initialized: + return + initial_tokens = ( + checkpoint.replay_token_ids if checkpoint.snapshot is None else [] + ) + self._request_locked( + "Init", session_id, {"token_ids": initial_tokens}, b"" + ) + if checkpoint.snapshot is not None: + self._request_locked( + "ImportSnapshot", + session_id, + {"compatibility": checkpoint.compatibility or {}}, + checkpoint.snapshot, + ) + if checkpoint.replay_token_ids: + self._request_locked( + "Append", + session_id, + {"token_ids": checkpoint.replay_token_ids}, + b"", + ) + self._restored_generation[session_id] = self._generation + + def _mark_current(self, session_id: str) -> None: + with self._lock: + self._restored_generation[session_id] = self._generation + + +class DecodeWorkerSession: + """Verifier-shaped proxy used by existing append/generate coordinators.""" + + is_decode_worker_proxy = True + + def __init__(self, client: DecodeWorkerClient, session_id: str) -> None: + self.client = client + self.session_id = session_id + self.cached_token_sequence: list[int] = [] + self.next_global_position = 0 + self._kv_live_bytes = 0 + + def _apply(self, state: dict[str, Any]) -> dict[str, Any]: + self.cached_token_sequence = [ + int(token) for token in state.get("cached_token_ids", ()) + ] + self.next_global_position = int( + state.get("next_global_position", self.next_global_position) + ) + self._kv_live_bytes = int(state.get("kv_live_bytes", 0)) + return state + + def prefill( + self, + prompt_ids: list[int], + cancel_event: threading.Event | None = None, + ) -> None: + tokens = [int(token) for token in prompt_ids] + state = self.client._session_request( + self.session_id, + "Init", + {"token_ids": tokens}, + cancel_event=cancel_event, + ) + checkpoint = self.client._checkpoints[self.session_id] + checkpoint.snapshot = None + checkpoint.compatibility = None + checkpoint.replay_token_ids = list(tokens) + checkpoint.initialized = True + self.client._mark_current(self.session_id) + self._apply(state) + + def append_accepted_tokens( + self, + tokens: list[int], + cancel_event: threading.Event | None = None, + ) -> None: + committed = [int(token) for token in tokens] + state = self.client._session_request( + self.session_id, + "Append", + {"token_ids": committed}, + cancel_event=cancel_event, + ) + self.client._checkpoints[self.session_id].replay_token_ids.extend(committed) + self._apply(state) + + def generate_step( + self, + cancel_event: threading.Event | None = None, + ) -> int: + state = self.client._session_request( + self.session_id, + "GenerateStep", + {}, + cancel_event=cancel_event, + ) + token_id = int(state["token_id"]) + self.client._checkpoints[self.session_id].replay_token_ids.append(token_id) + self._apply(state) + return token_id + + def import_snapshot( + self, + payload: bytes, + compatibility: Any, + ) -> dict[str, Any]: + compat = asdict(compatibility) + # Ensure a worker-side session exists before importing its cache. + self.client._session_request( + self.session_id, "Init", {"token_ids": []} + ) + state = self.client._session_request( + self.session_id, + "ImportSnapshot", + {"compatibility": compat}, + bytes(payload), + ) + checkpoint = self.client._checkpoints[self.session_id] + checkpoint.snapshot = bytes(payload) + checkpoint.compatibility = compat + checkpoint.replay_token_ids = [] + checkpoint.initialized = True + self.client._mark_current(self.session_id) + return self._apply(state) + + def reset(self) -> None: + state = self.client._session_request( + self.session_id, "Init", {"token_ids": []} + ) + checkpoint = self.client._checkpoints[self.session_id] + checkpoint.snapshot = None + checkpoint.compatibility = None + checkpoint.replay_token_ids = [] + checkpoint.initialized = True + self.client._mark_current(self.session_id) + self._apply(state) + + def k_seq_length(self, _session: Any) -> int: + return len(self.cached_token_sequence) + + def kv_live_bytes(self, _session: Any) -> int: + return self._kv_live_bytes diff --git a/inference_engine/backends/mlx/decode_worker_acceptance.py b/inference_engine/backends/mlx/decode_worker_acceptance.py new file mode 100644 index 00000000..5ea194e4 --- /dev/null +++ b/inference_engine/backends/mlx/decode_worker_acceptance.py @@ -0,0 +1,148 @@ +"""Local-only acceptance controls for the isolated MLX decode worker.""" + +from __future__ import annotations + +import hashlib +import os +import socket +import tempfile +import threading +from pathlib import Path +from typing import Any, Callable + +from inference_engine.backends.mlx.decode_worker import ( + DecodeWorkerClient, + _recv_frame, + _send_frame, +) + + +def normalize_control_socket_path(path: str) -> str: + if not path: + return "" + if len(os.fsencode(path)) < 100: + return path + return str( + Path(tempfile.gettempdir()) + / f"kdw-accept-{hashlib.sha256(os.fsencode(path)).hexdigest()[:20]}.sock" + ) + + +class DecodeWorkerAcceptanceServer: + """Serve fault injection and observations on a mode-0600 UDS.""" + + def __init__( + self, + *, + socket_path: str, + worker: DecodeWorkerClient, + runtime_pid: int, + active_sessions: Callable[[], int], + active_generations: Callable[[], int], + process_footprint: Callable[[], int], + ) -> None: + normalized = normalize_control_socket_path(socket_path) + if not normalized: + raise ValueError("acceptance control socket path must not be empty") + self.socket_path = normalized + self._worker = worker + self._runtime_pid = int(runtime_pid) + self._active_sessions = active_sessions + self._active_generations = active_generations + self._process_footprint = process_footprint + self._server: socket.socket | None = None + self._thread: threading.Thread | None = None + self._stop = threading.Event() + + def start(self) -> None: + path = Path(self.socket_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.unlink(missing_ok=True) + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + server.bind(self.socket_path) + os.chmod(self.socket_path, 0o600) + server.listen(8) + server.settimeout(0.2) + self._server = server + self._thread = threading.Thread( + target=self._serve, + name="kakeya-decode-acceptance", + daemon=True, + ) + self._thread.start() + + def close(self) -> None: + self._stop.set() + server = self._server + self._server = None + if server is not None: + server.close() + thread = self._thread + self._thread = None + if thread is not None: + thread.join(timeout=2.0) + Path(self.socket_path).unlink(missing_ok=True) + + def _serve(self) -> None: + while not self._stop.is_set(): + server = self._server + if server is None: + return + try: + connection, _ = server.accept() + except socket.timeout: + continue + except OSError: + return + with connection: + operation = "" + try: + request, _ = _recv_frame(connection) + if int(request.get("schema_version", -1)) != 1: + raise ValueError("acceptance protocol version mismatch") + operation = str(request.get("operation", "")) + data = self.dispatch( + operation, + dict(request.get("payload", {})), + ) + _send_frame( + connection, + { + "schema_version": 1, + "operation": operation, + "ok": True, + "data": data, + }, + ) + except BaseException as exc: + try: + _send_frame( + connection, + { + "schema_version": 1, + "operation": operation, + "ok": False, + "error": f"{type(exc).__name__}: {exc}", + "data": {}, + }, + ) + except OSError: + pass + + def dispatch(self, operation: str, payload: dict[str, Any]) -> dict[str, Any]: + if operation == "snapshot": + return self._worker.acceptance_snapshot( + runtime_pid=self._runtime_pid, + process_footprint_bytes=self._process_footprint(), + active_sessions=self._active_sessions(), + active_generations=self._active_generations(), + ) + if operation == "inject_hang": + if payload.get("phase") != "next_forward": + raise ValueError("inject_hang phase must be 'next_forward'") + return self._worker.inject_hang(int(payload["expected_worker_pid"])) + if operation == "kv_restore_parity": + return self._worker.kv_restore_parity( + [int(token) for token in payload["prompt_token_ids"]] + ) + raise ValueError(f"unknown acceptance operation {operation!r}") diff --git a/inference_engine/backends/mlx/verifier.py b/inference_engine/backends/mlx/verifier.py index 4268fee0..89b7e937 100644 --- a/inference_engine/backends/mlx/verifier.py +++ b/inference_engine/backends/mlx/verifier.py @@ -13,11 +13,11 @@ See `cache.py`'s module docstring for why direct mutation of ``mlx_lm.models.cache.KVCache`` was the source of the MLX-1b divergence-after-trim bug. - * Returned logits are converted from ``mx.array`` to - ``torch.Tensor`` at the API boundary so the speculative loop's - ``torch.argmax`` / ``.item()`` work unchanged. The conversion is - small-tensor (next-token slice or block-of-L slice), single-digit - ms. + * Greedy decode keeps the last logits row on-device and reduces it + with ``mx.argmax``. The legacy ``next_token_logits`` property + materializes that row as Torch only when a caller explicitly asks + for it. Speculative ``forward_block`` retains its full ``[L,V]`` + Torch result and semantics unchanged. The module imports ``mlx.core`` and ``mlx_lm`` at top level. Non-Apple- Silicon hosts cannot load this file — that is the intended behavior. @@ -90,7 +90,12 @@ def __init__(self, config: Optional[VerifierConfig] = None) -> None: # the global position; redundant but mirrors the PyTorch class). self.cache_logical_size: int = 0 self.next_global_position: int = 0 - self.next_token_logits: Optional[torch.Tensor] = None + # Last-position logits stay on MLX for the primary greedy path. + # The Torch value is a lazy compatibility cache for callers + # (notably speculative decoding) that still consume full logits. + self._next_token_logits_mx: Optional["mx.array"] = None + self._next_token_id_mx: Optional["mx.array"] = None + self._next_token_logits_torch: Optional[torch.Tensor] = None # Parallel record of the token id at every K/V cache slot, in # the same physical order as ``self.cache[*].keys``. See the CPU # verifier for the full motivation; in short, this is required @@ -143,8 +148,60 @@ def __init__(self, config: Optional[VerifierConfig] = None) -> None: for num_kv_heads, head_dim, _layer_type in geometry ) + def spawn(self) -> "MLXSinkWindowVerifier": + """Create an empty session verifier sharing this instance's model. + + Decode-worker mode owns one loaded model and one lightweight verifier + adapter per session. Constructing through ``__init__`` would call + ``mlx_lm.load`` for every session, so copy only immutable model + metadata and initialize independent KV/logit state here. + """ + child = object.__new__(type(self)) + child.config = self.config + child.model = self.model + child.tokenizer = self.tokenizer + child._mx_dtype = self._mx_dtype + child.cache = None + child.cache_logical_size = 0 + child.next_global_position = 0 + child._next_token_logits_mx = None + child._next_token_id_mx = None + child._next_token_logits_torch = None + child.cached_token_sequence = [] + child.quantization = self.quantization + child.stats = VerifierStats( + weight_bytes=self.quantization.total_weight_bytes, + ) + child._bytes_per_kv_token = self._bytes_per_kv_token + return child + # ---------------------------- public API ---------------------------- # + @property + def next_token_logits(self) -> Optional[torch.Tensor]: + """Return last-position logits as Torch, materializing lazily. + + Primary greedy decode uses :meth:`greedy_next_token_id` and + therefore never crosses the full vocabulary row to Torch. + """ + if ( + self._next_token_logits_torch is None + and self._next_token_logits_mx is not None + ): + self._next_token_logits_torch = mx_to_torch( + self._next_token_logits_mx + ) + return self._next_token_logits_torch + + @next_token_logits.setter + def next_token_logits(self, value: Optional[torch.Tensor]) -> None: + # An external Torch assignment (the legacy speculative path) + # becomes authoritative; retaining an older MLX row would make + # a later greedy argmax stale. + self._next_token_logits_torch = value + self._next_token_logits_mx = None + self._next_token_id_mx = None + def reset(self) -> None: self.cache = make_sink_window_cache( self.model, @@ -153,7 +210,9 @@ def reset(self) -> None: ) self.cache_logical_size = 0 self.next_global_position = 0 - self.next_token_logits = None + self._next_token_logits_mx = None + self._next_token_id_mx = None + self._next_token_logits_torch = None self.cached_token_sequence = [] self._assert_cache_invariant_1() @@ -164,15 +223,17 @@ def prefill(self, prompt_ids: List[int]) -> None: L = len(prompt_ids) arr = mx.array([prompt_ids], dtype=mx.int32) logits_mx = self.model(arr, cache=self.cache) - mx.eval(logits_mx) if logits_mx.ndim != 3 or int(logits_mx.shape[0]) != 1: # pragma: no cover - mlx_lm contract raise RuntimeError( f"prefill: model returned logits of shape {tuple(logits_mx.shape)} " "(expected [1, T, V])" ) - self._record_peak_activation(logits_mx) - self.next_token_logits = mx_to_torch(logits_mx[0, -1]) + # Slice before evaluation. MLX can then materialize just [V], + # rather than retaining the prompt-sized [1,T,V] output. + last_logits_mx = logits_mx[0, -1] + self._record_peak_activation(last_logits_mx) + self._set_next_token_logits_mx(last_logits_mx) self.next_global_position = L self.cache_logical_size = self._cache_buffer_size() # Compute the post-trim parallel token sequence directly. The @@ -218,6 +279,57 @@ def forward_block(self, tokens: List[int]) -> torch.Tensor: self._assert_cache_invariant_1() return block_logits + def greedy_next_token_id(self) -> int: + """Return the greedy token using an on-device MLX reduction. + + Only the scalar result crosses the synchronization boundary. + A Torch-only row supplied by the legacy speculative path still + works, but primary prefill/append paths always retain an MLX row. + """ + if self._next_token_id_mx is not None: + return int(self._next_token_id_mx.item()) + if self._next_token_logits_torch is not None: + return int(torch.argmax(self._next_token_logits_torch).item()) + raise RuntimeError("Verifier has no next-token logits.") + + def append_accepted_tokens(self, tokens: List[int]) -> None: + """Forward and commit an all-accepted non-speculative block. + + This is the last-logits-only path for primary AppendTokens and + single-token greedy decode. It intentionally does not replace + :meth:`forward_block`, whose full ``[L,V]`` result and + commit/truncate contract remain the speculative API. + """ + if self.cache is None: + raise RuntimeError("Verifier not prefilled.") + if not tokens: + raise ValueError("tokens must be non-empty") + + L = len(tokens) + arr = mx.array([tokens], dtype=mx.int32) + logits_mx = self.model(arr, cache=self.cache) + if ( + logits_mx.ndim != 3 + or int(logits_mx.shape[0]) != 1 + or int(logits_mx.shape[1]) != L + ): + raise RuntimeError( # pragma: no cover - mlx_lm contract + f"append_accepted_tokens: expected logits [1,{L},V], got " + f"{tuple(logits_mx.shape)}" + ) + + last_logits_mx = logits_mx[0, -1] + self._record_peak_activation(last_logits_mx) + self.cache_logical_size = self._cache_buffer_size() + extended = self.cached_token_sequence + list(tokens) + self.cached_token_sequence = self._sink_window_slice(extended) + self.next_global_position += L + self._set_next_token_logits_mx(last_logits_mx) + self._record_peak_kv() + self.stats.forward_calls += 1 + self.stats.tokens_consumed += L + self._assert_cache_invariant_1() + def commit_or_truncate(self, forwarded: int, accepted: int) -> None: if accepted < 0 or accepted > forwarded: raise ValueError("accepted must satisfy 0 <= accepted <= forwarded") @@ -240,10 +352,10 @@ def commit_or_truncate(self, forwarded: int, accepted: int) -> None: self._assert_cache_invariant_1() def append_token(self, token_id: int) -> torch.Tensor: - logits = self.forward_block([token_id]) - self.commit_or_truncate(forwarded=1, accepted=1) - self.next_token_logits = logits[-1].clone() - return self.next_token_logits + self.append_accepted_tokens([token_id]) + logits = self.next_token_logits + assert logits is not None + return logits # ---------------- CacheInspector protocol (ADR 0008 PR-A3b) ---------------- # # Mirrors the CPU verifier's CacheInspector implementation. The session @@ -275,6 +387,16 @@ def kv_live_bytes(self, session: object) -> int: # --------------------------- internals --------------------------- # + def _set_next_token_logits_mx(self, logits_mx: "mx.array") -> None: + """Install logits plus its fused on-device greedy reduction.""" + token_mx = mx.argmax(logits_mx) + # Evaluate the row and dependent scalar together. This avoids a + # second graph launch/synchronization at the next token boundary. + mx.eval(logits_mx, token_mx) + self._next_token_logits_mx = logits_mx + self._next_token_id_mx = token_mx + self._next_token_logits_torch = None + def _cache_buffer_size(self) -> int: """Return the actual seq dim of the cache buffer (post-trim).""" if self.cache is None: diff --git a/inference_engine/bench/primary_decode_acceptance.py b/inference_engine/bench/primary_decode_acceptance.py new file mode 100644 index 00000000..6c09116a --- /dev/null +++ b/inference_engine/bench/primary_decode_acceptance.py @@ -0,0 +1,491 @@ +"""Pure result and gate logic for Primary decode Mac acceptance. + +The hardware runner lives in ``scripts/bench_agentic``. This module has no +MLX, gRPC, or macOS imports so report decisions remain unit-testable in CI. +""" + +from __future__ import annotations + +import json +import math +import os +import statistics +import xml.etree.ElementTree as ET +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Iterable, Mapping, Optional + +from inference_engine.bench.session_long_run import aggregate_run + + +SCHEMA_ID = "https://kakeya.dev/schemas/primary-decode-acceptance-v1.json" +GIB = 1 << 30 + + +@dataclass(frozen=True) +class GateResult: + """One independently reportable acceptance decision.""" + + gate: str + status: str + summary: str + metrics: dict[str, Any] = field(default_factory=dict) + thresholds: dict[str, Any] = field(default_factory=dict) + evidence: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if self.status not in {"passed", "failed", "error", "skipped"}: + raise ValueError(f"invalid gate status: {self.status}") + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def _gate( + name: str, + passed: bool, + summary: str, + *, + metrics: Optional[dict[str, Any]] = None, + thresholds: Optional[dict[str, Any]] = None, + evidence: Optional[dict[str, Any]] = None, +) -> GateResult: + return GateResult( + gate=name, + status="passed" if passed else "failed", + summary=summary, + metrics=metrics or {}, + thresholds=thresholds or {}, + evidence=evidence or {}, + ) + + +def footprint_gate( + *, + fresh_bytes: int, + final_bytes: int, + sessions_completed: int, + required_sessions: int = 100, + max_growth_bytes: int = 2 * GIB, + active_sessions: int = 0, +) -> GateResult: + """Evaluate the 100 sequential-session process-footprint gate.""" + growth = final_bytes - fresh_bytes + passed = ( + sessions_completed == required_sessions + and growth <= max_growth_bytes + and active_sessions == 0 + ) + return _gate( + "sequential_session_footprint", + passed, + ( + f"{sessions_completed}/{required_sessions} sessions; " + f"footprint growth {growth} bytes; active sessions {active_sessions}" + ), + metrics={ + "fresh_process_footprint_bytes": fresh_bytes, + "final_process_footprint_bytes": final_bytes, + "growth_bytes": growth, + "sessions_completed": sessions_completed, + "active_sessions": active_sessions, + }, + thresholds={ + "required_sessions": required_sessions, + "max_growth_bytes": max_growth_bytes, + "required_active_sessions": 0, + }, + ) + + +def _bucket_medians(values: list[int], bucket_count: int = 8) -> list[float]: + if not values: + return [] + size = max(1, math.ceil(len(values) / bucket_count)) + return [ + float(statistics.median(values[start : start + size])) + for start in range(0, len(values), size) + ] + + +def monotonic_memory_growth( + values: Iterable[int], + *, + minimum_growth_bytes: int = 64 << 20, +) -> tuple[bool, list[float]]: + """Detect sustained bucket-to-bucket footprint growth, ignoring tiny noise.""" + buckets = _bucket_medians([int(value) for value in values]) + monotonic = ( + len(buckets) >= 4 + and all(right >= left for left, right in zip(buckets, buckets[1:])) + and buckets[-1] - buckets[0] >= minimum_growth_bytes + ) + return monotonic, buckets + + +def endurance_gate( + records: list[dict[str, Any]], + *, + duration_s: float, + target_duration_s: float = 4 * 60 * 60, + max_p50_drift_s: float = 5.0, + minimum_memory_growth_bytes: int = 64 << 20, +) -> GateResult: + """Evaluate mixed long/short prompt endurance records. + + ``aggregate_run`` is intentionally reused from the existing gRPC + long-session benchmark for latency/KV aggregation. + """ + aggregate = aggregate_run( + records, + duration_s=duration_s, + drift_threshold_s=max_p50_drift_s, + ) + successes = [record for record in records if record.get("ok")] + prompt_kinds = {str(record.get("prompt_kind")) for record in successes} + footprints = [ + int(record["process_footprint_bytes"]) + for record in successes + if record.get("process_footprint_bytes") is not None + ] + monotonic, memory_buckets = monotonic_memory_growth( + footprints, + minimum_growth_bytes=minimum_memory_growth_bytes, + ) + drift = aggregate["latency_drift_p50_s"] + passed = ( + duration_s >= target_duration_s + and aggregate["n_errors"] == 0 + and {"short", "long"}.issubset(prompt_kinds) + and not monotonic + and drift is not None + and drift <= max_p50_drift_s + ) + return _gate( + "mixed_prompt_endurance", + passed, + ( + f"{duration_s:.1f}/{target_duration_s:.1f}s; " + f"errors={aggregate['n_errors']}; monotonic_memory={monotonic}; " + f"p50_drift_s={drift}" + ), + metrics={ + "duration_s": duration_s, + "prompt_kinds": sorted(prompt_kinds), + "monotonic_memory_growth": monotonic, + "memory_bucket_medians_bytes": memory_buckets, + "aggregate": aggregate, + }, + thresholds={ + "target_duration_s": target_duration_s, + "max_errors": 0, + "required_prompt_kinds": ["short", "long"], + "max_p50_drift_s": max_p50_drift_s, + "minimum_monotonic_growth_bytes": minimum_memory_growth_bytes, + }, + evidence={"records": records}, + ) + + +def disconnect_gate( + *, + cancellation_elapsed_s: float, + active_generations: int, + active_sessions: int, + limit_s: float = 5.0, +) -> GateResult: + passed = ( + cancellation_elapsed_s <= limit_s + and active_generations == 0 + and active_sessions == 0 + ) + return _gate( + "disconnect_cancellation", + passed, + ( + f"settled in {cancellation_elapsed_s:.3f}s; " + f"active_generations={active_generations}; " + f"active_sessions={active_sessions}" + ), + metrics={ + "cancellation_elapsed_s": cancellation_elapsed_s, + "active_generations": active_generations, + "active_sessions": active_sessions, + }, + thresholds={ + "max_cancellation_elapsed_s": limit_s, + "required_active_generations": 0, + "required_active_sessions": 0, + }, + ) + + +def hang_recycle_gate( + *, + injection_accepted: bool, + recycle_elapsed_s: float, + worker_pid_before: int, + worker_pid_after: int, + restart_count_before: int, + restart_count_after: int, + limit_s: float = 120.0, +) -> GateResult: + recycled = ( + worker_pid_before != worker_pid_after + and restart_count_after > restart_count_before + ) + passed = injection_accepted and recycled and recycle_elapsed_s <= limit_s + return _gate( + "injected_hang_recycle", + passed, + ( + f"accepted={injection_accepted}; recycled={recycled}; " + f"elapsed={recycle_elapsed_s:.3f}s" + ), + metrics={ + "injection_accepted": injection_accepted, + "recycle_elapsed_s": recycle_elapsed_s, + "worker_pid_before": worker_pid_before, + "worker_pid_after": worker_pid_after, + "restart_count_before": restart_count_before, + "restart_count_after": restart_count_after, + }, + thresholds={"max_recycle_elapsed_s": limit_s}, + ) + + +def kv_restore_gate( + *, + baseline_first_token_id: int, + restored_first_token_id: int, + baseline_logits_sha256: str, + restored_logits_sha256: str, + restore_source: str, +) -> GateResult: + token_match = baseline_first_token_id == restored_first_token_id + logits_match = ( + bool(baseline_logits_sha256) + and baseline_logits_sha256 == restored_logits_sha256 + ) + source_ok = restore_source == "allens_kv+proof_checkpoint" + return _gate( + "kv_restore_parity", + token_match and logits_match and source_ok, + ( + f"token_match={token_match}; logits_match={logits_match}; " + f"restore_source={restore_source}" + ), + metrics={ + "baseline_first_token_id": baseline_first_token_id, + "restored_first_token_id": restored_first_token_id, + "token_match": token_match, + "baseline_logits_sha256": baseline_logits_sha256, + "restored_logits_sha256": restored_logits_sha256, + "logits_match": logits_match, + "restore_source": restore_source, + }, + thresholds={ + "require_token_match": True, + "require_logits_sha256_match": True, + "required_restore_source": "allens_kv+proof_checkpoint", + }, + ) + + +def _percentile(values: list[float], quantile: float) -> Optional[float]: + if not values: + return None + ordered = sorted(float(value) for value in values) + rank = quantile * (len(ordered) - 1) + low = int(rank) + high = min(low + 1, len(ordered) - 1) + return ordered[low] + (ordered[high] - ordered[low]) * (rank - low) + + +def latency_summary(samples_s: Iterable[float]) -> dict[str, Any]: + values = [float(value) for value in samples_s] + return { + "sample_count": len(values), + "p50_decode_latency_s": _percentile(values, 0.50), + "p95_decode_latency_s": _percentile(values, 0.95), + } + + +def latency_baseline_from_report(payload: Mapping[str, Any]) -> dict[str, Any]: + """Read either this harness schema or the existing MLX verifier report.""" + summary = payload.get("latency") + if isinstance(summary, Mapping): + return { + "p50_decode_latency_s": float(summary["p50_decode_latency_s"]), + "p95_decode_latency_s": float(summary["p95_decode_latency_s"]), + "source": "primary_decode_acceptance", + } + mlx = payload.get("mlx") + if isinstance(mlx, Mapping): + count = int(mlx["n_tokens"]) + if count <= 0: + raise ValueError("MLX benchmark baseline has no generated tokens") + per_token = float(mlx["generation_time_s"]) / count + return { + "p50_decode_latency_s": per_token, + "p95_decode_latency_s": per_token, + "source": "bench_mlx_verifier_mean_per_token", + } + raise ValueError("unsupported latency baseline report") + + +def latency_gate( + samples_s: Iterable[float], + *, + baseline: Mapping[str, Any], + long_run_p50_drift_s: float, + max_regression_fraction: float = 0.15, + max_long_run_p50_drift_s: float = 5.0, +) -> GateResult: + current = latency_summary(samples_s) + current_p50 = current["p50_decode_latency_s"] + current_p95 = current["p95_decode_latency_s"] + baseline_p50 = float(baseline["p50_decode_latency_s"]) + baseline_p95 = float(baseline["p95_decode_latency_s"]) + if current_p50 is None or current_p95 is None: + return GateResult( + gate="decode_latency_regression", + status="failed", + summary="no decode latency samples", + metrics=current, + ) + if baseline_p50 <= 0 or baseline_p95 <= 0: + raise ValueError("latency baseline values must be > 0") + p50_regression = current_p50 / baseline_p50 - 1.0 + p95_regression = current_p95 / baseline_p95 - 1.0 + passed = ( + p50_regression <= max_regression_fraction + and p95_regression <= max_regression_fraction + and long_run_p50_drift_s <= max_long_run_p50_drift_s + ) + return _gate( + "decode_latency_regression", + passed, + ( + f"p50 regression={p50_regression:.2%}; " + f"p95 regression={p95_regression:.2%}; " + f"long-run drift={long_run_p50_drift_s:.3f}s" + ), + metrics={ + **current, + "baseline_p50_decode_latency_s": baseline_p50, + "baseline_p95_decode_latency_s": baseline_p95, + "p50_regression_fraction": p50_regression, + "p95_regression_fraction": p95_regression, + "long_run_p50_drift_s": long_run_p50_drift_s, + "baseline_source": baseline.get("source", "unknown"), + }, + thresholds={ + "max_regression_fraction": max_regression_fraction, + "max_long_run_p50_drift_s": max_long_run_p50_drift_s, + }, + ) + + +def error_gate(name: str, exc: BaseException) -> GateResult: + return GateResult( + gate=name, + status="error", + summary=f"{type(exc).__name__}: {exc}", + evidence={"error_class": type(exc).__name__, "error": str(exc)}, + ) + + +def build_report( + *, + gates: Iterable[GateResult], + started_at: str, + finished_at: str, + duration_s: float, + config: Mapping[str, Any], + environment: Mapping[str, Any], + git: Mapping[str, Any], +) -> dict[str, Any]: + gate_dicts = [gate.to_dict() for gate in gates] + statuses = [gate["status"] for gate in gate_dicts] + if any(status in {"failed", "error"} for status in statuses): + status = "failed" + elif gate_dicts and all(item == "passed" for item in statuses): + status = "passed" + else: + status = "incomplete" + report = { + "$schema": SCHEMA_ID, + "schema_version": 1, + "kind": "primary_decode_mac_acceptance", + "status": status, + "started_at": started_at, + "finished_at": finished_at, + "duration_s": float(duration_s), + "config": dict(config), + "environment": dict(environment), + "git": dict(git), + "gates": gate_dicts, + } + latency = next( + ( + gate["metrics"] + for gate in gate_dicts + if gate["gate"] == "decode_latency_regression" + and gate["metrics"].get("p50_decode_latency_s") is not None + ), + None, + ) + if latency is not None: + report["latency"] = { + "p50_decode_latency_s": latency["p50_decode_latency_s"], + "p95_decode_latency_s": latency["p95_decode_latency_s"], + "sample_count": latency["sample_count"], + } + return report + + +def render_junit(report: Mapping[str, Any]) -> str: + gates = list(report["gates"]) + failures = sum(gate["status"] == "failed" for gate in gates) + errors = sum(gate["status"] == "error" for gate in gates) + skipped = sum(gate["status"] == "skipped" for gate in gates) + suite = ET.Element( + "testsuite", + { + "name": "primary-decode-mac-acceptance", + "tests": str(len(gates)), + "failures": str(failures), + "errors": str(errors), + "skipped": str(skipped), + "time": str(report["duration_s"]), + }, + ) + for gate in gates: + case = ET.SubElement( + suite, + "testcase", + { + "classname": "primary_decode.acceptance", + "name": str(gate["gate"]), + }, + ) + if gate["status"] == "failed": + ET.SubElement(case, "failure", {"message": gate["summary"]}).text = ( + json.dumps(gate, sort_keys=True) + ) + elif gate["status"] == "error": + ET.SubElement(case, "error", {"message": gate["summary"]}).text = ( + json.dumps(gate, sort_keys=True) + ) + elif gate["status"] == "skipped": + ET.SubElement(case, "skipped", {"message": gate["summary"]}) + ET.SubElement(case, "system-out").text = json.dumps(gate, sort_keys=True) + return ET.tostring(suite, encoding="unicode") + + +def atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + f".tmp-{os.getpid()}") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + os.replace(temporary, path) diff --git a/inference_engine/distributed/prefill_cache_runtime.py b/inference_engine/distributed/prefill_cache_runtime.py index bb492d59..1b95a3a0 100644 --- a/inference_engine/distributed/prefill_cache_runtime.py +++ b/inference_engine/distributed/prefill_cache_runtime.py @@ -238,22 +238,36 @@ def _try_import( payload, max_uncompressed_bytes=self.max_import_bytes, ) - verifier.reset() - imported = import_mlx_prefill_snapshot( - raw, - verifier.cache, - compatibility=self.compatibility, - ) + import_remote = getattr(verifier, "import_snapshot", None) + if import_remote is not None: + state = import_remote(raw, self.compatibility) + imported_token_count = int(state["next_global_position"]) + imported_cached_ids = tuple(state["cached_token_ids"]) + imported_block_hash = bytes.fromhex( + state.get("block_hash", hit.block_hash.hex()), + ) + imported_has_logits = True + else: + verifier.reset() + imported = import_mlx_prefill_snapshot( + raw, + verifier.cache, + compatibility=self.compatibility, + ) + imported_token_count = imported.token_count + imported_cached_ids = imported.cached_token_ids + imported_block_hash = imported.block_hash + imported_has_logits = imported.next_token_logits is not None if ( - not (0 < imported.token_count <= len(tokens)) - or imported.token_count != hit.hit_tokens + not (0 < imported_token_count <= len(tokens)) + or imported_token_count != hit.hit_tokens ): raise ValueError("prefill snapshot token_count is invalid") - if hit.block_hash and imported.block_hash != hit.block_hash: + if hit.block_hash and imported_block_hash != hit.block_hash: raise ValueError("prefill snapshot block hash mismatch") - if imported.next_token_logits is None: + if not imported_has_logits: raise ValueError("prefill snapshot is missing continuation logits") - reused = min(imported.token_count, len(tokens)) + reused = min(imported_token_count, len(tokens)) expected_prefix = tokens[:reused] sink_window = getattr(verifier, "_sink_window_slice", None) expected_cached = ( @@ -261,11 +275,11 @@ def _try_import( if callable(sink_window) else expected_prefix ) - if list(imported.cached_token_ids) != expected_cached: + if list(imported_cached_ids) != expected_cached: raise ValueError("prefill snapshot cached token sequence mismatch") - verifier.cached_token_sequence = list(imported.cached_token_ids) + verifier.cached_token_sequence = list(imported_cached_ids) verifier.next_global_position = reused - if imported.next_token_logits is not None: + if import_remote is None and imported.next_token_logits is not None: verifier.next_token_logits = imported.next_token_logits self.stats.tokens_reused += reused if self._on_reuse is not None: @@ -428,21 +442,27 @@ def _compute_and_publish( first_end = min(size, len(tokens)) verifier.prefill(tokens[:first_end]) self.stats.tokens_computed += first_end - self._publish_boundary(verifier, tokens, hashes, 0, first_end) + if not getattr(verifier, "is_decode_worker_proxy", False): + self._publish_boundary(verifier, tokens, hashes, 0, first_end) reused = first_end cursor = reused while cursor < len(tokens): block_index = cursor // size end = min((block_index + 1) * size, len(tokens)) block_tokens = tokens[cursor:end] - logits = verifier.forward_block(block_tokens) - verifier.commit_or_truncate( - forwarded=len(block_tokens), - accepted=len(block_tokens), - ) - verifier.next_token_logits = logits[-1].clone() + append_accepted = getattr(verifier, "append_accepted_tokens", None) + if append_accepted is not None: + append_accepted(block_tokens) + else: + logits = verifier.forward_block(block_tokens) + verifier.commit_or_truncate( + forwarded=len(block_tokens), + accepted=len(block_tokens), + ) + verifier.next_token_logits = logits[-1].clone() self.stats.tokens_computed += len(block_tokens) - self._publish_boundary(verifier, tokens, hashes, block_index, end) + if not getattr(verifier, "is_decode_worker_proxy", False): + self._publish_boundary(verifier, tokens, hashes, block_index, end) cursor = end def _publish_boundary( diff --git a/inference_engine/server/grpc_app.py b/inference_engine/server/grpc_app.py index 24539317..c102f106 100644 --- a/inference_engine/server/grpc_app.py +++ b/inference_engine/server/grpc_app.py @@ -29,7 +29,11 @@ from __future__ import annotations +import asyncio +import inspect import logging +import queue +import threading from dataclasses import dataclass from typing import Optional @@ -49,7 +53,9 @@ GenerationCoordinator, HistoryTruncatedEvent, InvariantViolation, + OperationCancelledError, SessionNotFoundError, + SessionGenerationBusyError, SessionStore, STOP_REASON_CANCELLED, STOP_REASON_EOS, @@ -74,6 +80,59 @@ } _logger = logging.getLogger(__name__) +_STREAM_END = object() + + +class _ThreadedEventStream: + """Drain a synchronous model iterator without blocking asyncio.""" + + def __init__(self, stream, cancel_event: threading.Event) -> None: + self._stream = stream + self._cancel_event = cancel_event + self._queue: queue.Queue = queue.Queue(maxsize=1) + self._thread = threading.Thread( + target=self._run, + name="kakeya-generate", + daemon=True, + ) + + def start(self) -> None: + self._thread.start() + + def cancel(self) -> None: + self._cancel_event.set() + + def _put(self, value) -> None: + while True: + try: + self._queue.put(value, timeout=0.1) + return + except queue.Full: + if self._cancel_event.is_set(): + return + + def _run(self) -> None: + try: + for event in self._stream: + self._put((event, None)) + if self._cancel_event.is_set(): + break + except BaseException as exc: + self._put((None, exc)) + finally: + self._put((_STREAM_END, None)) + + async def next(self): + return await asyncio.to_thread(self._queue.get) + + +def _accepts_cancel_event(method) -> bool: + parameters = inspect.signature(method).parameters.values() + return any( + parameter.name == "cancel_event" + or parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) DEFAULT_BIND_ADDRESS = "127.0.0.1:50051" """Default bind address for the gRPC server. @@ -140,7 +199,8 @@ def __init__( *, append_coordinator: Optional[AppendTokensCoordinator] = None, generation_coordinator: Optional[GenerationCoordinator] = None, - on_session_close=None, + memory_governor=None, + liveness=None, ) -> None: """Construct a Servicer. @@ -159,9 +219,41 @@ def __init__( self._store = session_store self._append = append_coordinator self._generate = generation_coordinator - # PR-A3c: optional ``session_id -> None`` hook invoked on CloseSession - # so a per-session verifier registry can free that session's KV. - self._on_session_close = on_session_close + self._memory_governor = memory_governor + self._liveness = liveness + self._operation_lock = threading.Lock() + self._active_operations: set[str] = set() + + def _acquire_operation(self, session_id: str) -> bool: + with self._operation_lock: + if session_id in self._active_operations: + return False + self._active_operations.add(session_id) + return True + + def _release_operation(self, session_id: str) -> None: + with self._operation_lock: + self._active_operations.discard(session_id) + + def _watch_context( + self, + context, + cancel_event: threading.Event, + session_id: str, + completed_event: threading.Event | None = None, + ) -> None: + callback = getattr(context, "add_done_callback", None) + if callback is not None: + def done(done_context) -> None: + logically_completed = ( + completed_event is not None and completed_event.is_set() + ) + if done_context.cancelled() and not logically_completed: + cancel_event.set() + self._store.remove_session_if_present( + session_id, reason="client_cancelled", + ) + callback(done) async def CreateSession( # noqa: N802 — gRPC-generated method casing self, @@ -173,6 +265,11 @@ async def CreateSession( # noqa: N802 — gRPC-generated method casing ADR 0008 §2.2 contract item 1: clients have no input on the ``session_id`` value; this RPC is the only producer. """ + if self._memory_governor is not None and self._memory_governor.draining: + await context.abort( + grpc.StatusCode.RESOURCE_EXHAUSTED, + "Primary runtime is draining due to unified-memory pressure", + ) try: session = self._store.create_session( eos_token_ids=list(request.eos_token_ids), @@ -206,11 +303,36 @@ async def AppendTokens( # noqa: N802 — gRPC-generated method casing "AppendTokens not configured on this Servicer " "(coordinator not provided)", ) + if not self._acquire_operation(request.session_id): + await context.abort( + grpc.StatusCode.ABORTED, + f"session {request.session_id!r} already has an active operation", + ) + cancel_event = threading.Event() + completed_event = threading.Event() + self._watch_context( + context, + cancel_event, + request.session_id, + completed_event, + ) try: - new_history_length = self._append.append_tokens( - session_id=request.session_id, - token_ids=list(request.token_ids), + kwargs = { + "session_id": request.session_id, + "token_ids": list(request.token_ids), + } + if _accepts_cancel_event(self._append.append_tokens): + kwargs["cancel_event"] = cancel_event + new_history_length = await asyncio.to_thread( + self._append.append_tokens, **kwargs, + ) + completed_event.set() + except asyncio.CancelledError: + cancel_event.set() + self._store.remove_session_if_present( + request.session_id, reason="client_cancelled", ) + raise except SessionNotFoundError as exc: await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) except RemotePrefillRequiredError as exc: @@ -219,6 +341,15 @@ async def AppendTokens( # noqa: N802 — gRPC-generated method casing await context.abort(grpc.StatusCode.INVALID_ARGUMENT, str(exc)) except InvariantViolation as exc: await context.abort(grpc.StatusCode.FAILED_PRECONDITION, str(exc)) + except OperationCancelledError: + self._store.remove_session_if_present( + request.session_id, reason="client_cancelled", + ) + await context.abort(grpc.StatusCode.CANCELLED, "AppendTokens cancelled") + finally: + self._release_operation(request.session_id) + if self._liveness is not None: + self._liveness.update("idle") return runtime_pb2.AppendTokensResponse( history_length=new_history_length, ) @@ -264,35 +395,47 @@ async def Generate( # noqa: N802 — gRPC-generated method casing top_p = request.top_p if request.HasField("top_p") else None top_k = request.top_k if request.HasField("top_k") else None - # GenerationCoordinator.generate is a generator function; the - # call itself returns a generator object without executing - # any of the body, so no exception is raised here. All typed - # errors (SessionNotFoundError, ValueError, InvariantViolation) - # propagate from the inner `for event in event_stream:` loop - # below and are caught there. - event_stream = self._generate.generate( - session_id=request.session_id, - max_tokens=request.max_tokens, - seed=seed, - temperature=temperature, - top_p=top_p, - top_k=top_k, + if not self._acquire_operation(request.session_id): + await context.abort( + grpc.StatusCode.ABORTED, + f"session {request.session_id!r} already has an active operation", + ) + cancel_event = threading.Event() + completed_event = threading.Event() + self._watch_context( + context, + cancel_event, + request.session_id, + completed_event, ) + kwargs = { + "session_id": request.session_id, + "max_tokens": request.max_tokens, + "seed": seed, + "temperature": temperature, + "top_p": top_p, + "top_k": top_k, + } + if _accepts_cancel_event(self._generate.generate): + kwargs["cancel_event"] = cancel_event + event_stream = self._generate.generate(**kwargs) + threaded_stream = _ThreadedEventStream(event_stream, cancel_event) + threaded_stream.start() token_count_so_far = 0 try: - for event in event_stream: + while True: + event, error = await threaded_stream.next() + if error is not None: + raise error + if event is _STREAM_END: + completed_event.set() + break if context.cancelled(): - yield runtime_pb2.GenerateResponse( - done=runtime_pb2.GenerateDone( - stop_reason=_STOP_REASON_TO_PROTO[ - STOP_REASON_CANCELLED - ], - generated_token_count=token_count_so_far, - prefill_duration_seconds=0.0, - total_duration_seconds=0.0, - ), + threaded_stream.cancel() + self._store.remove_session_if_present( + request.session_id, reason="client_cancelled", ) return @@ -311,6 +454,7 @@ async def Generate( # noqa: N802 — gRPC-generated method casing # DoneEvent — the only remaining event type per # the GenerateEvent union. assert isinstance(event, DoneEvent) + completed_event.set() yield runtime_pb2.GenerateResponse( done=runtime_pb2.GenerateDone( stop_reason=_STOP_REASON_TO_PROTO[ @@ -327,6 +471,18 @@ async def Generate( # noqa: N802 — gRPC-generated method casing await context.abort(grpc.StatusCode.INVALID_ARGUMENT, str(exc)) except InvariantViolation as exc: await context.abort(grpc.StatusCode.FAILED_PRECONDITION, str(exc)) + except SessionGenerationBusyError as exc: + await context.abort(grpc.StatusCode.ABORTED, str(exc)) + except asyncio.CancelledError: + threaded_stream.cancel() + if not completed_event.is_set(): + self._store.remove_session_if_present( + request.session_id, reason="client_cancelled", + ) + raise + finally: + threaded_stream.cancel() + self._release_operation(request.session_id) async def CloseSession( # noqa: N802 self, @@ -343,8 +499,6 @@ async def CloseSession( # noqa: N802 final_length = self._store.close_session(request.session_id) except SessionNotFoundError as exc: await context.abort(grpc.StatusCode.NOT_FOUND, str(exc)) - if self._on_session_close is not None: - self._on_session_close(request.session_id) return runtime_pb2.CloseSessionResponse( final_history_length=final_length, ) @@ -381,7 +535,8 @@ def create_grpc_server( append_coordinator: Optional[AppendTokensCoordinator] = None, generation_coordinator: Optional[GenerationCoordinator] = None, config: Optional[GrpcServerConfig] = None, - on_session_close=None, + memory_governor=None, + liveness=None, capability_registry: Optional[object] = None, proposers: Optional[object] = None, default_proposer_model_id: str = "", @@ -432,7 +587,8 @@ def create_grpc_server( session_store, append_coordinator=append_coordinator, generation_coordinator=generation_coordinator, - on_session_close=on_session_close, + memory_governor=memory_governor, + liveness=liveness, ), server, ) diff --git a/inference_engine/server/runtime_health.py b/inference_engine/server/runtime_health.py new file mode 100644 index 00000000..63a29691 --- /dev/null +++ b/inference_engine/server/runtime_health.py @@ -0,0 +1,237 @@ +"""Primary decode liveness and unified-memory governance.""" + +from __future__ import annotations + +import gc +import json +import os +import threading +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Callable, Optional + +import psutil +from fastapi import FastAPI +from fastapi.responses import JSONResponse + + +GIB = 1 << 30 + + +@dataclass(frozen=True) +class DecodeSnapshot: + phase: str + session_id: str + token_index: int + updated_at_unix: float + pid: int + + +class DecodeLiveness: + """Thread-safe liveness state mirrored to an atomically replaced file.""" + + def __init__( + self, + path: str | Path, + *, + clock: Callable[[], float] = time.time, + pid: Optional[int] = None, + unhealthy_path: str | Path | None = None, + ) -> None: + self.path = Path(path).expanduser() + self.unhealthy_path = ( + Path(unhealthy_path).expanduser() if unhealthy_path else None + ) + if self.unhealthy_path is not None: + self.unhealthy_path.unlink(missing_ok=True) + self._clock = clock + self._lock = threading.Lock() + self._snapshot = DecodeSnapshot( + phase="idle", + session_id="", + token_index=0, + updated_at_unix=clock(), + pid=os.getpid() if pid is None else pid, + ) + self._publish() + + def update(self, phase: str, session_id: str = "", token_index: int = 0) -> None: + with self._lock: + self._snapshot = DecodeSnapshot( + phase=phase, + session_id=session_id, + token_index=int(token_index), + updated_at_unix=self._clock(), + pid=self._snapshot.pid, + ) + self._publish() + + def snapshot(self) -> dict: + with self._lock: + result = asdict(self._snapshot) + result["unhealthy"] = bool( + self.unhealthy_path and self.unhealthy_path.exists() + ) + return result + + def _publish(self) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + temporary = self.path.with_suffix(f"{self.path.suffix}.tmp-{os.getpid()}") + temporary.write_text(json.dumps(asdict(self._snapshot), sort_keys=True)) + os.replace(temporary, self.path) + + +@dataclass(frozen=True) +class MemorySnapshot: + mlx_active_bytes: int + mlx_cache_bytes: int + mlx_peak_bytes: int + process_footprint_bytes: int + active_sessions: int + kv_bytes: int + level: str + updated_at_unix: float + + +def process_footprint_bytes(pid: Optional[int] = None) -> int: + """Return resident process bytes without forking the threaded runtime.""" + try: + return int(psutil.Process(pid or os.getpid()).memory_info().rss) + except (OSError, ValueError, psutil.Error): + return 0 + + +def mlx_memory_bytes() -> tuple[int, int, int]: + """Best-effort MLX active/cache/peak counters without importing MLX in CI.""" + try: + import mlx.core as mx + except ImportError: + return (0, 0, 0) + + def read(name: str) -> int: + getter = getattr(mx, name, None) + if getter is None: + getter = getattr(getattr(mx, "metal", None), name, None) + try: + return int(getter()) if getter is not None else 0 + except (RuntimeError, TypeError, ValueError): + return 0 + + return ( + read("get_active_memory"), + read("get_cache_memory"), + read("get_peak_memory"), + ) + + +class PrimaryMemoryGovernor: + """Classify memory pressure and clean idle shared-verifier state.""" + + def __init__( + self, + store, + verifier, + *, + warning_bytes: int = 18 * GIB, + drain_bytes: int = 20 * GIB, + unhealthy_bytes: int = int(21.5 * GIB), + memory_provider: Callable[[], tuple[int, int, int]] = mlx_memory_bytes, + footprint_provider: Callable[[], int] = process_footprint_bytes, + clear_cache: Optional[Callable[[], None]] = None, + ) -> None: + if not warning_bytes < drain_bytes < unhealthy_bytes: + raise ValueError("memory thresholds must satisfy warning < drain < unhealthy") + self.store = store + self.verifier = verifier + self.warning_bytes = warning_bytes + self.drain_bytes = drain_bytes + self.unhealthy_bytes = unhealthy_bytes + self._memory_provider = memory_provider + self._footprint_provider = footprint_provider + self._clear_cache = clear_cache or self._default_clear_cache + self._lock = threading.Lock() + self._last = self.sample() + + @property + def draining(self) -> bool: + with self._lock: + return self._last.level in {"drain", "unhealthy"} + + @property + def unhealthy(self) -> bool: + with self._lock: + return self._last.level == "unhealthy" + + def sample(self) -> MemorySnapshot: + with self._lock: + active, cache, peak = self._memory_provider() + footprint = self._footprint_provider() + pressure = max(active + cache, footprint) + if pressure >= self.unhealthy_bytes: + level = "unhealthy" + elif pressure >= self.drain_bytes: + level = "drain" + elif pressure >= self.warning_bytes: + level = "warning" + else: + level = "ok" + self._last = MemorySnapshot( + mlx_active_bytes=active, + mlx_cache_bytes=cache, + mlx_peak_bytes=peak, + process_footprint_bytes=footprint, + active_sessions=self.store.active_count, + kv_bytes=self.store.total_kv_live_bytes, + level=level, + updated_at_unix=time.time(), + ) + return self._last + + def on_session_removed(self, _session_id: str, _reason: str) -> None: + """Reset and reclaim MLX cache when the final session leaves under pressure.""" + snapshot = self.sample() + if snapshot.active_sessions or snapshot.level == "ok": + return + reset = getattr(self.verifier, "reset", None) + if reset is not None: + reset() + gc.collect() + self._clear_cache() + self.sample() + + @staticmethod + def _default_clear_cache() -> None: + try: + import mlx.core as mx + except ImportError: + return + mx.clear_cache() + + +def create_runtime_health_app( + liveness: DecodeLiveness, + memory: PrimaryMemoryGovernor, +) -> FastAPI: + """Create a read-only diagnostics app for operators and watchdogs.""" + app = FastAPI(title="Kakeya Primary Runtime Health", version="1") + + @app.get("/healthz") + def healthz(): + live = liveness.snapshot() + mem = asdict(memory.sample()) + unhealthy = bool(live["unhealthy"] or mem["level"] == "unhealthy") + return JSONResponse( + status_code=503 if unhealthy else 200, + content={"status": "unhealthy" if unhealthy else "ok", "liveness": live, "memory": mem}, + ) + + @app.get("/v1/runtime/liveness") + def runtime_liveness(): + return liveness.snapshot() + + @app.get("/v1/runtime/memory") + def runtime_memory(): + return asdict(memory.sample()) + + return app diff --git a/inference_engine/session/__init__.py b/inference_engine/session/__init__.py index 2e5ebe3e..35adfa17 100644 --- a/inference_engine/session/__init__.py +++ b/inference_engine/session/__init__.py @@ -18,6 +18,7 @@ from inference_engine.session.coordinator import ( AppendTokensCoordinator, + OperationCancelledError, VerifierProtocol, ) from inference_engine.session.generator import ( @@ -25,6 +26,7 @@ GenerateEvent, GenerationCoordinator, HistoryTruncatedEvent, + SessionGenerationBusyError, STOP_REASON_CANCELLED, STOP_REASON_EOS, STOP_REASON_MAX_TOKENS, @@ -48,11 +50,13 @@ "GenerationCoordinator", "HistoryTruncatedEvent", "InvariantViolation", + "OperationCancelledError", "STOP_REASON_CANCELLED", "STOP_REASON_EOS", "STOP_REASON_MAX_TOKENS", "STOP_REASON_TRUNCATED", "Session", + "SessionGenerationBusyError", "SessionNotFoundError", "SessionStore", "SessionStoreError", diff --git a/inference_engine/session/coordinator.py b/inference_engine/session/coordinator.py index 373739a3..b2846844 100644 --- a/inference_engine/session/coordinator.py +++ b/inference_engine/session/coordinator.py @@ -48,6 +48,7 @@ from __future__ import annotations +import threading from typing import Any, Callable, Iterable, List, Protocol import torch @@ -107,6 +108,10 @@ def prepare(self, verifier: Any, token_ids: List[int]) -> int: ... # pragma: no cover +class OperationCancelledError(RuntimeError): + """Raised when a blocking coordinator observes client cancellation.""" + + def _sync_slab_bytes(session: Session, verifier: "VerifierProtocol") -> None: """Mirror the verifier's current KV byte count onto the session's slab placeholder (PR-E1c). @@ -146,6 +151,7 @@ def __init__( resolver=None, prefill_cache: PrefillCacheHookProtocol | None = None, on_first_append: Callable[[Session, list[int]], None] | None = None, + liveness=None, ) -> None: self._store = store self._verifier = verifier @@ -155,6 +161,7 @@ def __init__( self._resolver = resolver self._prefill_cache = prefill_cache self._on_first_append = on_first_append + self._liveness = liveness def _verifier_for(self, session_id: str) -> "VerifierProtocol": return self._resolver(session_id) if self._resolver else self._verifier @@ -163,6 +170,7 @@ def append_tokens( self, session_id: str, token_ids: Iterable[int], + cancel_event: threading.Event | None = None, ) -> int: """Run the §2.3 byte-exact prefill-incremental for ``session_id``. @@ -181,6 +189,9 @@ def append_tokens( now-orphaned cache state and the caller must reset it before re-using the verifier instance. """ + if cancel_event is not None and cancel_event.is_set(): + raise OperationCancelledError("AppendTokens cancelled") + # Lookup early to surface NOT_FOUND before we touch the verifier. session = self._store.get_session(session_id) verifier = self._verifier_for(session_id) @@ -204,18 +215,40 @@ def append_tokens( # - next_global_position = sum of all tokens ever appended # - next_token_logits predicts position == next_global_position first_append = session.next_global_position == 0 + if self._liveness is not None: + self._liveness.update( + "prefill" if first_append else "decode", session_id, 0, + ) if first_append: if self._prefill_cache is not None: self._prefill_cache.prepare(verifier, token_list) else: - verifier.prefill(token_list) + if getattr(verifier, "is_decode_worker_proxy", False): + verifier.prefill(token_list, cancel_event=cancel_event) + else: + verifier.prefill(token_list) else: - block_logits = verifier.forward_block(token_list) - verifier.commit_or_truncate( - forwarded=len(token_list), - accepted=len(token_list), + append_accepted = getattr( + verifier, "append_accepted_tokens", None, ) - verifier.next_token_logits = block_logits[-1].clone() + if append_accepted is not None: + # MLX primary append needs only the final logits row; + # speculative verification continues to call + # forward_block and keeps full [L,V] semantics. + if getattr(verifier, "is_decode_worker_proxy", False): + append_accepted(token_list, cancel_event=cancel_event) + else: + append_accepted(token_list) + else: + block_logits = verifier.forward_block(token_list) + verifier.commit_or_truncate( + forwarded=len(token_list), + accepted=len(token_list), + ) + verifier.next_token_logits = block_logits[-1].clone() + + if cancel_event is not None and cancel_event.is_set(): + raise OperationCancelledError("AppendTokens cancelled") # Mirror the verifier's post-trim cached_token_sequence onto the # session BEFORE the store's INV-1 assertion runs (it compares diff --git a/inference_engine/session/generator.py b/inference_engine/session/generator.py index 6abf39ed..c158af25 100644 --- a/inference_engine/session/generator.py +++ b/inference_engine/session/generator.py @@ -46,6 +46,7 @@ from __future__ import annotations import time +import threading from dataclasses import dataclass from typing import Callable, Iterator, Optional, Union @@ -107,6 +108,10 @@ class DoneEvent: GenerateEvent = Union[TokenEvent, HistoryTruncatedEvent, DoneEvent] +class SessionGenerationBusyError(RuntimeError): + """Raised when a second Generate starts for the same session.""" + + class GenerationCoordinator: """Greedy session-aware token generation against a verifier.""" @@ -116,12 +121,21 @@ def __init__( verifier: VerifierProtocol, resolver=None, on_tokens: Optional[Callable[[int], None]] = None, + liveness=None, ) -> None: self._store = store self._verifier = verifier # PR-A3c: optional per-session verifier resolver (multi-tenant). self._resolver = resolver self._on_tokens = on_tokens + self._liveness = liveness + self._guard_lock = threading.Lock() + self._active_sessions: set[str] = set() + + @property + def active_count(self) -> int: + with self._guard_lock: + return len(self._active_sessions) def _verifier_for(self, session_id: str) -> "VerifierProtocol": return self._resolver(session_id) if self._resolver else self._verifier @@ -135,6 +149,49 @@ def generate( temperature: Optional[float] = None, top_p: Optional[float] = None, top_k: Optional[int] = None, + cancel_event: Optional[threading.Event] = None, + ) -> Iterator[GenerateEvent]: + """Return a cancellation-aware stream guarded per session.""" + stream = self._generate_events( + session_id, + max_tokens=max_tokens, + seed=seed, + temperature=temperature, + top_p=top_p, + top_k=top_k, + cancel_event=cancel_event, + ) + + def guarded() -> Iterator[GenerateEvent]: + acquired = False + try: + with self._guard_lock: + if session_id in self._active_sessions: + raise SessionGenerationBusyError( + f"Generate already active for session {session_id!r}" + ) + self._active_sessions.add(session_id) + acquired = True + yield from stream + finally: + if acquired: + with self._guard_lock: + self._active_sessions.discard(session_id) + if self._liveness is not None: + self._liveness.update("idle") + + return guarded() + + def _generate_events( + self, + session_id: str, + *, + max_tokens: int, + seed: Optional[int] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + cancel_event: Optional[threading.Event] = None, ) -> Iterator[GenerateEvent]: """Yield a stream of GenerateEvents for ``session_id``. @@ -208,19 +265,50 @@ def generate( # Generate has no separate prefill phase in PR-B3; report 0. prefill_seconds = 0.0 generated_count = 0 + if self._liveness is not None: + self._liveness.update("decode", session_id, generated_count) for _step in range(max_tokens): - # Greedy: argmax of the verifier's last next_token_logits. - next_token = int( - torch.argmax(verifier.next_token_logits).item() - ) - - # Forward + commit (forwarded == accepted for prompt-mode - # appends; same contract used by AppendTokens, just one - # token at a time). - block_logits = verifier.forward_block([next_token]) - verifier.commit_or_truncate(forwarded=1, accepted=1) - verifier.next_token_logits = block_logits[-1].clone() + if cancel_event is not None and cancel_event.is_set(): + yield DoneEvent( + stop_reason=STOP_REASON_CANCELLED, + generated_token_count=generated_count, + prefill_seconds=prefill_seconds, + total_seconds=time.perf_counter() - t0, + ) + return + # MLX exposes an on-device argmax plus a last-logits-only + # all-accepted append. Other verifiers retain the original + # Torch/full-block path. In particular, speculative callers + # still use forward_block directly and receive full [L,V]. + generate_step = getattr(verifier, "generate_step", None) + if generate_step is not None: + # Decode-worker mode keeps argmax + commit atomic inside the + # child. If the transport dies before replying, the router + # restores the last acknowledged checkpoint and retries this + # step without duplicating a token. + next_token = int(generate_step(cancel_event=cancel_event)) + else: + greedy_argmax = getattr(verifier, "greedy_next_token_id", None) + if greedy_argmax is not None: + next_token = int(greedy_argmax()) + else: + next_token = int( + torch.argmax(verifier.next_token_logits).item() + ) + + append_accepted = getattr( + verifier, "append_accepted_tokens", None, + ) + if append_accepted is not None: + append_accepted([next_token]) + else: + # Forward + commit (forwarded == accepted for + # prompt-mode appends; same contract used by + # AppendTokens, just one token at a time). + block_logits = verifier.forward_block([next_token]) + verifier.commit_or_truncate(forwarded=1, accepted=1) + verifier.next_token_logits = block_logits[-1].clone() # Mirror state from verifier onto session BEFORE the # store's INV-1 check runs (it compares @@ -234,6 +322,8 @@ def generate( session_id, verifier.next_global_position, ) generated_count += 1 + if self._liveness is not None: + self._liveness.update("decode", session_id, generated_count) yield TokenEvent(token_id=next_token) diff --git a/inference_engine/session/store.py b/inference_engine/session/store.py index 774ae1a5..82ecee41 100644 --- a/inference_engine/session/store.py +++ b/inference_engine/session/store.py @@ -32,21 +32,19 @@ surface for TTL-based eviction; PR-A3 will call it from a background task at the configured interval. -This module is intentionally **not thread-safe**. The gRPC server -(PR-B1) runs all RPCs on a single asyncio event loop, serializing -access at that layer; multi-worker support is v0.4 scope (ADR 0008 -§4.5). Direct concurrent access from threads is undefined behavior -and is documented as such here so reviewers can flag any future -caller that violates the contract. +Store-owned mutations are protected by a re-entrant lock. Blocking +model work runs in worker threads so cancellation can be handled by +the asyncio server; verifier mutation remains coordinator-owned. """ from __future__ import annotations import time +import threading import uuid from collections.abc import Iterable from dataclasses import dataclass, field -from typing import Optional, Protocol +from typing import Callable, Optional, Protocol from inference_engine.memory.pool import SlabPool from inference_engine.memory.slab import KVSlab @@ -213,9 +211,9 @@ class SessionStore: to admit the new one. :meth:`evict_idle` provides the TTL-based eviction surface that PR-A3 will drive from a background task. - The store is **not thread-safe** (see module docstring). All - callers in v0.3 enter via the gRPC asyncio event loop, which - serializes access. + Store-owned state is thread-safe. Returned ``Session`` objects + remain coordinator-owned and application code must not mutate + them concurrently. """ def __init__( @@ -224,6 +222,7 @@ def __init__( capacity: int, cache_inspector: Optional[CacheInspector] = None, slab_pool: Optional[SlabPool] = None, + on_session_removed: Optional[Callable[[str, str], None]] = None, ) -> None: if capacity < 1: raise ValueError( @@ -232,7 +231,9 @@ def __init__( self._capacity = capacity self._cache_inspector = cache_inspector self._slab_pool = slab_pool + self._on_session_removed = on_session_removed self._sessions: dict[str, Session] = {} + self._lock = threading.RLock() @property def slab_pool(self) -> Optional[SlabPool]: @@ -253,7 +254,8 @@ def capacity(self) -> int: @property def active_count(self) -> int: - return len(self._sessions) + with self._lock: + return len(self._sessions) @property def total_kv_live_bytes(self) -> int: @@ -263,7 +265,8 @@ def total_kv_live_bytes(self) -> int: return the real aggregate, which is the §2.9 ``session_kv_live_bytes`` Prometheus-gauge value. """ - return sum(s.kv_live_bytes() for s in self._sessions.values()) + with self._lock: + return sum(s.kv_live_bytes() for s in self._sessions.values()) def create_session( self, @@ -285,17 +288,18 @@ def create_session( *before* this acquire, so a single-pool / single-capacity configuration cannot deadlock on slab availability. """ - if self.active_count >= self._capacity: - self._evict_one_lru() - session = Session( - session_id=self._issue_id(), - eos_token_ids=tuple(eos_token_ids), - client_label=client_label, - ) - if self._slab_pool is not None: - session.slab = self._slab_pool.acquire() - self._sessions[session.session_id] = session - return session + with self._lock: + if self.active_count >= self._capacity: + self._evict_one_lru() + session = Session( + session_id=self._issue_id(), + eos_token_ids=tuple(eos_token_ids), + client_label=client_label, + ) + if self._slab_pool is not None: + session.slab = self._slab_pool.acquire() + self._sessions[session.session_id] = session + return session def get_session(self, session_id: str) -> Session: """Return the live session for ``session_id``. @@ -304,10 +308,11 @@ def get_session(self, session_id: str) -> Session: currently in the store (closed, evicted, never existed — callers cannot distinguish, by design). """ - try: - return self._sessions[session_id] - except KeyError as exc: - raise SessionNotFoundError(session_id) from exc + with self._lock: + try: + return self._sessions[session_id] + except KeyError as exc: + raise SessionNotFoundError(session_id) from exc def append_tokens( self, session_id: str, token_ids: Iterable[int], @@ -327,17 +332,18 @@ def append_tokens( post-mutation, which is the contract — INV-1 is "after every cache mutation"); the session is no longer usable. """ - session = self.get_session(session_id) - token_list = list(token_ids) - for tid in token_list: - if not isinstance(tid, int) or isinstance(tid, bool) or tid < 0: - raise ValueError( - f"token_id must be a non-negative int, got {tid!r}" - ) - session.history_token_ids.extend(token_list) - session.last_active_at = time.monotonic() - self._assert_inv1(session) - return session.history_length + with self._lock: + session = self.get_session(session_id) + token_list = list(token_ids) + for tid in token_list: + if not isinstance(tid, int) or isinstance(tid, bool) or tid < 0: + raise ValueError( + f"token_id must be a non-negative int, got {tid!r}" + ) + session.history_token_ids.extend(token_list) + session.last_active_at = time.monotonic() + self._assert_inv1(session) + return session.history_length def close_session(self, session_id: str) -> int: """Close a session and return its final ``history_length``. @@ -352,11 +358,25 @@ def close_session(self, session_id: str) -> int: so a follow-up ``create_session`` is guaranteed to have at least one free slab. """ - session = self.get_session(session_id) - final_length = session.history_length - self._release_slab_if_held(session) - del self._sessions[session_id] - return final_length + with self._lock: + session = self.get_session(session_id) + final_length = session.history_length + self._remove_session(session, reason="explicit_close") + return final_length + + def remove_session_if_present(self, session_id: str, *, reason: str) -> bool: + """Remove a session when present without hiding double-close errors. + + Cancellation paths use this idempotent helper because a client + disconnect can race an explicit CloseSession. Normal callers should + continue to use :meth:`close_session`. + """ + with self._lock: + session = self._sessions.get(session_id) + if session is None: + return False + self._remove_session(session, reason=reason) + return True def record_position_advance( self, session_id: str, new_position: int, @@ -373,22 +393,22 @@ def record_position_advance( Called by the verifier after every prefill / generation step (PR-A3b wiring); PR-A2 tests exercise it directly. """ - session = self.get_session(session_id) - if new_position < session.next_global_position: - session.inv2_violations += 1 - self._release_slab_if_held(session) - del self._sessions[session_id] - raise InvariantViolation( - kind="2", - session_id=session_id, - detail=( - f"position must be non-decreasing; " - f"current={session.next_global_position}, " - f"requested={new_position}" - ), - ) - session.next_global_position = new_position - session.last_active_at = time.monotonic() + with self._lock: + session = self.get_session(session_id) + if new_position < session.next_global_position: + session.inv2_violations += 1 + self._remove_session(session, reason="inv2_failure") + raise InvariantViolation( + kind="2", + session_id=session_id, + detail=( + f"position must be non-decreasing; " + f"current={session.next_global_position}, " + f"requested={new_position}" + ), + ) + session.next_global_position = new_position + session.last_active_at = time.monotonic() def evict_idle( self, @@ -409,16 +429,16 @@ def evict_idle( without monkey-patching ``time.monotonic``. Production callers omit it to use the current monotonic clock. """ - clock = now if now is not None else time.monotonic() - evicted: list[Session] = [] - for sid in list(self._sessions.keys()): - session = self._sessions[sid] - idle = clock - session.last_active_at - if idle >= ttl_seconds: - self._release_slab_if_held(session) - del self._sessions[sid] - evicted.append(session) - return evicted + with self._lock: + clock = now if now is not None else time.monotonic() + evicted: list[Session] = [] + for sid in list(self._sessions.keys()): + session = self._sessions[sid] + idle = clock - session.last_active_at + if idle >= ttl_seconds: + self._remove_session(session, reason="ttl_eviction") + evicted.append(session) + return evicted def _assert_inv1(self, session: Session) -> None: """Enforce INV-1 for ``session``. @@ -437,8 +457,7 @@ def _assert_inv1(self, session: Session) -> None: cached_len = len(session.cached_token_sequence) if actual_k_seq_len != cached_len: session.inv1_violations += 1 - self._release_slab_if_held(session) - del self._sessions[session.session_id] + self._remove_session(session, reason="inv1_failure") raise InvariantViolation( kind="1", session_id=session.session_id, @@ -467,6 +486,13 @@ def _release_slab_if_held(self, session: Session) -> None: session.slab = None self._slab_pool.release(slab) + def _remove_session(self, session: Session, *, reason: str) -> None: + """Release all store-owned state and notify the cleanup owner once.""" + self._release_slab_if_held(session) + self._sessions.pop(session.session_id, None) + if self._on_session_removed is not None: + self._on_session_removed(session.session_id, reason) + def _evict_one_lru(self) -> Session: """Evict the least-recently-active session. @@ -480,8 +506,8 @@ def _evict_one_lru(self) -> Session: self._sessions.items(), key=lambda kv: kv[1].last_active_at, ) - self._release_slab_if_held(evicted) - del self._sessions[evicted_id] + del evicted_id + self._remove_session(evicted, reason="lru_eviction") return evicted @staticmethod diff --git a/requirements.txt b/requirements.txt index 6e007e09..cd41d2f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,6 +44,7 @@ uvicorn>=0.32,<1.0 pydantic>=2.7,<3.0 httpx>=0.27,<1.0 prometheus-client>=0.20,<1.0 +psutil>=5.9,<8.0 # gRPC runtime (PR-B1 of ADR 0008 Phase B; the runtime/SDK protocol). # grpcio-tools is needed only at build time (regenerating stubs from diff --git a/results/platform-tests/primary_decode_acceptance.json b/results/platform-tests/primary_decode_acceptance.json new file mode 100644 index 00000000..e48e9197 --- /dev/null +++ b/results/platform-tests/primary_decode_acceptance.json @@ -0,0 +1,26652 @@ +{ + "$schema": "https://kakeya.dev/schemas/primary-decode-acceptance-v1.json", + "config": { + "control_timeout_s": 240.0, + "disconnect_limit_s": 5.0, + "disconnect_max_tokens": 4096, + "endurance_duration_s": 14400.0, + "endurance_max_tokens": 32, + "endurance_turn_spacing_s": 5.0, + "footprint_settle_s": 5.0, + "grpc_address": "127.0.0.1:50051", + "hang_limit_s": 120.0, + "junit_output": "results/platform-tests/primary_decode_acceptance.xml", + "latency_baseline": "results/platform-tests/bench_mlx_verifier_1779507043.json", + "latency_max_tokens": 32, + "latency_samples": 20, + "long_run_report": null, + "mode": "all", + "output": "results/platform-tests/primary_decode_acceptance.json", + "poll_interval_s": 0.25, + "probe_start_timeout_s": 15.0, + "session_count": 100, + "session_max_tokens": 8, + "settle_timeout_s": 30.0, + "tokenizer_id": "/Users/fluffy314/kakeya-models/gemma-4-26B-A4B-it-mlx-4bit" + }, + "duration_s": 14683.496339958161, + "environment": { + "machine": "arm64", + "platform": "macOS-26.5-arm64-arm-64bit-Mach-O", + "python": "3.14.3" + }, + "finished_at": "2026-07-21T18:34:45.983785+00:00", + "gates": [ + { + "evidence": {}, + "gate": "sequential_session_footprint", + "metrics": { + "active_sessions": 0, + "final_process_footprint_bytes": 54394880, + "fresh_process_footprint_bytes": 414597120, + "growth_bytes": -360202240, + "sessions_completed": 100 + }, + "status": "passed", + "summary": "100/100 sessions; footprint growth -360202240 bytes; active sessions 0", + "thresholds": { + "max_growth_bytes": 2147483648, + "required_active_sessions": 0, + "required_sessions": 100 + } + }, + { + "evidence": { + "records": [ + { + "decode_latency_s": 0.04194158075551968, + "kv_live_bytes": 9011200, + "latency_s": 1.3450685827992857, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70680576, + "prompt_kind": "short", + "t_relative_s": 0.003435875289142132 + }, + { + "decode_latency_s": 0.04726975780795328, + "kv_live_bytes": 81776640, + "latency_s": 1.514664291869849, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 49594368, + "prompt_kind": "long", + "t_relative_s": 5.008463582955301 + }, + { + "decode_latency_s": 0.19448144662601408, + "kv_live_bytes": 90787840, + "latency_s": 6.224768208805472, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 65798144, + "prompt_kind": "short", + "t_relative_s": 10.013505792245269 + }, + { + "decode_latency_s": 0.049309817724861205, + "kv_live_bytes": 163553280, + "latency_s": 1.5792533331550658, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 63193088, + "prompt_kind": "long", + "t_relative_s": 18.73200725018978 + }, + { + "decode_latency_s": 0.04958822394837625, + "kv_live_bytes": 172339200, + "latency_s": 1.587842124979943, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 54722560, + "prompt_kind": "short", + "t_relative_s": 23.737073583062738 + }, + { + "decode_latency_s": 1.6356374218739802, + "kv_live_bytes": 245104640, + "latency_s": 52.34493345860392, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 42876928, + "prompt_kind": "long", + "t_relative_s": 28.742133708205074 + }, + { + "decode_latency_s": 0.05208623046928551, + "kv_live_bytes": 254115840, + "latency_s": 1.6679823333397508, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 64307200, + "prompt_kind": "short", + "t_relative_s": 85.75188595801592 + }, + { + "decode_latency_s": 0.07313610415440053, + "kv_live_bytes": 326881280, + "latency_s": 2.341742292046547, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 60276736, + "prompt_kind": "long", + "t_relative_s": 90.75712504191324 + }, + { + "decode_latency_s": 0.05946037500689272, + "kv_live_bytes": 335892480, + "latency_s": 1.9039542502723634, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 65142784, + "prompt_kind": "short", + "t_relative_s": 97.20100595802069 + }, + { + "decode_latency_s": 1.7831243528489722, + "kv_live_bytes": 408657920, + "latency_s": 57.09534433390945, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 27983872, + "prompt_kind": "long", + "t_relative_s": 102.20604270836338 + }, + { + "decode_latency_s": 0.6393040963448584, + "kv_live_bytes": 417443840, + "latency_s": 20.466463500168175, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 57819136, + "prompt_kind": "short", + "t_relative_s": 168.81527866702527 + }, + { + "decode_latency_s": 0.6018479453196051, + "kv_live_bytes": 462274560, + "latency_s": 19.264735292177647, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 58245120, + "prompt_kind": "long", + "t_relative_s": 192.83681804221123 + }, + { + "decode_latency_s": 0.08589695965929423, + "kv_live_bytes": 462274560, + "latency_s": 2.755477999802679, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 64569344, + "prompt_kind": "short", + "t_relative_s": 216.8982549579814 + }, + { + "decode_latency_s": 0.07168494009238202, + "kv_live_bytes": 462274560, + "latency_s": 2.2953389170579612, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 28344320, + "prompt_kind": "long", + "t_relative_s": 222.6500195832923 + }, + { + "decode_latency_s": 2.6319101692788536, + "kv_live_bytes": 462274560, + "latency_s": 84.22491829190403, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 58425344, + "prompt_kind": "short", + "t_relative_s": 232.5751048750244 + }, + { + "decode_latency_s": 1.5243876965978416, + "kv_live_bytes": 462274560, + "latency_s": 48.78169579105452, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 58146816, + "prompt_kind": "long", + "t_relative_s": 322.84100687503815 + }, + { + "decode_latency_s": 0.12474652343371417, + "kv_live_bytes": 462274560, + "latency_s": 3.9938134998083115, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 30277632, + "prompt_kind": "short", + "t_relative_s": 379.8718310832046 + }, + { + "decode_latency_s": 0.07301290234318003, + "kv_live_bytes": 462274560, + "latency_s": 2.3377482080832124, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 57720832, + "prompt_kind": "long", + "t_relative_s": 388.27465179236606 + }, + { + "decode_latency_s": 0.0644523242226569, + "kv_live_bytes": 462274560, + "latency_s": 2.0634644581004977, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 54542336, + "prompt_kind": "short", + "t_relative_s": 398.39009783323854 + }, + { + "decode_latency_s": 0.3206847539113369, + "kv_live_bytes": 462274560, + "latency_s": 10.264617708977312, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 56082432, + "prompt_kind": "long", + "t_relative_s": 403.3951367922127 + }, + { + "decode_latency_s": 0.18872125518100802, + "kv_live_bytes": 462274560, + "latency_s": 6.040029584430158, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 39190528, + "prompt_kind": "short", + "t_relative_s": 419.06325933290645 + }, + { + "decode_latency_s": 0.06580676302837674, + "kv_live_bytes": 462274560, + "latency_s": 2.1071725417859852, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 59637760, + "prompt_kind": "long", + "t_relative_s": 430.4762354171835 + }, + { + "decode_latency_s": 0.24017046485096216, + "kv_live_bytes": 462274560, + "latency_s": 7.687336832750589, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 56754176, + "prompt_kind": "short", + "t_relative_s": 435.4812792921439 + }, + { + "decode_latency_s": 0.07155795834842138, + "kv_live_bytes": 462274560, + "latency_s": 2.2936204578727484, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 62603264, + "prompt_kind": "long", + "t_relative_s": 448.2242825832218 + }, + { + "decode_latency_s": 0.06366177997551858, + "kv_live_bytes": 462274560, + "latency_s": 2.0380771248601377, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 63799296, + "prompt_kind": "short", + "t_relative_s": 453.2293172501959 + }, + { + "decode_latency_s": 0.2601845377503196, + "kv_live_bytes": 462274560, + "latency_s": 8.328730124980211, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 55181312, + "prompt_kind": "long", + "t_relative_s": 458.2343935421668 + }, + { + "decode_latency_s": 0.06552486069267616, + "kv_live_bytes": 462274560, + "latency_s": 2.0978402919135988, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 62636032, + "prompt_kind": "short", + "t_relative_s": 473.40069587528706 + }, + { + "decode_latency_s": 0.06835086199862417, + "kv_live_bytes": 462274560, + "latency_s": 2.188219875097275, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 64012288, + "prompt_kind": "long", + "t_relative_s": 478.40577000007033 + }, + { + "decode_latency_s": 0.06286907158209942, + "kv_live_bytes": 462274560, + "latency_s": 2.0126317078247666, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 62914560, + "prompt_kind": "short", + "t_relative_s": 489.8329248330556 + }, + { + "decode_latency_s": 0.10900253646832425, + "kv_live_bytes": 462274560, + "latency_s": 3.489047249779105, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 58638336, + "prompt_kind": "long", + "t_relative_s": 494.8380233752541 + }, + { + "decode_latency_s": 0.06553762759722304, + "kv_live_bytes": 462274560, + "latency_s": 2.098435875028372, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 63455232, + "prompt_kind": "short", + "t_relative_s": 505.06951637519524 + }, + { + "decode_latency_s": 0.06605861718708184, + "kv_live_bytes": 462274560, + "latency_s": 2.114868250209838, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 64389120, + "prompt_kind": "long", + "t_relative_s": 510.0703265829943 + }, + { + "decode_latency_s": 0.07553266405011527, + "kv_live_bytes": 462274560, + "latency_s": 2.421180416829884, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 63619072, + "prompt_kind": "short", + "t_relative_s": 515.5636366251856 + }, + { + "decode_latency_s": 0.10975640753167681, + "kv_live_bytes": 462274560, + "latency_s": 3.513442291878164, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 57294848, + "prompt_kind": "long", + "t_relative_s": 520.5657405829988 + }, + { + "decode_latency_s": 0.06548752343223896, + "kv_live_bytes": 462274560, + "latency_s": 2.0968259163200855, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 60456960, + "prompt_kind": "short", + "t_relative_s": 531.8434088751674 + }, + { + "decode_latency_s": 0.10898987759719603, + "kv_live_bytes": 462274560, + "latency_s": 3.489520125091076, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 59326464, + "prompt_kind": "long", + "t_relative_s": 536.9728945000097 + }, + { + "decode_latency_s": 0.069050514313858, + "kv_live_bytes": 462274560, + "latency_s": 2.2117402497678995, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 63569920, + "prompt_kind": "short", + "t_relative_s": 545.129778875038 + }, + { + "decode_latency_s": 0.33017402472614776, + "kv_live_bytes": 462274560, + "latency_s": 10.56664541689679, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 34750464, + "prompt_kind": "long", + "t_relative_s": 550.1348357922398 + }, + { + "decode_latency_s": 0.0631501640600618, + "kv_live_bytes": 462274560, + "latency_s": 2.0221756659448147, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 62226432, + "prompt_kind": "short", + "t_relative_s": 567.10911016725 + }, + { + "decode_latency_s": 0.3369332591246348, + "kv_live_bytes": 462274560, + "latency_s": 10.782905833330005, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 32407552, + "prompt_kind": "long", + "t_relative_s": 572.1091809170321 + }, + { + "decode_latency_s": 0.3492901536228601, + "kv_live_bytes": 462274560, + "latency_s": 11.182629541959614, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 52183040, + "prompt_kind": "short", + "t_relative_s": 591.8658308750018 + }, + { + "decode_latency_s": 0.11856966925552115, + "kv_live_bytes": 462274560, + "latency_s": 3.7953444998711348, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 62488576, + "prompt_kind": "long", + "t_relative_s": 609.2900544581935 + }, + { + "decode_latency_s": 0.06353884634154383, + "kv_live_bytes": 462274560, + "latency_s": 2.035615958739072, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 63668224, + "prompt_kind": "short", + "t_relative_s": 623.6584696671925 + }, + { + "decode_latency_s": 1.5098980716138612, + "kv_live_bytes": 462274560, + "latency_s": 48.318434208165854, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 58392576, + "prompt_kind": "long", + "t_relative_s": 628.6685086670332 + }, + { + "decode_latency_s": 0.3939237994636642, + "kv_live_bytes": 462274560, + "latency_s": 12.60687604220584, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 38125568, + "prompt_kind": "short", + "t_relative_s": 683.3131812503561 + }, + { + "decode_latency_s": 0.12754465625039302, + "kv_live_bytes": 462274560, + "latency_s": 4.083868416026235, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 60669952, + "prompt_kind": "long", + "t_relative_s": 700.8209411669523 + }, + { + "decode_latency_s": 0.09734933462459594, + "kv_live_bytes": 462274560, + "latency_s": 3.1164666661061347, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 58245120, + "prompt_kind": "short", + "t_relative_s": 714.1765475003049 + }, + { + "decode_latency_s": 0.07417377083038446, + "kv_live_bytes": 462274560, + "latency_s": 2.375381208024919, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 62406656, + "prompt_kind": "long", + "t_relative_s": 722.0384831251577 + }, + { + "decode_latency_s": 0.526508941402426, + "kv_live_bytes": 462274560, + "latency_s": 16.849341417197138, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 58294272, + "prompt_kind": "short", + "t_relative_s": 727.0485808332451 + }, + { + "decode_latency_s": 0.0677816380339209, + "kv_live_bytes": 462274560, + "latency_s": 2.1703126253560185, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 61718528, + "prompt_kind": "long", + "t_relative_s": 748.0128453751095 + }, + { + "decode_latency_s": 0.06548524477693718, + "kv_live_bytes": 462274560, + "latency_s": 2.096369792241603, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 62865408, + "prompt_kind": "short", + "t_relative_s": 753.0228761672042 + }, + { + "decode_latency_s": 0.0662957070308039, + "kv_live_bytes": 462274560, + "latency_s": 2.122789291664958, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 55820288, + "prompt_kind": "long", + "t_relative_s": 758.0329328333028 + }, + { + "decode_latency_s": 1.4323664895928232, + "kv_live_bytes": 462274560, + "latency_s": 45.85464516701177, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 60178432, + "prompt_kind": "short", + "t_relative_s": 763.103163999971 + }, + { + "decode_latency_s": 0.20300411459174938, + "kv_live_bytes": 462274560, + "latency_s": 6.501160624902695, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 65388544, + "prompt_kind": "long", + "t_relative_s": 817.8108744169585 + }, + { + "decode_latency_s": 0.07539148307114374, + "kv_live_bytes": 462274560, + "latency_s": 2.4134672917425632, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 64487424, + "prompt_kind": "short", + "t_relative_s": 846.1153635829687 + }, + { + "decode_latency_s": 11.979533445308334, + "kv_live_bytes": 462274560, + "latency_s": 383.3526329579763, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 52494336, + "prompt_kind": "long", + "t_relative_s": 851.1254146252759 + }, + { + "decode_latency_s": 3.6457390859432053, + "kv_live_bytes": 462274560, + "latency_s": 116.68958675023168, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 61210624, + "prompt_kind": "short", + "t_relative_s": 1241.3207086250186 + }, + { + "decode_latency_s": 0.12497417446866166, + "kv_live_bytes": 462274560, + "latency_s": 4.002859834115952, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 63176704, + "prompt_kind": "long", + "t_relative_s": 1403.8526265830733 + }, + { + "decode_latency_s": 0.39175537499249913, + "kv_live_bytes": 462274560, + "latency_s": 12.538035833276808, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 63258624, + "prompt_kind": "short", + "t_relative_s": 1414.6154155833647 + }, + { + "decode_latency_s": 0.12239820053218864, + "kv_live_bytes": 462274560, + "latency_s": 3.920055208262056, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 62996480, + "prompt_kind": "long", + "t_relative_s": 1433.3145287921652 + }, + { + "decode_latency_s": 0.06380257030832581, + "kv_live_bytes": 462274560, + "latency_s": 2.0425695003941655, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 64045056, + "prompt_kind": "short", + "t_relative_s": 1445.099639333319 + }, + { + "decode_latency_s": 0.06976525258505717, + "kv_live_bytes": 462274560, + "latency_s": 2.2349761249497533, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66355200, + "prompt_kind": "long", + "t_relative_s": 1450.1097154170275 + }, + { + "decode_latency_s": 0.06197110157518182, + "kv_live_bytes": 462274560, + "latency_s": 1.9838502081111073, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66912256, + "prompt_kind": "short", + "t_relative_s": 1455.1197580420412 + }, + { + "decode_latency_s": 0.06964224218972959, + "kv_live_bytes": 462274560, + "latency_s": 2.2310369997285306, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67010560, + "prompt_kind": "long", + "t_relative_s": 1460.1297835833393 + }, + { + "decode_latency_s": 0.061490838532336056, + "kv_live_bytes": 462274560, + "latency_s": 1.9685160829685628, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67567616, + "prompt_kind": "short", + "t_relative_s": 1465.1398555422202 + }, + { + "decode_latency_s": 0.06939648046682123, + "kv_live_bytes": 462274560, + "latency_s": 2.222664291970432, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67682304, + "prompt_kind": "long", + "t_relative_s": 1470.1499116253108 + }, + { + "decode_latency_s": 0.06110750522930175, + "kv_live_bytes": 462274560, + "latency_s": 1.9563647089526057, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67895296, + "prompt_kind": "short", + "t_relative_s": 1475.159966125153 + }, + { + "decode_latency_s": 0.06878758331004065, + "kv_live_bytes": 462274560, + "latency_s": 2.2033472917973995, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67944448, + "prompt_kind": "long", + "t_relative_s": 1480.170044875238 + }, + { + "decode_latency_s": 0.06390035156800877, + "kv_live_bytes": 462274560, + "latency_s": 2.045735416933894, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67993600, + "prompt_kind": "short", + "t_relative_s": 1485.1801037923433 + }, + { + "decode_latency_s": 0.06744628775049932, + "kv_live_bytes": 462274560, + "latency_s": 2.1602202500216663, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67239936, + "prompt_kind": "long", + "t_relative_s": 1490.1901762080379 + }, + { + "decode_latency_s": 0.06029493099777028, + "kv_live_bytes": 462274560, + "latency_s": 1.9303322918713093, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68337664, + "prompt_kind": "short", + "t_relative_s": 1495.2002274999395 + }, + { + "decode_latency_s": 0.06809578387765214, + "kv_live_bytes": 462274560, + "latency_s": 2.18114379234612, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68403200, + "prompt_kind": "long", + "t_relative_s": 1500.210306667257 + }, + { + "decode_latency_s": 0.06417240365408361, + "kv_live_bytes": 462274560, + "latency_s": 2.0544218332506716, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 65519616, + "prompt_kind": "short", + "t_relative_s": 1505.2203742079437 + }, + { + "decode_latency_s": 0.062218934908742085, + "kv_live_bytes": 462274560, + "latency_s": 1.9919655830599368, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66912256, + "prompt_kind": "long", + "t_relative_s": 1510.2304633753374 + }, + { + "decode_latency_s": 0.060914585934369825, + "kv_live_bytes": 462274560, + "latency_s": 1.9500990421511233, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 64782336, + "prompt_kind": "short", + "t_relative_s": 1515.2405150830746 + }, + { + "decode_latency_s": 0.06787935027386993, + "kv_live_bytes": 462274560, + "latency_s": 2.1750666247680783, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67846144, + "prompt_kind": "long", + "t_relative_s": 1520.2505790423602 + }, + { + "decode_latency_s": 0.06037422006193083, + "kv_live_bytes": 462274560, + "latency_s": 1.932741791009903, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68468736, + "prompt_kind": "short", + "t_relative_s": 1525.260664250236 + }, + { + "decode_latency_s": 0.06774625390244182, + "kv_live_bytes": 462274560, + "latency_s": 2.1700409171171486, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68665344, + "prompt_kind": "long", + "t_relative_s": 1530.2707407921553 + }, + { + "decode_latency_s": 0.060579565120860934, + "kv_live_bytes": 462274560, + "latency_s": 1.9394515831954777, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68714496, + "prompt_kind": "short", + "t_relative_s": 1535.2807750003412 + }, + { + "decode_latency_s": 0.06833879426994827, + "kv_live_bytes": 462274560, + "latency_s": 2.1888357917778194, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68960256, + "prompt_kind": "long", + "t_relative_s": 1540.2908508749679 + }, + { + "decode_latency_s": 0.06286633203853853, + "kv_live_bytes": 462274560, + "latency_s": 2.012590333353728, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67272704, + "prompt_kind": "short", + "t_relative_s": 1545.3009446249343 + }, + { + "decode_latency_s": 0.06788635415432509, + "kv_live_bytes": 462274560, + "latency_s": 2.1742948330938816, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68763648, + "prompt_kind": "long", + "t_relative_s": 1550.305206167046 + }, + { + "decode_latency_s": 0.06042916928709019, + "kv_live_bytes": 462274560, + "latency_s": 1.934629333205521, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69222400, + "prompt_kind": "short", + "t_relative_s": 1555.3152527082711 + }, + { + "decode_latency_s": 0.06745064971619286, + "kv_live_bytes": 462274560, + "latency_s": 2.1609560842625797, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69550080, + "prompt_kind": "long", + "t_relative_s": 1560.325327792205 + }, + { + "decode_latency_s": 0.06055718100105878, + "kv_live_bytes": 462274560, + "latency_s": 1.9386349166743457, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69566464, + "prompt_kind": "short", + "t_relative_s": 1565.3354199579917 + }, + { + "decode_latency_s": 0.06850576824217569, + "kv_live_bytes": 462274560, + "latency_s": 2.194456750061363, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69074944, + "prompt_kind": "long", + "t_relative_s": 1570.3454619171098 + }, + { + "decode_latency_s": 0.060767669274355285, + "kv_live_bytes": 462274560, + "latency_s": 1.9454601253382862, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 60260352, + "prompt_kind": "short", + "t_relative_s": 1575.3555335421115 + }, + { + "decode_latency_s": 0.06772953906329349, + "kv_live_bytes": 462274560, + "latency_s": 2.169820624869317, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68927488, + "prompt_kind": "long", + "t_relative_s": 1580.365621500183 + }, + { + "decode_latency_s": 0.06059357163030654, + "kv_live_bytes": 462274560, + "latency_s": 1.939764834009111, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69550080, + "prompt_kind": "short", + "t_relative_s": 1585.3756862501614 + }, + { + "decode_latency_s": 0.06800447915156838, + "kv_live_bytes": 462274560, + "latency_s": 2.178376874886453, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69877760, + "prompt_kind": "long", + "t_relative_s": 1590.3857313329354 + }, + { + "decode_latency_s": 0.06058308725187089, + "kv_live_bytes": 462274560, + "latency_s": 1.9395621661096811, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69746688, + "prompt_kind": "short", + "t_relative_s": 1595.395796125289 + }, + { + "decode_latency_s": 0.06819628775701858, + "kv_live_bytes": 462274560, + "latency_s": 2.183902082964778, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70074368, + "prompt_kind": "long", + "t_relative_s": 1600.4058715002611 + }, + { + "decode_latency_s": 0.06073883202043362, + "kv_live_bytes": 462274560, + "latency_s": 1.9444553749635816, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70057984, + "prompt_kind": "short", + "t_relative_s": 1605.415924708359 + }, + { + "decode_latency_s": 0.060563320308574475, + "kv_live_bytes": 462274560, + "latency_s": 1.9392815409228206, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69107712, + "prompt_kind": "long", + "t_relative_s": 1610.4260107502341 + }, + { + "decode_latency_s": 0.06670996484172065, + "kv_live_bytes": 462274560, + "latency_s": 2.1362737081944942, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69730304, + "prompt_kind": "short", + "t_relative_s": 1615.4361197082326 + }, + { + "decode_latency_s": 0.06796755990944803, + "kv_live_bytes": 462274560, + "latency_s": 2.1773668327368796, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70139904, + "prompt_kind": "long", + "t_relative_s": 1620.4445876250975 + }, + { + "decode_latency_s": 0.060543020837940276, + "kv_live_bytes": 462274560, + "latency_s": 1.9382731658406556, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70516736, + "prompt_kind": "short", + "t_relative_s": 1625.4546589581296 + }, + { + "decode_latency_s": 0.0680649531132076, + "kv_live_bytes": 462274560, + "latency_s": 2.1795728341676295, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70582272, + "prompt_kind": "long", + "t_relative_s": 1630.4647497502156 + }, + { + "decode_latency_s": 0.061579805973451585, + "kv_live_bytes": 462274560, + "latency_s": 1.971954165957868, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 61931520, + "prompt_kind": "short", + "t_relative_s": 1635.4747808333486 + }, + { + "decode_latency_s": 0.06839418750314508, + "kv_live_bytes": 462274560, + "latency_s": 2.191048333887011, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70270976, + "prompt_kind": "long", + "t_relative_s": 1640.4848520001397 + }, + { + "decode_latency_s": 0.06030623828701209, + "kv_live_bytes": 462274560, + "latency_s": 1.9306882498785853, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70778880, + "prompt_kind": "short", + "t_relative_s": 1645.4898629579693 + }, + { + "decode_latency_s": 0.06820715362846386, + "kv_live_bytes": 462274560, + "latency_s": 2.1842903341166675, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70877184, + "prompt_kind": "long", + "t_relative_s": 1650.499942583032 + }, + { + "decode_latency_s": 0.060544937499798834, + "kv_live_bytes": 462274560, + "latency_s": 1.9382719998247921, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70942720, + "prompt_kind": "short", + "t_relative_s": 1655.5099880420603 + }, + { + "decode_latency_s": 0.06755092968523968, + "kv_live_bytes": 462274560, + "latency_s": 2.1640255418606102, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71024640, + "prompt_kind": "long", + "t_relative_s": 1660.520058083348 + }, + { + "decode_latency_s": 0.06069472525268793, + "kv_live_bytes": 462274560, + "latency_s": 1.9430142911151052, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71663616, + "prompt_kind": "short", + "t_relative_s": 1665.5301331672817 + }, + { + "decode_latency_s": 0.06778555859636981, + "kv_live_bytes": 462274560, + "latency_s": 2.171236792113632, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70844416, + "prompt_kind": "long", + "t_relative_s": 1670.54019012535 + }, + { + "decode_latency_s": 0.060307847656076774, + "kv_live_bytes": 462274560, + "latency_s": 1.9307794170454144, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70762496, + "prompt_kind": "short", + "t_relative_s": 1675.5502710831352 + }, + { + "decode_latency_s": 0.0674680833471939, + "kv_live_bytes": 462274560, + "latency_s": 2.161213458981365, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71467008, + "prompt_kind": "long", + "t_relative_s": 1680.5603376249783 + }, + { + "decode_latency_s": 0.06093237368622795, + "kv_live_bytes": 462274560, + "latency_s": 1.9506503329612315, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71499776, + "prompt_kind": "short", + "t_relative_s": 1685.570397167001 + }, + { + "decode_latency_s": 0.06794419010111596, + "kv_live_bytes": 462274560, + "latency_s": 2.175957459025085, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 64995328, + "prompt_kind": "long", + "t_relative_s": 1690.580472792033 + }, + { + "decode_latency_s": 0.06097813934320584, + "kv_live_bytes": 462274560, + "latency_s": 1.9522199998609722, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67584000, + "prompt_kind": "short", + "t_relative_s": 1695.5905627501197 + }, + { + "decode_latency_s": 0.06792411590868142, + "kv_live_bytes": 462274560, + "latency_s": 2.175510459113866, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71090176, + "prompt_kind": "long", + "t_relative_s": 1700.600634417031 + }, + { + "decode_latency_s": 0.06055870311683975, + "kv_live_bytes": 462274560, + "latency_s": 1.9388122092932463, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71712768, + "prompt_kind": "short", + "t_relative_s": 1705.610688582994 + }, + { + "decode_latency_s": 0.06800493490300141, + "kv_live_bytes": 462274560, + "latency_s": 2.1785297081805766, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71811072, + "prompt_kind": "long", + "t_relative_s": 1710.6207912499085 + }, + { + "decode_latency_s": 0.06071289842657279, + "kv_live_bytes": 462274560, + "latency_s": 1.9435736248269677, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71860224, + "prompt_kind": "short", + "t_relative_s": 1715.6308932919055 + }, + { + "decode_latency_s": 0.06834724350483157, + "kv_live_bytes": 462274560, + "latency_s": 2.1892688339576125, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71811072, + "prompt_kind": "long", + "t_relative_s": 1720.6371674579568 + }, + { + "decode_latency_s": 0.06109645703691058, + "kv_live_bytes": 462274560, + "latency_s": 1.9560174997895956, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72040448, + "prompt_kind": "short", + "t_relative_s": 1725.6472457922064 + }, + { + "decode_latency_s": 0.06814573047449812, + "kv_live_bytes": 462274560, + "latency_s": 2.183418999891728, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71925760, + "prompt_kind": "long", + "t_relative_s": 1730.6572999581695 + }, + { + "decode_latency_s": 0.06035154296841938, + "kv_live_bytes": 462274560, + "latency_s": 1.9320242921821773, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72515584, + "prompt_kind": "short", + "t_relative_s": 1735.6673743752763 + }, + { + "decode_latency_s": 0.0682242070324719, + "kv_live_bytes": 462274560, + "latency_s": 2.1847572079859674, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69500928, + "prompt_kind": "long", + "t_relative_s": 1740.677415125072 + }, + { + "decode_latency_s": 0.06051239580847323, + "kv_live_bytes": 462274560, + "latency_s": 1.9373468337580562, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71041024, + "prompt_kind": "short", + "t_relative_s": 1745.6874622502364 + }, + { + "decode_latency_s": 0.06788441014941782, + "kv_live_bytes": 462274560, + "latency_s": 2.1747739580459893, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71843840, + "prompt_kind": "long", + "t_relative_s": 1750.697519833222 + }, + { + "decode_latency_s": 0.06088101952627767, + "kv_live_bytes": 462274560, + "latency_s": 1.9491099170409143, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68206592, + "prompt_kind": "short", + "t_relative_s": 1755.7076048329473 + }, + { + "decode_latency_s": 0.06775340234162286, + "kv_live_bytes": 462274560, + "latency_s": 2.170519624836743, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71516160, + "prompt_kind": "long", + "t_relative_s": 1760.7176568750292 + }, + { + "decode_latency_s": 0.06677685416070744, + "kv_live_bytes": 462274560, + "latency_s": 2.138525791000575, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72089600, + "prompt_kind": "short", + "t_relative_s": 1765.7277350421064 + }, + { + "decode_latency_s": 0.06794726171938237, + "kv_live_bytes": 462274560, + "latency_s": 2.176390249747783, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72351744, + "prompt_kind": "long", + "t_relative_s": 1770.737786917016 + }, + { + "decode_latency_s": 0.060487596347229555, + "kv_live_bytes": 462274560, + "latency_s": 1.9365250002592802, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72400896, + "prompt_kind": "short", + "t_relative_s": 1775.7478611250408 + }, + { + "decode_latency_s": 0.06899679428897798, + "kv_live_bytes": 462274560, + "latency_s": 2.2097465000115335, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71516160, + "prompt_kind": "long", + "t_relative_s": 1780.7579247499816 + }, + { + "decode_latency_s": 0.0615046588354744, + "kv_live_bytes": 462274560, + "latency_s": 1.969066542107612, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71614464, + "prompt_kind": "short", + "t_relative_s": 1785.7636851249263 + }, + { + "decode_latency_s": 0.06056658203306142, + "kv_live_bytes": 462274560, + "latency_s": 1.9391940422356129, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71860224, + "prompt_kind": "long", + "t_relative_s": 1790.7737428331748 + }, + { + "decode_latency_s": 0.06044264194497373, + "kv_live_bytes": 462274560, + "latency_s": 1.9350872919894755, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71794688, + "prompt_kind": "short", + "t_relative_s": 1795.7838294170797 + }, + { + "decode_latency_s": 0.06797263280896004, + "kv_live_bytes": 462274560, + "latency_s": 2.1771807922050357, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72515584, + "prompt_kind": "long", + "t_relative_s": 1800.7939154170454 + }, + { + "decode_latency_s": 0.060891898450790904, + "kv_live_bytes": 462274560, + "latency_s": 1.9494722918607295, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72613888, + "prompt_kind": "short", + "t_relative_s": 1805.8039679583162 + }, + { + "decode_latency_s": 0.06778315624978859, + "kv_live_bytes": 462274560, + "latency_s": 2.171589750330895, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72663040, + "prompt_kind": "long", + "t_relative_s": 1810.8140465002507 + }, + { + "decode_latency_s": 0.06323009896732401, + "kv_live_bytes": 462274560, + "latency_s": 2.0251695830374956, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69238784, + "prompt_kind": "short", + "t_relative_s": 1815.8241708329879 + }, + { + "decode_latency_s": 0.06745718487945851, + "kv_live_bytes": 462274560, + "latency_s": 2.161487374920398, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71778304, + "prompt_kind": "long", + "t_relative_s": 1820.834213042166 + }, + { + "decode_latency_s": 0.06025869531731587, + "kv_live_bytes": 462274560, + "latency_s": 1.9291422087699175, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72433664, + "prompt_kind": "short", + "t_relative_s": 1825.844289583154 + }, + { + "decode_latency_s": 0.06785117837716825, + "kv_live_bytes": 462274560, + "latency_s": 2.1732110837474465, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72613888, + "prompt_kind": "long", + "t_relative_s": 1830.8543591252528 + }, + { + "decode_latency_s": 0.060498847655253485, + "kv_live_bytes": 462274560, + "latency_s": 1.9382116249762475, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72761344, + "prompt_kind": "short", + "t_relative_s": 1835.8644121252 + }, + { + "decode_latency_s": 0.06807242319337092, + "kv_live_bytes": 462274560, + "latency_s": 2.1806609169580042, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72794112, + "prompt_kind": "long", + "t_relative_s": 1840.8744786670431 + }, + { + "decode_latency_s": 0.06792516278801486, + "kv_live_bytes": 462274560, + "latency_s": 2.175082332920283, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70090752, + "prompt_kind": "short", + "t_relative_s": 1845.884611458052 + }, + { + "decode_latency_s": 0.06083390624553431, + "kv_live_bytes": 462274560, + "latency_s": 1.9480622918345034, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71630848, + "prompt_kind": "long", + "t_relative_s": 1850.8947176672518 + }, + { + "decode_latency_s": 0.06534662758349441, + "kv_live_bytes": 462274560, + "latency_s": 2.092951124999672, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72237056, + "prompt_kind": "short", + "t_relative_s": 1855.9017727919854 + }, + { + "decode_latency_s": 0.06761543618631549, + "kv_live_bytes": 462274560, + "latency_s": 2.166257250122726, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72794112, + "prompt_kind": "long", + "t_relative_s": 1860.9118597921915 + }, + { + "decode_latency_s": 0.06725896224088501, + "kv_live_bytes": 462274560, + "latency_s": 2.1541982502676547, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71647232, + "prompt_kind": "short", + "t_relative_s": 1865.921930667013 + }, + { + "decode_latency_s": 0.06818767187360208, + "kv_live_bytes": 462274560, + "latency_s": 2.1843582089059055, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71974912, + "prompt_kind": "long", + "t_relative_s": 1870.93198016705 + }, + { + "decode_latency_s": 0.06104045834217686, + "kv_live_bytes": 462274560, + "latency_s": 1.9542069998569787, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 61702144, + "prompt_kind": "short", + "t_relative_s": 1875.9420476253144 + }, + { + "decode_latency_s": 0.06788143618905451, + "kv_live_bytes": 462274560, + "latency_s": 2.175105709116906, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71843840, + "prompt_kind": "long", + "t_relative_s": 1880.952098458074 + }, + { + "decode_latency_s": 0.06040409894194454, + "kv_live_bytes": 462274560, + "latency_s": 1.9336960832588375, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72499200, + "prompt_kind": "short", + "t_relative_s": 1885.9621815420687 + }, + { + "decode_latency_s": 0.0684048294351669, + "kv_live_bytes": 462274560, + "latency_s": 2.190417249687016, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72351744, + "prompt_kind": "long", + "t_relative_s": 1890.9722335832193 + }, + { + "decode_latency_s": 0.060702587259584107, + "kv_live_bytes": 462274560, + "latency_s": 1.9434289578348398, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72548352, + "prompt_kind": "short", + "t_relative_s": 1895.9822907499038 + }, + { + "decode_latency_s": 0.06821351694816258, + "kv_live_bytes": 462274560, + "latency_s": 2.1845196252688766, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72564736, + "prompt_kind": "long", + "t_relative_s": 1900.9923702920787 + }, + { + "decode_latency_s": 0.06760286977805663, + "kv_live_bytes": 462274560, + "latency_s": 2.1647273330017924, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70762496, + "prompt_kind": "short", + "t_relative_s": 1906.002415667288 + }, + { + "decode_latency_s": 0.06769464453100227, + "kv_live_bytes": 462274560, + "latency_s": 2.1683454159647226, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71909376, + "prompt_kind": "long", + "t_relative_s": 1911.012500333134 + }, + { + "decode_latency_s": 0.06069512109388597, + "kv_live_bytes": 462274560, + "latency_s": 1.943179083056748, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72679424, + "prompt_kind": "short", + "t_relative_s": 1916.022547250148 + }, + { + "decode_latency_s": 0.06782105467573274, + "kv_live_bytes": 462274560, + "latency_s": 2.1730297077447176, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72794112, + "prompt_kind": "long", + "t_relative_s": 1921.032626207918 + }, + { + "decode_latency_s": 0.06091237367945723, + "kv_live_bytes": 462274560, + "latency_s": 1.9499806673265994, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72859648, + "prompt_kind": "short", + "t_relative_s": 1926.0427029170096 + }, + { + "decode_latency_s": 0.06789897002454381, + "kv_live_bytes": 462274560, + "latency_s": 2.174862415995449, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "long", + "t_relative_s": 1931.0527466251515 + }, + { + "decode_latency_s": 0.06096967056510039, + "kv_live_bytes": 462274560, + "latency_s": 1.9519523340277374, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 58834944, + "prompt_kind": "short", + "t_relative_s": 1936.0628281673416 + }, + { + "decode_latency_s": 0.06846514712378848, + "kv_live_bytes": 462274560, + "latency_s": 2.1933315419591963, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70303744, + "prompt_kind": "long", + "t_relative_s": 1941.0729217920452 + }, + { + "decode_latency_s": 0.06028796485043131, + "kv_live_bytes": 462274560, + "latency_s": 1.9300916250795126, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72630272, + "prompt_kind": "short", + "t_relative_s": 1946.0829816251062 + }, + { + "decode_latency_s": 0.06801981640455779, + "kv_live_bytes": 462274560, + "latency_s": 2.178756915964186, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72728576, + "prompt_kind": "long", + "t_relative_s": 1951.0930411671288 + }, + { + "decode_latency_s": 0.06087672393186949, + "kv_live_bytes": 462274560, + "latency_s": 1.9488792079500854, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72761344, + "prompt_kind": "short", + "t_relative_s": 1956.1031152079813 + }, + { + "decode_latency_s": 0.06079816406418104, + "kv_live_bytes": 462274560, + "latency_s": 1.9465238749980927, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72843264, + "prompt_kind": "long", + "t_relative_s": 1961.1131708333269 + }, + { + "decode_latency_s": 0.06047411328472663, + "kv_live_bytes": 462274560, + "latency_s": 1.935925125144422, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72843264, + "prompt_kind": "short", + "t_relative_s": 1966.123244999908 + }, + { + "decode_latency_s": 0.06874520440760534, + "kv_live_bytes": 462274560, + "latency_s": 2.202666707802564, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70254592, + "prompt_kind": "long", + "t_relative_s": 1971.1333146672696 + }, + { + "decode_latency_s": 0.06282107293372974, + "kv_live_bytes": 462274560, + "latency_s": 2.0113020832650363, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69730304, + "prompt_kind": "short", + "t_relative_s": 1976.1434309580363 + }, + { + "decode_latency_s": 0.06815806639497168, + "kv_live_bytes": 462274560, + "latency_s": 2.183096833061427, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71843840, + "prompt_kind": "long", + "t_relative_s": 1981.1535049169324 + }, + { + "decode_latency_s": 0.06067771225934848, + "kv_live_bytes": 462274560, + "latency_s": 1.9425205830484629, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71909376, + "prompt_kind": "short", + "t_relative_s": 1986.1635585422628 + }, + { + "decode_latency_s": 0.06794416147749871, + "kv_live_bytes": 462274560, + "latency_s": 2.176565082743764, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71991296, + "prompt_kind": "long", + "t_relative_s": 1991.1736097079702 + }, + { + "decode_latency_s": 0.06296080209722277, + "kv_live_bytes": 462274560, + "latency_s": 2.015626832842827, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71417856, + "prompt_kind": "short", + "t_relative_s": 1996.1837067082524 + }, + { + "decode_latency_s": 0.06806397525360808, + "kv_live_bytes": 462274560, + "latency_s": 2.180029584094882, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72105984, + "prompt_kind": "long", + "t_relative_s": 2001.193786750082 + }, + { + "decode_latency_s": 0.060763690096791834, + "kv_live_bytes": 462274560, + "latency_s": 1.9452970419079065, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "short", + "t_relative_s": 2006.1981925829314 + }, + { + "decode_latency_s": 0.06794467056170106, + "kv_live_bytes": 462274560, + "latency_s": 2.1764059588313103, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72695808, + "prompt_kind": "long", + "t_relative_s": 2011.208236375358 + }, + { + "decode_latency_s": 0.0606741549418075, + "kv_live_bytes": 462274560, + "latency_s": 1.9423540839925408, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72777728, + "prompt_kind": "short", + "t_relative_s": 2016.2183205001056 + }, + { + "decode_latency_s": 0.06814986196695827, + "kv_live_bytes": 462274560, + "latency_s": 2.182687332853675, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72056832, + "prompt_kind": "long", + "t_relative_s": 2021.2283797082491 + }, + { + "decode_latency_s": 0.06042550000711344, + "kv_live_bytes": 462274560, + "latency_s": 1.9345734999515116, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70746112, + "prompt_kind": "short", + "t_relative_s": 2026.2384292921051 + }, + { + "decode_latency_s": 0.0679433801851701, + "kv_live_bytes": 462274560, + "latency_s": 2.1759172920137644, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72187904, + "prompt_kind": "long", + "t_relative_s": 2031.248514083214 + }, + { + "decode_latency_s": 0.060490044284961186, + "kv_live_bytes": 462274560, + "latency_s": 1.936506707686931, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72712192, + "prompt_kind": "short", + "t_relative_s": 2036.2585586672649 + }, + { + "decode_latency_s": 0.06038169794192072, + "kv_live_bytes": 462274560, + "latency_s": 1.9334554160013795, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72728576, + "prompt_kind": "long", + "t_relative_s": 2041.2686271253042 + }, + { + "decode_latency_s": 0.060685393211315386, + "kv_live_bytes": 462274560, + "latency_s": 1.9427237082272768, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72810496, + "prompt_kind": "short", + "t_relative_s": 2046.2737797079608 + }, + { + "decode_latency_s": 0.06818147006561048, + "kv_live_bytes": 462274560, + "latency_s": 2.1838472499512136, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71974912, + "prompt_kind": "long", + "t_relative_s": 2051.283832750283 + }, + { + "decode_latency_s": 0.06098677216505166, + "kv_live_bytes": 462274560, + "latency_s": 1.9525262918323278, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71221248, + "prompt_kind": "short", + "t_relative_s": 2056.293911250308 + }, + { + "decode_latency_s": 0.06796326172479894, + "kv_live_bytes": 462274560, + "latency_s": 2.1764466669410467, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72024064, + "prompt_kind": "long", + "t_relative_s": 2061.3039859579876 + }, + { + "decode_latency_s": 0.06051109507097863, + "kv_live_bytes": 462274560, + "latency_s": 1.9371712910942733, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72581120, + "prompt_kind": "short", + "t_relative_s": 2066.31403445825 + }, + { + "decode_latency_s": 0.06778459636552725, + "kv_live_bytes": 462274560, + "latency_s": 2.1727698747999966, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72630272, + "prompt_kind": "long", + "t_relative_s": 2071.324111542199 + }, + { + "decode_latency_s": 0.06076252993079834, + "kv_live_bytes": 462274560, + "latency_s": 1.945192875340581, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72777728, + "prompt_kind": "short", + "t_relative_s": 2076.3342170421965 + }, + { + "decode_latency_s": 0.06795046615297906, + "kv_live_bytes": 462274560, + "latency_s": 2.1768742501735687, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72859648, + "prompt_kind": "long", + "t_relative_s": 2081.3442560001276 + }, + { + "decode_latency_s": 0.06180050912371371, + "kv_live_bytes": 462274560, + "latency_s": 1.9788226671516895, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71417856, + "prompt_kind": "short", + "t_relative_s": 2086.3543121670373 + }, + { + "decode_latency_s": 0.06815846743120346, + "kv_live_bytes": 462274560, + "latency_s": 2.1828344156965613, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69173248, + "prompt_kind": "long", + "t_relative_s": 2091.364367207978 + }, + { + "decode_latency_s": 0.06649683984869625, + "kv_live_bytes": 462274560, + "latency_s": 2.1296003330498934, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71745536, + "prompt_kind": "short", + "t_relative_s": 2096.37444412522 + }, + { + "decode_latency_s": 0.06858281508903019, + "kv_live_bytes": 462274560, + "latency_s": 2.1968507920391858, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "long", + "t_relative_s": 2101.384518583305 + }, + { + "decode_latency_s": 0.060613234381889924, + "kv_live_bytes": 462274560, + "latency_s": 1.9404157921671867, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72761344, + "prompt_kind": "short", + "t_relative_s": 2106.3946230830625 + }, + { + "decode_latency_s": 0.06856772786704823, + "kv_live_bytes": 462274560, + "latency_s": 2.196238291915506, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72122368, + "prompt_kind": "long", + "t_relative_s": 2111.4046715423465 + }, + { + "decode_latency_s": 0.06589160936709959, + "kv_live_bytes": 462274560, + "latency_s": 2.1098048333078623, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71106560, + "prompt_kind": "short", + "t_relative_s": 2116.414737583138 + }, + { + "decode_latency_s": 0.06837312108837068, + "kv_live_bytes": 462274560, + "latency_s": 2.1903833341784775, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72040448, + "prompt_kind": "long", + "t_relative_s": 2121.424785000272 + }, + { + "decode_latency_s": 0.060407764322008006, + "kv_live_bytes": 462274560, + "latency_s": 1.933817416895181, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "short", + "t_relative_s": 2126.434856292326 + }, + { + "decode_latency_s": 0.06778889062115923, + "kv_live_bytes": 462274560, + "latency_s": 2.1713567501865327, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72564736, + "prompt_kind": "long", + "t_relative_s": 2131.4448943752795 + }, + { + "decode_latency_s": 0.06047181382018607, + "kv_live_bytes": 462274560, + "latency_s": 1.9360209172591567, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72269824, + "prompt_kind": "short", + "t_relative_s": 2136.454947750084 + }, + { + "decode_latency_s": 0.06956989715399686, + "kv_live_bytes": 462274560, + "latency_s": 2.228588374797255, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71778304, + "prompt_kind": "long", + "t_relative_s": 2141.4650292503648 + }, + { + "decode_latency_s": 0.060421160160331056, + "kv_live_bytes": 462274560, + "latency_s": 1.9343499997630715, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71663616, + "prompt_kind": "short", + "t_relative_s": 2146.4751159171574 + }, + { + "decode_latency_s": 0.06710695312358439, + "kv_live_bytes": 462274560, + "latency_s": 2.149330750107765, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71712768, + "prompt_kind": "long", + "t_relative_s": 2151.485223833006 + }, + { + "decode_latency_s": 0.06071485416032374, + "kv_live_bytes": 462274560, + "latency_s": 1.9437371250241995, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67567616, + "prompt_kind": "short", + "t_relative_s": 2156.4952562502585 + }, + { + "decode_latency_s": 0.06763348959793802, + "kv_live_bytes": 462274560, + "latency_s": 2.1670751250348985, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70303744, + "prompt_kind": "long", + "t_relative_s": 2161.5052988333628 + }, + { + "decode_latency_s": 0.06080800521885976, + "kv_live_bytes": 462274560, + "latency_s": 1.9468324580229819, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70877184, + "prompt_kind": "short", + "t_relative_s": 2166.515368500259 + }, + { + "decode_latency_s": 0.0609132122626761, + "kv_live_bytes": 462274560, + "latency_s": 1.9501121249049902, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71696384, + "prompt_kind": "long", + "t_relative_s": 2171.52543991711 + }, + { + "decode_latency_s": 0.06661226562573574, + "kv_live_bytes": 462274560, + "latency_s": 2.1333864997141063, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71286784, + "prompt_kind": "short", + "t_relative_s": 2176.535493250005 + }, + { + "decode_latency_s": 0.0679944010626059, + "kv_live_bytes": 462274560, + "latency_s": 2.1779697500169277, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72187904, + "prompt_kind": "long", + "t_relative_s": 2181.5455449582078 + }, + { + "decode_latency_s": 0.06032263413362671, + "kv_live_bytes": 462274560, + "latency_s": 1.93118058424443, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72777728, + "prompt_kind": "short", + "t_relative_s": 2186.555616958067 + }, + { + "decode_latency_s": 0.06801215757150203, + "kv_live_bytes": 462274560, + "latency_s": 2.177969792392105, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72237056, + "prompt_kind": "long", + "t_relative_s": 2191.565686458256 + }, + { + "decode_latency_s": 0.06060220312792808, + "kv_live_bytes": 462274560, + "latency_s": 1.9401437500491738, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71778304, + "prompt_kind": "short", + "t_relative_s": 2196.5757410419174 + }, + { + "decode_latency_s": 0.06882680469425395, + "kv_live_bytes": 462274560, + "latency_s": 2.2047631251625717, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72417280, + "prompt_kind": "long", + "t_relative_s": 2201.58581583295 + }, + { + "decode_latency_s": 0.06200250258552842, + "kv_live_bytes": 462274560, + "latency_s": 1.9850927917286754, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68714496, + "prompt_kind": "short", + "t_relative_s": 2206.5958785829134 + }, + { + "decode_latency_s": 0.06829261197708547, + "kv_live_bytes": 462274560, + "latency_s": 2.187285208143294, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71761920, + "prompt_kind": "long", + "t_relative_s": 2211.606047499925 + }, + { + "decode_latency_s": 0.0604536198079586, + "kv_live_bytes": 462274560, + "latency_s": 1.93554054107517, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72417280, + "prompt_kind": "short", + "t_relative_s": 2216.616096457932 + }, + { + "decode_latency_s": 0.06783333983912598, + "kv_live_bytes": 462274560, + "latency_s": 2.1729906662367284, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73138176, + "prompt_kind": "long", + "t_relative_s": 2221.6261602500454 + }, + { + "decode_latency_s": 0.060807201807620004, + "kv_live_bytes": 462274560, + "latency_s": 1.9466360001824796, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73154560, + "prompt_kind": "short", + "t_relative_s": 2226.6362675833516 + }, + { + "decode_latency_s": 0.06898434375762008, + "kv_live_bytes": 462274560, + "latency_s": 2.209586290642619, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72630272, + "prompt_kind": "long", + "t_relative_s": 2231.646324167028 + }, + { + "decode_latency_s": 0.0674438724090578, + "kv_live_bytes": 462274560, + "latency_s": 2.160233417060226, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71483392, + "prompt_kind": "short", + "t_relative_s": 2236.6563998330384 + }, + { + "decode_latency_s": 0.06075381771370303, + "kv_live_bytes": 462274560, + "latency_s": 1.9450717498548329, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72318976, + "prompt_kind": "long", + "t_relative_s": 2241.666467042174 + }, + { + "decode_latency_s": 0.060531139344675466, + "kv_live_bytes": 462274560, + "latency_s": 1.9377922089770436, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72892416, + "prompt_kind": "short", + "t_relative_s": 2246.676550167147 + }, + { + "decode_latency_s": 0.06806225390755571, + "kv_live_bytes": 462274560, + "latency_s": 2.1801820411346853, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73039872, + "prompt_kind": "long", + "t_relative_s": 2251.6866057501175 + }, + { + "decode_latency_s": 0.06067408721719403, + "kv_live_bytes": 462274560, + "latency_s": 1.9425032916478813, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73138176, + "prompt_kind": "short", + "t_relative_s": 2256.6966630001552 + }, + { + "decode_latency_s": 0.06847732422465924, + "kv_live_bytes": 462274560, + "latency_s": 2.1930442498996854, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 51445760, + "prompt_kind": "long", + "t_relative_s": 2261.7067375420593 + }, + { + "decode_latency_s": 0.06052622133574914, + "kv_live_bytes": 462274560, + "latency_s": 1.937839874997735, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71712768, + "prompt_kind": "short", + "t_relative_s": 2266.716984833125 + }, + { + "decode_latency_s": 0.06766675389371812, + "kv_live_bytes": 462274560, + "latency_s": 2.1673479578457773, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71974912, + "prompt_kind": "long", + "t_relative_s": 2271.727052875329 + }, + { + "decode_latency_s": 0.06026338801893871, + "kv_live_bytes": 462274560, + "latency_s": 1.92940125009045, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72515584, + "prompt_kind": "short", + "t_relative_s": 2276.7371020829305 + }, + { + "decode_latency_s": 0.06756020702596288, + "kv_live_bytes": 462274560, + "latency_s": 2.1642151246778667, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73089024, + "prompt_kind": "long", + "t_relative_s": 2281.747189000249 + }, + { + "decode_latency_s": 0.06087866016605403, + "kv_live_bytes": 462274560, + "latency_s": 1.9489317922852933, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73302016, + "prompt_kind": "short", + "t_relative_s": 2286.757259833161 + }, + { + "decode_latency_s": 0.06846426299307495, + "kv_live_bytes": 462274560, + "latency_s": 2.1930574579164386, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73138176, + "prompt_kind": "long", + "t_relative_s": 2291.7673141672276 + }, + { + "decode_latency_s": 0.06319523956335615, + "kv_live_bytes": 462274560, + "latency_s": 2.02318162471056, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71303168, + "prompt_kind": "short", + "t_relative_s": 2296.777416958008 + }, + { + "decode_latency_s": 0.06788892447366379, + "kv_live_bytes": 462274560, + "latency_s": 2.174705000128597, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72400896, + "prompt_kind": "long", + "t_relative_s": 2301.78749662498 + }, + { + "decode_latency_s": 0.06058566666615661, + "kv_live_bytes": 462274560, + "latency_s": 1.9395244577899575, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72990720, + "prompt_kind": "short", + "t_relative_s": 2306.7975960420445 + }, + { + "decode_latency_s": 0.06036817577842157, + "kv_live_bytes": 462274560, + "latency_s": 1.9330954174511135, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73023488, + "prompt_kind": "long", + "t_relative_s": 2311.807639125269 + }, + { + "decode_latency_s": 0.0618202447949443, + "kv_live_bytes": 462274560, + "latency_s": 1.979215290863067, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73039872, + "prompt_kind": "short", + "t_relative_s": 2316.817718292121 + }, + { + "decode_latency_s": 0.06188895311788656, + "kv_live_bytes": 462274560, + "latency_s": 1.9813696253113449, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72384512, + "prompt_kind": "long", + "t_relative_s": 2321.827795207966 + }, + { + "decode_latency_s": 0.060985196629189886, + "kv_live_bytes": 462274560, + "latency_s": 1.952390959020704, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72040448, + "prompt_kind": "short", + "t_relative_s": 2326.8378718751483 + }, + { + "decode_latency_s": 0.06791804425301962, + "kv_live_bytes": 462274560, + "latency_s": 2.1755422088317573, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72433664, + "prompt_kind": "long", + "t_relative_s": 2331.8479261249304 + }, + { + "decode_latency_s": 0.06060562368656974, + "kv_live_bytes": 462274560, + "latency_s": 1.9401909578591585, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71958528, + "prompt_kind": "short", + "t_relative_s": 2336.8580053751357 + }, + { + "decode_latency_s": 0.06805175261979457, + "kv_live_bytes": 462274560, + "latency_s": 2.179639416281134, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73023488, + "prompt_kind": "long", + "t_relative_s": 2341.8680610000156 + }, + { + "decode_latency_s": 0.060505798159283586, + "kv_live_bytes": 462274560, + "latency_s": 1.9371342500671744, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72925184, + "prompt_kind": "short", + "t_relative_s": 2346.878145792056 + }, + { + "decode_latency_s": 0.0678695729438914, + "kv_live_bytes": 462274560, + "latency_s": 2.1736599169671535, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73056256, + "prompt_kind": "long", + "t_relative_s": 2351.8882205830887 + }, + { + "decode_latency_s": 0.06760681640298571, + "kv_live_bytes": 462274560, + "latency_s": 2.1651767501607537, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71286784, + "prompt_kind": "short", + "t_relative_s": 2356.8982814173214 + }, + { + "decode_latency_s": 0.06789383331488352, + "kv_live_bytes": 462274560, + "latency_s": 2.175034458283335, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72466432, + "prompt_kind": "long", + "t_relative_s": 2361.9083653753623 + }, + { + "decode_latency_s": 0.0608592734351987, + "kv_live_bytes": 462274560, + "latency_s": 1.9482890837825835, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73007104, + "prompt_kind": "short", + "t_relative_s": 2366.918480833061 + }, + { + "decode_latency_s": 0.06807060027495027, + "kv_live_bytes": 462274560, + "latency_s": 2.180694750044495, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73089024, + "prompt_kind": "long", + "t_relative_s": 2371.9285275419243 + }, + { + "decode_latency_s": 0.06041717187326867, + "kv_live_bytes": 462274560, + "latency_s": 1.934191457927227, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73187328, + "prompt_kind": "short", + "t_relative_s": 2376.938597917091 + }, + { + "decode_latency_s": 0.0684563333488768, + "kv_live_bytes": 462274560, + "latency_s": 2.192115291953087, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71925760, + "prompt_kind": "long", + "t_relative_s": 2381.9486948330887 + }, + { + "decode_latency_s": 0.06492669403087348, + "kv_live_bytes": 462274560, + "latency_s": 2.079634541179985, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70696960, + "prompt_kind": "short", + "t_relative_s": 2386.950455875136 + }, + { + "decode_latency_s": 0.0680259987275349, + "kv_live_bytes": 462274560, + "latency_s": 2.178735540714115, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72073216, + "prompt_kind": "long", + "t_relative_s": 2391.960510833189 + }, + { + "decode_latency_s": 0.0606039361882722, + "kv_live_bytes": 462274560, + "latency_s": 1.9402524158358574, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71532544, + "prompt_kind": "short", + "t_relative_s": 2396.9705729172565 + }, + { + "decode_latency_s": 0.06514013670675922, + "kv_live_bytes": 462274560, + "latency_s": 2.0858104578219354, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67305472, + "prompt_kind": "long", + "t_relative_s": 2401.9806201253086 + }, + { + "decode_latency_s": 0.060837449214886874, + "kv_live_bytes": 462274560, + "latency_s": 1.947667459025979, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69779456, + "prompt_kind": "short", + "t_relative_s": 2406.990701749921 + }, + { + "decode_latency_s": 0.06855314194399398, + "kv_live_bytes": 462274560, + "latency_s": 2.1951927919872105, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70713344, + "prompt_kind": "long", + "t_relative_s": 2412.000752333086 + }, + { + "decode_latency_s": 0.06303739713621326, + "kv_live_bytes": 462274560, + "latency_s": 2.018073834013194, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70254592, + "prompt_kind": "short", + "t_relative_s": 2417.010814375244 + }, + { + "decode_latency_s": 0.06043339581810869, + "kv_live_bytes": 462274560, + "latency_s": 1.9350458751432598, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69566464, + "prompt_kind": "long", + "t_relative_s": 2422.0208647502586 + }, + { + "decode_latency_s": 0.06052679818822071, + "kv_live_bytes": 462274560, + "latency_s": 1.9376516249030828, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70189056, + "prompt_kind": "short", + "t_relative_s": 2427.030937250238 + }, + { + "decode_latency_s": 0.06814521484193392, + "kv_live_bytes": 462274560, + "latency_s": 2.18235683394596, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70221824, + "prompt_kind": "long", + "t_relative_s": 2432.041015041992 + }, + { + "decode_latency_s": 0.0606170560058672, + "kv_live_bytes": 462274560, + "latency_s": 1.9407069589942694, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70270976, + "prompt_kind": "short", + "t_relative_s": 2437.0510620833375 + }, + { + "decode_latency_s": 0.06817932812555227, + "kv_live_bytes": 462274560, + "latency_s": 2.1835262500680983, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71008256, + "prompt_kind": "long", + "t_relative_s": 2442.0611350000836 + }, + { + "decode_latency_s": 0.061429859371855855, + "kv_live_bytes": 462274560, + "latency_s": 1.9665376250632107, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69386240, + "prompt_kind": "short", + "t_relative_s": 2447.0711906249635 + }, + { + "decode_latency_s": 0.06581366015598178, + "kv_live_bytes": 462274560, + "latency_s": 2.108117625117302, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70402048, + "prompt_kind": "long", + "t_relative_s": 2452.0812528333627 + }, + { + "decode_latency_s": 0.06061456771567464, + "kv_live_bytes": 462274560, + "latency_s": 1.9404036672785878, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70909952, + "prompt_kind": "short", + "t_relative_s": 2457.0913189579733 + }, + { + "decode_latency_s": 0.06758460415585432, + "kv_live_bytes": 462274560, + "latency_s": 2.16500825015828, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70959104, + "prompt_kind": "long", + "t_relative_s": 2462.1013822080567 + }, + { + "decode_latency_s": 0.060401380207622424, + "kv_live_bytes": 462274560, + "latency_s": 1.9337861253879964, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69648384, + "prompt_kind": "short", + "t_relative_s": 2467.1114567080513 + }, + { + "decode_latency_s": 0.06801823437854182, + "kv_live_bytes": 462274560, + "latency_s": 2.1783165829256177, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70418432, + "prompt_kind": "long", + "t_relative_s": 2472.121535083279 + }, + { + "decode_latency_s": 0.06119620834942907, + "kv_live_bytes": 462274560, + "latency_s": 1.9591498747467995, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69746688, + "prompt_kind": "short", + "t_relative_s": 2477.131584917195 + }, + { + "decode_latency_s": 0.06787326952326111, + "kv_live_bytes": 462274560, + "latency_s": 2.1735981251113117, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69042176, + "prompt_kind": "long", + "t_relative_s": 2482.141635583248 + }, + { + "decode_latency_s": 0.06050372656318359, + "kv_live_bytes": 462274560, + "latency_s": 1.9371052500791848, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69238784, + "prompt_kind": "short", + "t_relative_s": 2487.1516871252097 + }, + { + "decode_latency_s": 0.06814316277450416, + "kv_live_bytes": 462274560, + "latency_s": 2.18243029108271, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70959104, + "prompt_kind": "long", + "t_relative_s": 2492.161755166948 + }, + { + "decode_latency_s": 0.0604976393369725, + "kv_live_bytes": 462274560, + "latency_s": 1.936729250010103, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70991872, + "prompt_kind": "short", + "t_relative_s": 2497.1718272082508 + }, + { + "decode_latency_s": 0.06197552607045509, + "kv_live_bytes": 462274560, + "latency_s": 1.984041666612029, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70942720, + "prompt_kind": "long", + "t_relative_s": 2502.1818875833414 + }, + { + "decode_latency_s": 0.08938437369943131, + "kv_live_bytes": 462274560, + "latency_s": 2.8611742919310927, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66748416, + "prompt_kind": "short", + "t_relative_s": 2507.1839160830714 + }, + { + "decode_latency_s": 0.0675552395550767, + "kv_live_bytes": 462274560, + "latency_s": 2.164108084049076, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67371008, + "prompt_kind": "long", + "t_relative_s": 2514.563008583151 + }, + { + "decode_latency_s": 0.06680906769179273, + "kv_live_bytes": 462274560, + "latency_s": 2.1394009590148926, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67977216, + "prompt_kind": "short", + "t_relative_s": 2519.5731230000965 + }, + { + "decode_latency_s": 0.06095748697407544, + "kv_live_bytes": 462274560, + "latency_s": 1.9513886249624193, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68009984, + "prompt_kind": "long", + "t_relative_s": 2524.5831935000606 + }, + { + "decode_latency_s": 0.06059767316037323, + "kv_live_bytes": 462274560, + "latency_s": 1.9400095003657043, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67911680, + "prompt_kind": "short", + "t_relative_s": 2529.5932622500695 + }, + { + "decode_latency_s": 0.06846305467479397, + "kv_live_bytes": 462274560, + "latency_s": 2.1929673748090863, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 60030976, + "prompt_kind": "long", + "t_relative_s": 2534.603335833177 + }, + { + "decode_latency_s": 0.060713808590662666, + "kv_live_bytes": 462274560, + "latency_s": 1.9436084171757102, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 65060864, + "prompt_kind": "short", + "t_relative_s": 2539.6134069580585 + }, + { + "decode_latency_s": 0.06841148568491917, + "kv_live_bytes": 462274560, + "latency_s": 2.191669291816652, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67911680, + "prompt_kind": "long", + "t_relative_s": 2544.6234619999304 + }, + { + "decode_latency_s": 0.06035246353712864, + "kv_live_bytes": 462274560, + "latency_s": 1.9321657503023744, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68452352, + "prompt_kind": "short", + "t_relative_s": 2549.6335194171406 + }, + { + "decode_latency_s": 0.06773687110398896, + "kv_live_bytes": 462274560, + "latency_s": 2.1692857081070542, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68616192, + "prompt_kind": "long", + "t_relative_s": 2554.643592624925 + }, + { + "decode_latency_s": 0.0665502629999537, + "kv_live_bytes": 462274560, + "latency_s": 2.131417542230338, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68763648, + "prompt_kind": "short", + "t_relative_s": 2559.653641208075 + }, + { + "decode_latency_s": 0.06862719921628013, + "kv_live_bytes": 462274560, + "latency_s": 2.1985412081703544, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68173824, + "prompt_kind": "long", + "t_relative_s": 2564.663738375064 + }, + { + "decode_latency_s": 0.06091420180746354, + "kv_live_bytes": 462274560, + "latency_s": 1.9501002500765026, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68386816, + "prompt_kind": "short", + "t_relative_s": 2569.6738178329542 + }, + { + "decode_latency_s": 0.0679313320433721, + "kv_live_bytes": 462274560, + "latency_s": 2.1758236247114837, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68829184, + "prompt_kind": "long", + "t_relative_s": 2574.683870833367 + }, + { + "decode_latency_s": 0.060425733056035824, + "kv_live_bytes": 462274560, + "latency_s": 1.9345247498713434, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69353472, + "prompt_kind": "short", + "t_relative_s": 2579.69394012494 + }, + { + "decode_latency_s": 0.06838295052875765, + "kv_live_bytes": 462274560, + "latency_s": 2.1894746250472963, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69386240, + "prompt_kind": "long", + "t_relative_s": 2584.704026750289 + }, + { + "decode_latency_s": 0.06658519789925776, + "kv_live_bytes": 462274560, + "latency_s": 2.132905332837254, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69419008, + "prompt_kind": "short", + "t_relative_s": 2589.7140768752433 + }, + { + "decode_latency_s": 0.06950667320052162, + "kv_live_bytes": 462274560, + "latency_s": 2.226425249595195, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 61718528, + "prompt_kind": "long", + "t_relative_s": 2594.724163208157 + }, + { + "decode_latency_s": 0.06125021225307137, + "kv_live_bytes": 462274560, + "latency_s": 1.9609932089224458, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 64962560, + "prompt_kind": "short", + "t_relative_s": 2599.734238583129 + }, + { + "decode_latency_s": 0.06762338930275291, + "kv_live_bytes": 462274560, + "latency_s": 2.165542332921177, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68435968, + "prompt_kind": "long", + "t_relative_s": 2604.7443277081475 + }, + { + "decode_latency_s": 0.060638371083769016, + "kv_live_bytes": 462274560, + "latency_s": 1.9412924167700112, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69124096, + "prompt_kind": "short", + "t_relative_s": 2609.7543727499433 + }, + { + "decode_latency_s": 0.06827897006587591, + "kv_live_bytes": 462274560, + "latency_s": 2.18707637488842, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69566464, + "prompt_kind": "long", + "t_relative_s": 2614.7644443330355 + }, + { + "decode_latency_s": 0.06668726303905714, + "kv_live_bytes": 462274560, + "latency_s": 2.1356739164330065, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69681152, + "prompt_kind": "short", + "t_relative_s": 2619.7745320000686 + }, + { + "decode_latency_s": 0.06818250780634116, + "kv_live_bytes": 462274560, + "latency_s": 2.184393874835223, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68960256, + "prompt_kind": "long", + "t_relative_s": 2624.7845898331143 + }, + { + "decode_latency_s": 0.06049900781363249, + "kv_live_bytes": 462274560, + "latency_s": 1.9368225000798702, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69599232, + "prompt_kind": "short", + "t_relative_s": 2629.7946863332763 + }, + { + "decode_latency_s": 0.06511200782551896, + "kv_live_bytes": 462274560, + "latency_s": 2.084859250113368, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69353472, + "prompt_kind": "long", + "t_relative_s": 2634.8047800422646 + }, + { + "decode_latency_s": 0.06047232162381988, + "kv_live_bytes": 462274560, + "latency_s": 1.936022374778986, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68272128, + "prompt_kind": "short", + "t_relative_s": 2639.814827958122 + }, + { + "decode_latency_s": 0.06487816665321589, + "kv_live_bytes": 462274560, + "latency_s": 2.0773811251856387, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68763648, + "prompt_kind": "long", + "t_relative_s": 2644.8248500423506 + }, + { + "decode_latency_s": 0.06552805077808443, + "kv_live_bytes": 462274560, + "latency_s": 2.0986275421455503, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67780608, + "prompt_kind": "short", + "t_relative_s": 2649.834905249998 + }, + { + "decode_latency_s": 0.06791826304106507, + "kv_live_bytes": 462274560, + "latency_s": 2.1761467093601823, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69320704, + "prompt_kind": "long", + "t_relative_s": 2654.8449790831655 + }, + { + "decode_latency_s": 0.0606640599144157, + "kv_live_bytes": 462274560, + "latency_s": 1.9420542907901108, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69959680, + "prompt_kind": "short", + "t_relative_s": 2659.855081083253 + }, + { + "decode_latency_s": 0.06854122393997386, + "kv_live_bytes": 462274560, + "latency_s": 2.196035500150174, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70057984, + "prompt_kind": "long", + "t_relative_s": 2664.8651256249286 + }, + { + "decode_latency_s": 0.060436481740907766, + "kv_live_bytes": 462274560, + "latency_s": 1.9348557502962649, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70500352, + "prompt_kind": "short", + "t_relative_s": 2669.875200957991 + }, + { + "decode_latency_s": 0.06829017706331797, + "kv_live_bytes": 462274560, + "latency_s": 2.1867584590800107, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70696960, + "prompt_kind": "long", + "t_relative_s": 2674.8852830831893 + }, + { + "decode_latency_s": 0.060538582038134336, + "kv_live_bytes": 462274560, + "latency_s": 1.9381156670860946, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70844416, + "prompt_kind": "short", + "t_relative_s": 2679.8953270423226 + }, + { + "decode_latency_s": 0.06783645311952569, + "kv_live_bytes": 462274560, + "latency_s": 2.172951125074178, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69058560, + "prompt_kind": "long", + "t_relative_s": 2684.9053881252185 + }, + { + "decode_latency_s": 0.06053589843213558, + "kv_live_bytes": 462274560, + "latency_s": 1.9380185841582716, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67993600, + "prompt_kind": "short", + "t_relative_s": 2689.9154611672275 + }, + { + "decode_latency_s": 0.06803127865714487, + "kv_live_bytes": 462274560, + "latency_s": 2.179083624854684, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70778880, + "prompt_kind": "long", + "t_relative_s": 2694.9255232079886 + }, + { + "decode_latency_s": 0.060625790370977484, + "kv_live_bytes": 462274560, + "latency_s": 1.940855666063726, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71385088, + "prompt_kind": "short", + "t_relative_s": 2699.9355722079054 + }, + { + "decode_latency_s": 0.06818326041684486, + "kv_live_bytes": 462274560, + "latency_s": 2.184345708694309, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71516160, + "prompt_kind": "long", + "t_relative_s": 2704.9456612919457 + }, + { + "decode_latency_s": 0.060846368491183966, + "kv_live_bytes": 462274560, + "latency_s": 1.9478280418552458, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71532544, + "prompt_kind": "short", + "t_relative_s": 2709.9557314580306 + }, + { + "decode_latency_s": 0.06817817577393726, + "kv_live_bytes": 462274560, + "latency_s": 2.1840185415931046, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70877184, + "prompt_kind": "long", + "t_relative_s": 2714.96578541724 + }, + { + "decode_latency_s": 0.06041672137507703, + "kv_live_bytes": 462274560, + "latency_s": 1.9342200420796871, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70975488, + "prompt_kind": "short", + "t_relative_s": 2719.9758609170094 + }, + { + "decode_latency_s": 0.0682867499999702, + "kv_live_bytes": 462274560, + "latency_s": 2.1868697092868388, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71598080, + "prompt_kind": "long", + "t_relative_s": 2724.9859518329613 + }, + { + "decode_latency_s": 0.06138890885631554, + "kv_live_bytes": 462274560, + "latency_s": 1.9657960841432214, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72138752, + "prompt_kind": "short", + "t_relative_s": 2729.9960164581425 + }, + { + "decode_latency_s": 0.06845481118944008, + "kv_live_bytes": 462274560, + "latency_s": 2.1929053752683103, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72073216, + "prompt_kind": "long", + "t_relative_s": 2735.006104625296 + }, + { + "decode_latency_s": 0.0610607604321558, + "kv_live_bytes": 462274560, + "latency_s": 1.9547017919830978, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72286208, + "prompt_kind": "short", + "t_relative_s": 2740.016138625331 + }, + { + "decode_latency_s": 0.06789723696419969, + "kv_live_bytes": 462274560, + "latency_s": 2.1748552499338984, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72286208, + "prompt_kind": "long", + "t_relative_s": 2745.0261842082255 + }, + { + "decode_latency_s": 0.06035337240609806, + "kv_live_bytes": 462274560, + "latency_s": 1.9322142084129155, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72187904, + "prompt_kind": "short", + "t_relative_s": 2750.0362640419044 + }, + { + "decode_latency_s": 0.06818289193324745, + "kv_live_bytes": 462274560, + "latency_s": 2.1836080830544233, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72302592, + "prompt_kind": "long", + "t_relative_s": 2755.04634379223 + }, + { + "decode_latency_s": 0.06646100521902554, + "kv_live_bytes": 462274560, + "latency_s": 2.128519124817103, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72892416, + "prompt_kind": "short", + "t_relative_s": 2760.0563969169743 + }, + { + "decode_latency_s": 0.06811908853705972, + "kv_live_bytes": 462274560, + "latency_s": 2.1832982092164457, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72974336, + "prompt_kind": "long", + "t_relative_s": 2765.066492666956 + }, + { + "decode_latency_s": 0.06072622397914529, + "kv_live_bytes": 462274560, + "latency_s": 1.9440167909488082, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72957952, + "prompt_kind": "short", + "t_relative_s": 2770.0765698752366 + }, + { + "decode_latency_s": 0.06795740366214886, + "kv_live_bytes": 462274560, + "latency_s": 2.1770694158039987, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66682880, + "prompt_kind": "long", + "t_relative_s": 2775.086604082957 + }, + { + "decode_latency_s": 0.0605945195275126, + "kv_live_bytes": 462274560, + "latency_s": 1.940013958606869, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68616192, + "prompt_kind": "short", + "t_relative_s": 2780.096700707916 + }, + { + "decode_latency_s": 0.0682522343704477, + "kv_live_bytes": 462274560, + "latency_s": 2.185770457610488, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72581120, + "prompt_kind": "long", + "t_relative_s": 2785.106771042105 + }, + { + "decode_latency_s": 0.06048222265962977, + "kv_live_bytes": 462274560, + "latency_s": 1.936314499936998, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73138176, + "prompt_kind": "short", + "t_relative_s": 2790.1168124172837 + }, + { + "decode_latency_s": 0.06795369272003882, + "kv_live_bytes": 462274560, + "latency_s": 2.1768485829234123, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73170944, + "prompt_kind": "long", + "t_relative_s": 2795.126892332919 + }, + { + "decode_latency_s": 0.06055398048192728, + "kv_live_bytes": 462274560, + "latency_s": 1.9384595407173038, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73203712, + "prompt_kind": "short", + "t_relative_s": 2800.1369676250033 + }, + { + "decode_latency_s": 0.06839508333359845, + "kv_live_bytes": 462274560, + "latency_s": 2.190771209076047, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73039872, + "prompt_kind": "long", + "t_relative_s": 2805.1470225830562 + }, + { + "decode_latency_s": 0.060440079410909675, + "kv_live_bytes": 462274560, + "latency_s": 1.9349780841730535, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73072640, + "prompt_kind": "short", + "t_relative_s": 2810.157079708297 + }, + { + "decode_latency_s": 0.06828288543329109, + "kv_live_bytes": 462274560, + "latency_s": 2.1868920838460326, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73252864, + "prompt_kind": "long", + "t_relative_s": 2815.167168417014 + }, + { + "decode_latency_s": 0.06055228515469935, + "kv_live_bytes": 462274560, + "latency_s": 1.938529749866575, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73793536, + "prompt_kind": "short", + "t_relative_s": 2820.177229083143 + }, + { + "decode_latency_s": 0.06841699349752162, + "kv_live_bytes": 462274560, + "latency_s": 2.1917193331755698, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73842688, + "prompt_kind": "long", + "t_relative_s": 2825.1873016669415 + }, + { + "decode_latency_s": 0.06054487889923621, + "kv_live_bytes": 462274560, + "latency_s": 1.9382549999281764, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73924608, + "prompt_kind": "short", + "t_relative_s": 2830.197385125328 + }, + { + "decode_latency_s": 0.06761664974328596, + "kv_live_bytes": 462274560, + "latency_s": 2.165926791727543, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73187328, + "prompt_kind": "long", + "t_relative_s": 2835.207440542057 + }, + { + "decode_latency_s": 0.06044286327960435, + "kv_live_bytes": 462274560, + "latency_s": 1.9350350419990718, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73220096, + "prompt_kind": "short", + "t_relative_s": 2840.217510000337 + }, + { + "decode_latency_s": 0.0603405481815571, + "kv_live_bytes": 462274560, + "latency_s": 1.9318105420097709, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73416704, + "prompt_kind": "long", + "t_relative_s": 2845.2275952920318 + }, + { + "decode_latency_s": 0.06044399739766959, + "kv_live_bytes": 462274560, + "latency_s": 1.9350703749805689, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73940992, + "prompt_kind": "short", + "t_relative_s": 2850.237648332957 + }, + { + "decode_latency_s": 0.06860232811595779, + "kv_live_bytes": 462274560, + "latency_s": 2.1973955002613366, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73990144, + "prompt_kind": "long", + "t_relative_s": 2855.247719042003 + }, + { + "decode_latency_s": 0.06658005858480465, + "kv_live_bytes": 462274560, + "latency_s": 2.132236875128001, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74022912, + "prompt_kind": "short", + "t_relative_s": 2860.257800917141 + }, + { + "decode_latency_s": 0.060444192684371956, + "kv_live_bytes": 462274560, + "latency_s": 1.9349620421417058, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73826304, + "prompt_kind": "long", + "t_relative_s": 2865.2678572922014 + }, + { + "decode_latency_s": 0.060374438820872456, + "kv_live_bytes": 462274560, + "latency_s": 1.932901583146304, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72613888, + "prompt_kind": "short", + "t_relative_s": 2870.277907792013 + }, + { + "decode_latency_s": 0.06838487631466705, + "kv_live_bytes": 462274560, + "latency_s": 2.1903539588674903, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73302016, + "prompt_kind": "long", + "t_relative_s": 2875.287987458054 + }, + { + "decode_latency_s": 0.06642931641545147, + "kv_live_bytes": 462274560, + "latency_s": 2.1273969169706106, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73875456, + "prompt_kind": "short", + "t_relative_s": 2880.2967965421267 + }, + { + "decode_latency_s": 0.06814110808772966, + "kv_live_bytes": 462274560, + "latency_s": 2.1834060410037637, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73924608, + "prompt_kind": "long", + "t_relative_s": 2885.306875000242 + }, + { + "decode_latency_s": 0.06061867318931036, + "kv_live_bytes": 462274560, + "latency_s": 1.9405330419540405, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73990144, + "prompt_kind": "short", + "t_relative_s": 2890.3169611669146 + }, + { + "decode_latency_s": 0.06767214584397152, + "kv_live_bytes": 462274560, + "latency_s": 2.167483499739319, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70025216, + "prompt_kind": "long", + "t_relative_s": 2895.3270136252977 + }, + { + "decode_latency_s": 0.060447350289905444, + "kv_live_bytes": 462274560, + "latency_s": 1.9352482915855944, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69074944, + "prompt_kind": "short", + "t_relative_s": 2900.3370729582384 + }, + { + "decode_latency_s": 0.0677968841191614, + "kv_live_bytes": 462274560, + "latency_s": 2.1716394582763314, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73203712, + "prompt_kind": "long", + "t_relative_s": 2905.3471519169398 + }, + { + "decode_latency_s": 0.06668044271646068, + "kv_live_bytes": 462274560, + "latency_s": 2.135425833053887, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73777152, + "prompt_kind": "short", + "t_relative_s": 2910.3572235419415 + }, + { + "decode_latency_s": 0.06830880595953204, + "kv_live_bytes": 462274560, + "latency_s": 2.188519374933094, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73793536, + "prompt_kind": "long", + "t_relative_s": 2915.367278708145 + }, + { + "decode_latency_s": 0.06050122265878599, + "kv_live_bytes": 462274560, + "latency_s": 1.9368119579739869, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73842688, + "prompt_kind": "short", + "t_relative_s": 2920.3773639579304 + }, + { + "decode_latency_s": 0.0680674544419162, + "kv_live_bytes": 462274560, + "latency_s": 2.1802205410785973, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73678848, + "prompt_kind": "long", + "t_relative_s": 2925.3874224172905 + }, + { + "decode_latency_s": 0.06033238803502172, + "kv_live_bytes": 462274560, + "latency_s": 1.9315590001642704, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73646080, + "prompt_kind": "short", + "t_relative_s": 2930.397475332953 + }, + { + "decode_latency_s": 0.06779477083182428, + "kv_live_bytes": 462274560, + "latency_s": 2.1715497910045087, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73809920, + "prompt_kind": "long", + "t_relative_s": 2935.4075500420295 + }, + { + "decode_latency_s": 0.0664679505280219, + "kv_live_bytes": 462274560, + "latency_s": 2.1287325830198824, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73859072, + "prompt_kind": "short", + "t_relative_s": 2940.417606708128 + }, + { + "decode_latency_s": 0.06811999477213249, + "kv_live_bytes": 462274560, + "latency_s": 2.182383374776691, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73875456, + "prompt_kind": "long", + "t_relative_s": 2945.427649416961 + }, + { + "decode_latency_s": 0.06048641925735865, + "kv_live_bytes": 462274560, + "latency_s": 1.9363510003313422, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73924608, + "prompt_kind": "short", + "t_relative_s": 2950.4377290420234 + }, + { + "decode_latency_s": 0.06799773174861912, + "kv_live_bytes": 462274560, + "latency_s": 2.1776782078668475, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72613888, + "prompt_kind": "long", + "t_relative_s": 2955.4477580832317 + }, + { + "decode_latency_s": 0.060441067689680494, + "kv_live_bytes": 462274560, + "latency_s": 1.9351172908209264, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72663040, + "prompt_kind": "short", + "t_relative_s": 2960.457792292349 + }, + { + "decode_latency_s": 0.06827686197357252, + "kv_live_bytes": 462274560, + "latency_s": 2.186856208834797, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73416704, + "prompt_kind": "long", + "t_relative_s": 2965.467882832978 + }, + { + "decode_latency_s": 0.06664539975463413, + "kv_live_bytes": 462274560, + "latency_s": 2.1343881669454277, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73924608, + "prompt_kind": "short", + "t_relative_s": 2970.4779487499036 + }, + { + "decode_latency_s": 0.06816869793692604, + "kv_live_bytes": 462274560, + "latency_s": 2.1837023752741516, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73990144, + "prompt_kind": "long", + "t_relative_s": 2975.4880230831914 + }, + { + "decode_latency_s": 0.06067421221814584, + "kv_live_bytes": 462274560, + "latency_s": 1.942350750323385, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74039296, + "prompt_kind": "short", + "t_relative_s": 2980.4981082081795 + }, + { + "decode_latency_s": 0.06900908071838785, + "kv_live_bytes": 462274560, + "latency_s": 2.2091679158620536, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70811648, + "prompt_kind": "long", + "t_relative_s": 2985.5004010000266 + }, + { + "decode_latency_s": 0.06026842709979974, + "kv_live_bytes": 462274560, + "latency_s": 1.9295442500151694, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70860800, + "prompt_kind": "short", + "t_relative_s": 2992.0980941252783 + }, + { + "decode_latency_s": 0.06761970443767495, + "kv_live_bytes": 462274560, + "latency_s": 2.1657490422949195, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70926336, + "prompt_kind": "long", + "t_relative_s": 2997.1081738332286 + }, + { + "decode_latency_s": 0.06053645831707399, + "kv_live_bytes": 462274560, + "latency_s": 1.9379031667485833, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70942720, + "prompt_kind": "short", + "t_relative_s": 3002.118259458337 + }, + { + "decode_latency_s": 0.06797898178047035, + "kv_live_bytes": 462274560, + "latency_s": 2.177660208195448, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70991872, + "prompt_kind": "long", + "t_relative_s": 3007.128315458074 + }, + { + "decode_latency_s": 0.060302197918645106, + "kv_live_bytes": 462274560, + "latency_s": 1.9305458329617977, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71024640, + "prompt_kind": "short", + "t_relative_s": 3012.138391250279 + }, + { + "decode_latency_s": 0.06850798046798445, + "kv_live_bytes": 462274560, + "latency_s": 2.194347749929875, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69451776, + "prompt_kind": "long", + "t_relative_s": 3017.1484757079743 + }, + { + "decode_latency_s": 0.06034547265153378, + "kv_live_bytes": 462274560, + "latency_s": 1.9319065832532942, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69615616, + "prompt_kind": "short", + "t_relative_s": 3022.158503250219 + }, + { + "decode_latency_s": 0.06827813280688133, + "kv_live_bytes": 462274560, + "latency_s": 2.1869594999589026, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70238208, + "prompt_kind": "long", + "t_relative_s": 3027.1685834582895 + }, + { + "decode_latency_s": 0.0606798033695668, + "kv_live_bytes": 462274560, + "latency_s": 1.94252908276394, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70647808, + "prompt_kind": "short", + "t_relative_s": 3032.178709917236 + }, + { + "decode_latency_s": 0.06788707162195351, + "kv_live_bytes": 462274560, + "latency_s": 2.1749161249026656, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70795264, + "prompt_kind": "long", + "t_relative_s": 3037.1887586670928 + }, + { + "decode_latency_s": 0.060530983057105914, + "kv_live_bytes": 462274560, + "latency_s": 1.93782962532714, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70647808, + "prompt_kind": "short", + "t_relative_s": 3042.1988306250423 + }, + { + "decode_latency_s": 0.0673741588689154, + "kv_live_bytes": 462274560, + "latency_s": 2.157914541196078, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70680576, + "prompt_kind": "long", + "t_relative_s": 3047.208904375322 + }, + { + "decode_latency_s": 0.0708355859387666, + "kv_live_bytes": 462274560, + "latency_s": 2.273103083949536, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 50036736, + "prompt_kind": "short", + "t_relative_s": 3052.2189392922446 + }, + { + "decode_latency_s": 0.0696925716183614, + "kv_live_bytes": 462274560, + "latency_s": 2.2336307079531252, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70352896, + "prompt_kind": "long", + "t_relative_s": 3057.2290083752014 + }, + { + "decode_latency_s": 0.06062698567984626, + "kv_live_bytes": 462274560, + "latency_s": 1.940862957853824, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70664192, + "prompt_kind": "short", + "t_relative_s": 3062.2390959169716 + }, + { + "decode_latency_s": 0.06814589061832521, + "kv_live_bytes": 462274560, + "latency_s": 2.1832705000415444, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70729728, + "prompt_kind": "long", + "t_relative_s": 3067.2491645421833 + }, + { + "decode_latency_s": 0.060357361973728985, + "kv_live_bytes": 462274560, + "latency_s": 1.9323735409416258, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69255168, + "prompt_kind": "short", + "t_relative_s": 3072.2592371250503 + }, + { + "decode_latency_s": 0.060567260428797454, + "kv_live_bytes": 462274560, + "latency_s": 1.9391226670704782, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69926912, + "prompt_kind": "long", + "t_relative_s": 3077.269312541932 + }, + { + "decode_latency_s": 0.06037736982398201, + "kv_live_bytes": 462274560, + "latency_s": 1.9331481247209013, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70664192, + "prompt_kind": "short", + "t_relative_s": 3082.279367875308 + }, + { + "decode_latency_s": 0.06770763412350789, + "kv_live_bytes": 462274560, + "latency_s": 2.169224707875401, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70877184, + "prompt_kind": "long", + "t_relative_s": 3087.289434667211 + }, + { + "decode_latency_s": 0.060451239594840445, + "kv_live_bytes": 462274560, + "latency_s": 1.9351246668957174, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71434240, + "prompt_kind": "short", + "t_relative_s": 3092.2995168329217 + }, + { + "decode_latency_s": 0.0609192877454916, + "kv_live_bytes": 462274560, + "latency_s": 1.950750334188342, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71565312, + "prompt_kind": "long", + "t_relative_s": 3097.303608875256 + }, + { + "decode_latency_s": 0.060582282560062595, + "kv_live_bytes": 462274560, + "latency_s": 1.9395204591564834, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71532544, + "prompt_kind": "short", + "t_relative_s": 3102.3136829999276 + }, + { + "decode_latency_s": 0.06827467060065828, + "kv_live_bytes": 462274560, + "latency_s": 2.1868406673893332, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71024640, + "prompt_kind": "long", + "t_relative_s": 3107.323753166944 + }, + { + "decode_latency_s": 0.06628883462690283, + "kv_live_bytes": 462274560, + "latency_s": 2.122891040984541, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71761920, + "prompt_kind": "short", + "t_relative_s": 3112.3338038329966 + }, + { + "decode_latency_s": 0.06812807684764266, + "kv_live_bytes": 462274560, + "latency_s": 2.182680499739945, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71434240, + "prompt_kind": "long", + "t_relative_s": 3117.3439042079262 + }, + { + "decode_latency_s": 0.06050203647464514, + "kv_live_bytes": 462274560, + "latency_s": 1.936884207651019, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72155136, + "prompt_kind": "short", + "t_relative_s": 3122.3540093330666 + }, + { + "decode_latency_s": 0.06791881250683218, + "kv_live_bytes": 462274560, + "latency_s": 2.175685332622379, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72351744, + "prompt_kind": "long", + "t_relative_s": 3127.364048167132 + }, + { + "decode_latency_s": 0.06030649090826046, + "kv_live_bytes": 462274560, + "latency_s": 1.930684084072709, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71876608, + "prompt_kind": "short", + "t_relative_s": 3132.373785292264 + }, + { + "decode_latency_s": 0.06839869140821975, + "kv_live_bytes": 462274560, + "latency_s": 2.1906459159217775, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71106560, + "prompt_kind": "long", + "t_relative_s": 3137.383871458005 + }, + { + "decode_latency_s": 0.0604186158743687, + "kv_live_bytes": 462274560, + "latency_s": 1.934256125241518, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71172096, + "prompt_kind": "short", + "t_relative_s": 3142.393915625289 + }, + { + "decode_latency_s": 0.06765714843641035, + "kv_live_bytes": 462274560, + "latency_s": 2.1666911663487554, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71614464, + "prompt_kind": "long", + "t_relative_s": 3147.403985292185 + }, + { + "decode_latency_s": 0.06053536328545306, + "kv_live_bytes": 462274560, + "latency_s": 1.937912874855101, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72204288, + "prompt_kind": "short", + "t_relative_s": 3152.4140729173087 + }, + { + "decode_latency_s": 0.06801925259060226, + "kv_live_bytes": 462274560, + "latency_s": 2.17918345797807, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72253440, + "prompt_kind": "long", + "t_relative_s": 3157.4241189169697 + }, + { + "decode_latency_s": 0.06077515885408502, + "kv_live_bytes": 462274560, + "latency_s": 1.9455761667340994, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71974912, + "prompt_kind": "short", + "t_relative_s": 3162.43419237528 + }, + { + "decode_latency_s": 0.06863668491132557, + "kv_live_bytes": 462274560, + "latency_s": 2.1983133340254426, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71532544, + "prompt_kind": "long", + "t_relative_s": 3167.4442793750204 + }, + { + "decode_latency_s": 0.060258053403231315, + "kv_live_bytes": 462274560, + "latency_s": 1.9291384578682482, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72024064, + "prompt_kind": "short", + "t_relative_s": 3172.454339625314 + }, + { + "decode_latency_s": 0.06772228906629607, + "kv_live_bytes": 462274560, + "latency_s": 2.1690537496469915, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72138752, + "prompt_kind": "long", + "t_relative_s": 3177.464433291927 + }, + { + "decode_latency_s": 0.06061379816674162, + "kv_live_bytes": 462274560, + "latency_s": 1.9404585831798613, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72679424, + "prompt_kind": "short", + "t_relative_s": 3182.4745464171283 + }, + { + "decode_latency_s": 0.06768843490863219, + "kv_live_bytes": 462274560, + "latency_s": 2.168389833997935, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72777728, + "prompt_kind": "long", + "t_relative_s": 3187.484604583122 + }, + { + "decode_latency_s": 0.06038571354292799, + "kv_live_bytes": 462274560, + "latency_s": 1.9331845422275364, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72876032, + "prompt_kind": "short", + "t_relative_s": 3192.494683458004 + }, + { + "decode_latency_s": 0.0682988763001049, + "kv_live_bytes": 462274560, + "latency_s": 2.1876652501523495, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69877760, + "prompt_kind": "long", + "t_relative_s": 3197.5047596669756 + }, + { + "decode_latency_s": 0.06054678387590684, + "kv_live_bytes": 462274560, + "latency_s": 1.9384226673282683, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71958528, + "prompt_kind": "short", + "t_relative_s": 3202.514811167028 + }, + { + "decode_latency_s": 0.06796525912068319, + "kv_live_bytes": 462274560, + "latency_s": 2.177317707799375, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72089600, + "prompt_kind": "long", + "t_relative_s": 3207.524901667144 + }, + { + "decode_latency_s": 0.0668861301819561, + "kv_live_bytes": 462274560, + "latency_s": 2.1421050000935793, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72613888, + "prompt_kind": "short", + "t_relative_s": 3212.5349874999374 + }, + { + "decode_latency_s": 0.060912443994311616, + "kv_live_bytes": 462274560, + "latency_s": 1.950556042138487, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72663040, + "prompt_kind": "long", + "t_relative_s": 3217.5450401250273 + }, + { + "decode_latency_s": 0.060603165373322554, + "kv_live_bytes": 462274560, + "latency_s": 1.940154124982655, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72433664, + "prompt_kind": "short", + "t_relative_s": 3222.555128583219 + }, + { + "decode_latency_s": 0.06794035024358891, + "kv_live_bytes": 462274560, + "latency_s": 2.1761380839161575, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71909376, + "prompt_kind": "long", + "t_relative_s": 3227.565205417108 + }, + { + "decode_latency_s": 0.06033135674078949, + "kv_live_bytes": 462274560, + "latency_s": 1.93148074997589, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72581120, + "prompt_kind": "short", + "t_relative_s": 3232.575274417177 + }, + { + "decode_latency_s": 0.06764338022912852, + "kv_live_bytes": 462274560, + "latency_s": 2.167024917434901, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "long", + "t_relative_s": 3237.5853598332033 + }, + { + "decode_latency_s": 0.060581700527109206, + "kv_live_bytes": 462274560, + "latency_s": 1.939382459037006, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73236480, + "prompt_kind": "short", + "t_relative_s": 3242.595449458342 + }, + { + "decode_latency_s": 0.06751182030711789, + "kv_live_bytes": 462274560, + "latency_s": 2.1629204587079585, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70729728, + "prompt_kind": "long", + "t_relative_s": 3247.605502374936 + }, + { + "decode_latency_s": 0.06068085419246927, + "kv_live_bytes": 462274560, + "latency_s": 1.9427691660821438, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70942720, + "prompt_kind": "short", + "t_relative_s": 3252.615550958086 + }, + { + "decode_latency_s": 0.17723929688509088, + "kv_live_bytes": 462274560, + "latency_s": 5.672626083251089, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66027520, + "prompt_kind": "long", + "t_relative_s": 3257.6256534582935 + }, + { + "decode_latency_s": 0.06047476953244768, + "kv_live_bytes": 462274560, + "latency_s": 1.9360612910240889, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66715648, + "prompt_kind": "short", + "t_relative_s": 3266.359179667197 + }, + { + "decode_latency_s": 0.06798669531417545, + "kv_live_bytes": 462274560, + "latency_s": 2.1779510830529034, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67764224, + "prompt_kind": "long", + "t_relative_s": 3271.3692274582572 + }, + { + "decode_latency_s": 0.06065109897463117, + "kv_live_bytes": 462274560, + "latency_s": 1.9416742497123778, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68354048, + "prompt_kind": "short", + "t_relative_s": 3276.379298250191 + }, + { + "decode_latency_s": 0.06763904297258705, + "kv_live_bytes": 462274560, + "latency_s": 2.1660304167307913, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66977792, + "prompt_kind": "long", + "t_relative_s": 3281.3894331669435 + }, + { + "decode_latency_s": 0.06535520311444998, + "kv_live_bytes": 462274560, + "latency_s": 2.0929697919636965, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66994176, + "prompt_kind": "short", + "t_relative_s": 3286.3995043751784 + }, + { + "decode_latency_s": 0.06761109894432593, + "kv_live_bytes": 462274560, + "latency_s": 2.1659241248853505, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67731456, + "prompt_kind": "long", + "t_relative_s": 3291.4095551669598 + }, + { + "decode_latency_s": 0.06041770185402129, + "kv_live_bytes": 462274560, + "latency_s": 1.9342665411531925, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68288512, + "prompt_kind": "short", + "t_relative_s": 3296.4196276250295 + }, + { + "decode_latency_s": 0.06096824737323914, + "kv_live_bytes": 462274560, + "latency_s": 1.9522171667777002, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68354048, + "prompt_kind": "long", + "t_relative_s": 3301.429701332934 + }, + { + "decode_latency_s": 0.06706080469302833, + "kv_live_bytes": 462274560, + "latency_s": 2.1478887908160686, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68943872, + "prompt_kind": "short", + "t_relative_s": 3306.436193624977 + }, + { + "decode_latency_s": 0.06820044141204562, + "kv_live_bytes": 462274560, + "latency_s": 2.1850739582441747, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69025792, + "prompt_kind": "long", + "t_relative_s": 3311.44627375016 + }, + { + "decode_latency_s": 0.06977463541261386, + "kv_live_bytes": 462274560, + "latency_s": 2.2345474995672703, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 64897024, + "prompt_kind": "short", + "t_relative_s": 3316.456369082909 + }, + { + "decode_latency_s": 0.06834447657456622, + "kv_live_bytes": 462274560, + "latency_s": 2.1901638749986887, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66306048, + "prompt_kind": "long", + "t_relative_s": 3321.466474249959 + }, + { + "decode_latency_s": 0.06075278256321326, + "kv_live_bytes": 462274560, + "latency_s": 1.9448617077432573, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68780032, + "prompt_kind": "short", + "t_relative_s": 3326.4765462921932 + }, + { + "decode_latency_s": 0.06781717446574476, + "kv_live_bytes": 462274560, + "latency_s": 2.172998458147049, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68780032, + "prompt_kind": "long", + "t_relative_s": 3331.486119000241 + }, + { + "decode_latency_s": 0.06092502735555172, + "kv_live_bytes": 462274560, + "latency_s": 1.9503504168242216, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69320704, + "prompt_kind": "short", + "t_relative_s": 3336.4962070421316 + }, + { + "decode_latency_s": 0.06842740492720623, + "kv_live_bytes": 462274560, + "latency_s": 2.191692791879177, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68239360, + "prompt_kind": "long", + "t_relative_s": 3341.5062622502446 + }, + { + "decode_latency_s": 0.06072361327824183, + "kv_live_bytes": 462274560, + "latency_s": 1.9439384168945253, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68239360, + "prompt_kind": "short", + "t_relative_s": 3346.5163347921334 + }, + { + "decode_latency_s": 0.06668194531812333, + "kv_live_bytes": 462274560, + "latency_s": 2.1359765003435314, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68763648, + "prompt_kind": "long", + "t_relative_s": 3351.526381792035 + }, + { + "decode_latency_s": 0.0667291926802136, + "kv_live_bytes": 462274560, + "latency_s": 2.1364208748564124, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67207168, + "prompt_kind": "short", + "t_relative_s": 3356.5364681249484 + }, + { + "decode_latency_s": 0.06804698046471458, + "kv_live_bytes": 462274560, + "latency_s": 2.179820166900754, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68681728, + "prompt_kind": "long", + "t_relative_s": 3361.5465412922204 + }, + { + "decode_latency_s": 0.06111341147334315, + "kv_live_bytes": 462274560, + "latency_s": 1.9564124578610063, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69124096, + "prompt_kind": "short", + "t_relative_s": 3366.5566140003502 + }, + { + "decode_latency_s": 0.06982604946824722, + "kv_live_bytes": 462274560, + "latency_s": 2.2353633330203593, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 65421312, + "prompt_kind": "long", + "t_relative_s": 3371.5666654580273 + }, + { + "decode_latency_s": 0.06804448696493637, + "kv_live_bytes": 462274560, + "latency_s": 2.178731416352093, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67567616, + "prompt_kind": "short", + "t_relative_s": 3376.5767061673105 + }, + { + "decode_latency_s": 0.06843766536621843, + "kv_live_bytes": 462274560, + "latency_s": 2.1925551663152874, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68534272, + "prompt_kind": "long", + "t_relative_s": 3381.5867907083593 + }, + { + "decode_latency_s": 0.06073246484447736, + "kv_live_bytes": 462274560, + "latency_s": 1.9442622913047671, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69124096, + "prompt_kind": "short", + "t_relative_s": 3386.596863999963 + }, + { + "decode_latency_s": 0.06791000519297086, + "kv_live_bytes": 462274560, + "latency_s": 2.1757732089608908, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69173248, + "prompt_kind": "long", + "t_relative_s": 3391.606916333083 + }, + { + "decode_latency_s": 0.06071188149508089, + "kv_live_bytes": 462274560, + "latency_s": 1.9435547916218638, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69697536, + "prompt_kind": "short", + "t_relative_s": 3396.6169941672124 + }, + { + "decode_latency_s": 0.06800223045866005, + "kv_live_bytes": 462274560, + "latency_s": 2.1778652500361204, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68583424, + "prompt_kind": "long", + "t_relative_s": 3401.6270448751748 + }, + { + "decode_latency_s": 0.06035160677856766, + "kv_live_bytes": 462274560, + "latency_s": 1.9322249577380717, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68468736, + "prompt_kind": "short", + "t_relative_s": 3406.6371070002206 + }, + { + "decode_latency_s": 0.06773081119172275, + "kv_live_bytes": 462274560, + "latency_s": 2.169413583818823, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69189632, + "prompt_kind": "long", + "t_relative_s": 3411.6471885000356 + }, + { + "decode_latency_s": 0.060540260397829115, + "kv_live_bytes": 462274560, + "latency_s": 1.9380573751404881, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69779456, + "prompt_kind": "short", + "t_relative_s": 3416.6572207920253 + }, + { + "decode_latency_s": 0.06817205468541943, + "kv_live_bytes": 462274560, + "latency_s": 2.183778249658644, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69828608, + "prompt_kind": "long", + "t_relative_s": 3421.6672722920775 + }, + { + "decode_latency_s": 0.060787756490753964, + "kv_live_bytes": 462274560, + "latency_s": 1.9459887919947505, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70254592, + "prompt_kind": "short", + "t_relative_s": 3426.6773742502555 + }, + { + "decode_latency_s": 0.06865222656051628, + "kv_live_bytes": 462274560, + "latency_s": 2.198921542149037, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69877760, + "prompt_kind": "long", + "t_relative_s": 3431.687432500068 + }, + { + "decode_latency_s": 0.060864614555612206, + "kv_live_bytes": 462274560, + "latency_s": 1.948520042002201, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67584000, + "prompt_kind": "short", + "t_relative_s": 3436.697488625068 + }, + { + "decode_latency_s": 0.06802398699801415, + "kv_live_bytes": 462274560, + "latency_s": 2.1792366253212094, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69763072, + "prompt_kind": "long", + "t_relative_s": 3441.707549124956 + }, + { + "decode_latency_s": 0.06024608726147562, + "kv_live_bytes": 462274560, + "latency_s": 1.9288158332929015, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70336512, + "prompt_kind": "short", + "t_relative_s": 3446.717651333194 + }, + { + "decode_latency_s": 0.060280285149929114, + "kv_live_bytes": 462274560, + "latency_s": 1.9302429580129683, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70172672, + "prompt_kind": "long", + "t_relative_s": 3451.7277281251736 + }, + { + "decode_latency_s": 0.060490070303785615, + "kv_live_bytes": 462274560, + "latency_s": 1.9365737908519804, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70811648, + "prompt_kind": "short", + "t_relative_s": 3456.737778916955 + }, + { + "decode_latency_s": 0.06816206118674017, + "kv_live_bytes": 462274560, + "latency_s": 2.1834916658699512, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70909952, + "prompt_kind": "long", + "t_relative_s": 3461.7478682082146 + }, + { + "decode_latency_s": 0.06069432421645615, + "kv_live_bytes": 462274560, + "latency_s": 1.9430585000663996, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70909952, + "prompt_kind": "short", + "t_relative_s": 3466.757965875324 + }, + { + "decode_latency_s": 0.06798688018170651, + "kv_live_bytes": 462274560, + "latency_s": 2.17803491698578, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71139328, + "prompt_kind": "long", + "t_relative_s": 3471.768020375166 + }, + { + "decode_latency_s": 0.06045774478116073, + "kv_live_bytes": 462274560, + "latency_s": 1.9354783752933145, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71761920, + "prompt_kind": "short", + "t_relative_s": 3476.7780952919275 + }, + { + "decode_latency_s": 0.06078859505942091, + "kv_live_bytes": 462274560, + "latency_s": 1.9464072501286864, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71057408, + "prompt_kind": "long", + "t_relative_s": 3481.7881761672907 + }, + { + "decode_latency_s": 0.060821378909167834, + "kv_live_bytes": 462274560, + "latency_s": 1.947163457982242, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71680000, + "prompt_kind": "short", + "t_relative_s": 3486.7982310829684 + }, + { + "decode_latency_s": 0.06799735806998797, + "kv_live_bytes": 462274560, + "latency_s": 2.178432165645063, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71761920, + "prompt_kind": "long", + "t_relative_s": 3491.808298625052 + }, + { + "decode_latency_s": 0.06259197393956129, + "kv_live_bytes": 462274560, + "latency_s": 2.003856707829982, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70385664, + "prompt_kind": "short", + "t_relative_s": 3496.8183632083237 + }, + { + "decode_latency_s": 0.06842638412490487, + "kv_live_bytes": 462274560, + "latency_s": 2.1913169999606907, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71221248, + "prompt_kind": "long", + "t_relative_s": 3501.8284314172342 + }, + { + "decode_latency_s": 0.0605008046986768, + "kv_live_bytes": 462274560, + "latency_s": 1.9368097083643079, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71778304, + "prompt_kind": "short", + "t_relative_s": 3506.8384897499345 + }, + { + "decode_latency_s": 0.06795277862693183, + "kv_live_bytes": 462274560, + "latency_s": 2.176926165819168, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71827456, + "prompt_kind": "long", + "t_relative_s": 3511.84857004229 + }, + { + "decode_latency_s": 0.06072245571704116, + "kv_live_bytes": 462274560, + "latency_s": 1.9439425421878695, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72384512, + "prompt_kind": "short", + "t_relative_s": 3516.854624541942 + }, + { + "decode_latency_s": 0.0680469804792665, + "kv_live_bytes": 462274560, + "latency_s": 2.179417208302766, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72237056, + "prompt_kind": "long", + "t_relative_s": 3521.8646810832433 + }, + { + "decode_latency_s": 0.0605891731538577, + "kv_live_bytes": 462274560, + "latency_s": 1.9397849580273032, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72204288, + "prompt_kind": "short", + "t_relative_s": 3526.8747533331625 + }, + { + "decode_latency_s": 0.06811518878384959, + "kv_live_bytes": 462274560, + "latency_s": 2.181374041829258, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72384512, + "prompt_kind": "long", + "t_relative_s": 3531.884851917159 + }, + { + "decode_latency_s": 0.06046997266821563, + "kv_live_bytes": 462274560, + "latency_s": 1.935849417001009, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72908800, + "prompt_kind": "short", + "t_relative_s": 3536.894922333304 + }, + { + "decode_latency_s": 0.060424683601013385, + "kv_live_bytes": 462274560, + "latency_s": 1.9346056669019163, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72957952, + "prompt_kind": "long", + "t_relative_s": 3541.9030587500893 + }, + { + "decode_latency_s": 0.06076185806887224, + "kv_live_bytes": 462274560, + "latency_s": 1.9451512922532856, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72974336, + "prompt_kind": "short", + "t_relative_s": 3546.9131443751976 + }, + { + "decode_latency_s": 0.060644763012533076, + "kv_live_bytes": 462274560, + "latency_s": 1.9419668749906123, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72990720, + "prompt_kind": "long", + "t_relative_s": 3551.9232037919573 + }, + { + "decode_latency_s": 0.06804445052694064, + "kv_live_bytes": 462274560, + "latency_s": 2.1790770827792585, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69894144, + "prompt_kind": "short", + "t_relative_s": 3556.93327533314 + }, + { + "decode_latency_s": 0.06806966145813931, + "kv_live_bytes": 462274560, + "latency_s": 2.1809090841561556, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70680576, + "prompt_kind": "long", + "t_relative_s": 3561.9433302921243 + }, + { + "decode_latency_s": 0.06030609374283813, + "kv_live_bytes": 462274560, + "latency_s": 1.9306367500685155, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72941568, + "prompt_kind": "short", + "t_relative_s": 3566.9534219172783 + }, + { + "decode_latency_s": 0.06827787762449589, + "kv_live_bytes": 462274560, + "latency_s": 2.1865613753907382, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72712192, + "prompt_kind": "long", + "t_relative_s": 3571.96352066705 + }, + { + "decode_latency_s": 0.06043506249261554, + "kv_live_bytes": 462274560, + "latency_s": 1.9348064167425036, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73547776, + "prompt_kind": "short", + "t_relative_s": 3576.9735732502304 + }, + { + "decode_latency_s": 0.06739484115678351, + "kv_live_bytes": 462274560, + "latency_s": 2.1588588752783835, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73728000, + "prompt_kind": "long", + "t_relative_s": 3581.9836655831896 + }, + { + "decode_latency_s": 0.06203888019081205, + "kv_live_bytes": 462274560, + "latency_s": 1.986158833373338, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72073216, + "prompt_kind": "short", + "t_relative_s": 3586.993757833261 + }, + { + "decode_latency_s": 0.06794765756058041, + "kv_live_bytes": 462274560, + "latency_s": 2.176030958071351, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73187328, + "prompt_kind": "long", + "t_relative_s": 3592.0038375421427 + }, + { + "decode_latency_s": 0.060526167959324084, + "kv_live_bytes": 462274560, + "latency_s": 1.9376824996434152, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73285632, + "prompt_kind": "short", + "t_relative_s": 3597.013902292121 + }, + { + "decode_latency_s": 0.06792727214633487, + "kv_live_bytes": 462274560, + "latency_s": 2.175884374883026, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73367552, + "prompt_kind": "long", + "t_relative_s": 3602.023975333199 + }, + { + "decode_latency_s": 0.060962309915339574, + "kv_live_bytes": 462274560, + "latency_s": 1.9515700000338256, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73924608, + "prompt_kind": "short", + "t_relative_s": 3607.0340533331037 + }, + { + "decode_latency_s": 0.06817057423177175, + "kv_live_bytes": 462274560, + "latency_s": 2.183360833209008, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73498624, + "prompt_kind": "long", + "t_relative_s": 3612.044105750043 + }, + { + "decode_latency_s": 0.06738488932023756, + "kv_live_bytes": 462274560, + "latency_s": 2.1587488329969347, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69058560, + "prompt_kind": "short", + "t_relative_s": 3617.0541633330286 + }, + { + "decode_latency_s": 0.0615160794404801, + "kv_live_bytes": 462274560, + "latency_s": 1.9695493751205504, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68845568, + "prompt_kind": "long", + "t_relative_s": 3622.06423874991 + }, + { + "decode_latency_s": 0.06044585678318981, + "kv_live_bytes": 462274560, + "latency_s": 1.935148708987981, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69271552, + "prompt_kind": "short", + "t_relative_s": 3627.074303708039 + }, + { + "decode_latency_s": 0.0682875859492924, + "kv_live_bytes": 462274560, + "latency_s": 2.1868207091465592, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71827456, + "prompt_kind": "long", + "t_relative_s": 3632.084367958363 + }, + { + "decode_latency_s": 0.06073119920620229, + "kv_live_bytes": 462274560, + "latency_s": 1.9442323329858482, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72384512, + "prompt_kind": "short", + "t_relative_s": 3637.094442333095 + }, + { + "decode_latency_s": 0.06796595444029663, + "kv_live_bytes": 462274560, + "latency_s": 2.1773860421963036, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72433664, + "prompt_kind": "long", + "t_relative_s": 3642.1045163753442 + }, + { + "decode_latency_s": 0.06223586459236685, + "kv_live_bytes": 462274560, + "latency_s": 1.9925566669553518, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70828032, + "prompt_kind": "short", + "t_relative_s": 3647.114585333038 + }, + { + "decode_latency_s": 0.06675421092950273, + "kv_live_bytes": 462274560, + "latency_s": 2.138224707916379, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72089600, + "prompt_kind": "long", + "t_relative_s": 3652.12465825025 + }, + { + "decode_latency_s": 0.060570091125555336, + "kv_live_bytes": 462274560, + "latency_s": 1.9390572500415146, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "short", + "t_relative_s": 3657.1347363330424 + }, + { + "decode_latency_s": 0.06848615493800025, + "kv_live_bytes": 462274560, + "latency_s": 2.194362792186439, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "long", + "t_relative_s": 3662.1447880421765 + }, + { + "decode_latency_s": 0.06063169660046697, + "kv_live_bytes": 462274560, + "latency_s": 1.9409891669638455, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73203712, + "prompt_kind": "short", + "t_relative_s": 3667.154867417179 + }, + { + "decode_latency_s": 0.06821241146826651, + "kv_live_bytes": 462274560, + "latency_s": 2.184967666864395, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72073216, + "prompt_kind": "long", + "t_relative_s": 3672.1649191672914 + }, + { + "decode_latency_s": 0.06140590492577758, + "kv_live_bytes": 462274560, + "latency_s": 1.965851916000247, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71122944, + "prompt_kind": "short", + "t_relative_s": 3677.174998542294 + }, + { + "decode_latency_s": 0.06789069659134839, + "kv_live_bytes": 462274560, + "latency_s": 2.174946791958064, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72105984, + "prompt_kind": "long", + "t_relative_s": 3682.185054417234 + }, + { + "decode_latency_s": 0.06047713803127408, + "kv_live_bytes": 462274560, + "latency_s": 1.9361756248399615, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72187904, + "prompt_kind": "short", + "t_relative_s": 3687.195142417215 + }, + { + "decode_latency_s": 0.0679823372483952, + "kv_live_bytes": 462274560, + "latency_s": 2.1770186657086015, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72515584, + "prompt_kind": "long", + "t_relative_s": 3692.205224458128 + }, + { + "decode_latency_s": 0.060612921879510395, + "kv_live_bytes": 462274560, + "latency_s": 1.9405318750068545, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73105408, + "prompt_kind": "short", + "t_relative_s": 3697.215269875247 + }, + { + "decode_latency_s": 0.06972469273023307, + "kv_live_bytes": 462274560, + "latency_s": 2.2333737080916762, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69730304, + "prompt_kind": "long", + "t_relative_s": 3702.2253502500243 + }, + { + "decode_latency_s": 0.060413071638322435, + "kv_live_bytes": 462274560, + "latency_s": 1.9340905416756868, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71794688, + "prompt_kind": "short", + "t_relative_s": 3707.2354282923043 + }, + { + "decode_latency_s": 0.06827319922740571, + "kv_live_bytes": 462274560, + "latency_s": 2.186426708009094, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72482816, + "prompt_kind": "long", + "t_relative_s": 3712.245495249983 + }, + { + "decode_latency_s": 0.06066615888266824, + "kv_live_bytes": 462274560, + "latency_s": 1.9421397079713643, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72712192, + "prompt_kind": "short", + "t_relative_s": 3717.2555470000952 + }, + { + "decode_latency_s": 0.06806967579177581, + "kv_live_bytes": 462274560, + "latency_s": 2.180955708026886, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72794112, + "prompt_kind": "long", + "t_relative_s": 3722.2656165831722 + }, + { + "decode_latency_s": 0.060597006493480876, + "kv_live_bytes": 462274560, + "latency_s": 1.9398338748142123, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73318400, + "prompt_kind": "short", + "t_relative_s": 3727.269062791951 + }, + { + "decode_latency_s": 0.06847890494100284, + "kv_live_bytes": 462274560, + "latency_s": 2.1932409163564444, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73416704, + "prompt_kind": "long", + "t_relative_s": 3732.2791095003486 + }, + { + "decode_latency_s": 0.060716795560438186, + "kv_live_bytes": 462274560, + "latency_s": 1.9439452500082552, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67911680, + "prompt_kind": "short", + "t_relative_s": 3737.2891844999976 + }, + { + "decode_latency_s": 0.06806530729227234, + "kv_live_bytes": 462274560, + "latency_s": 2.179955916944891, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72597504, + "prompt_kind": "long", + "t_relative_s": 3742.299250208307 + }, + { + "decode_latency_s": 0.06645655207103118, + "kv_live_bytes": 462274560, + "latency_s": 2.128273040987551, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72695808, + "prompt_kind": "short", + "t_relative_s": 3747.3092967919074 + }, + { + "decode_latency_s": 0.06799022915947717, + "kv_live_bytes": 462274560, + "latency_s": 2.1782250832766294, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72613888, + "prompt_kind": "long", + "t_relative_s": 3752.3193796249107 + }, + { + "decode_latency_s": 0.06070382290636189, + "kv_live_bytes": 462274560, + "latency_s": 1.9433303331024945, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73220096, + "prompt_kind": "short", + "t_relative_s": 3757.329448542092 + }, + { + "decode_latency_s": 0.06796785937331151, + "kv_live_bytes": 462274560, + "latency_s": 2.1770270843990147, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72794112, + "prompt_kind": "long", + "t_relative_s": 3762.339499874972 + }, + { + "decode_latency_s": 0.06033669011958409, + "kv_live_bytes": 462274560, + "latency_s": 1.9317061668261886, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72335360, + "prompt_kind": "short", + "t_relative_s": 3767.349570624996 + }, + { + "decode_latency_s": 0.06809606509341393, + "kv_live_bytes": 462274560, + "latency_s": 2.181189457885921, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72826880, + "prompt_kind": "long", + "t_relative_s": 3772.359646332916 + }, + { + "decode_latency_s": 0.06679064320633188, + "kv_live_bytes": 462274560, + "latency_s": 2.139764499850571, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72925184, + "prompt_kind": "short", + "t_relative_s": 3777.3696945421398 + }, + { + "decode_latency_s": 0.06821716015110724, + "kv_live_bytes": 462274560, + "latency_s": 2.185600040946156, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72974336, + "prompt_kind": "long", + "t_relative_s": 3782.379800416995 + }, + { + "decode_latency_s": 0.0667516497487668, + "kv_live_bytes": 462274560, + "latency_s": 2.137841542251408, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73433088, + "prompt_kind": "short", + "t_relative_s": 3787.38987383293 + }, + { + "decode_latency_s": 0.06070575912599452, + "kv_live_bytes": 462274560, + "latency_s": 1.9439373752102256, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73515008, + "prompt_kind": "long", + "t_relative_s": 3792.3999226251617 + }, + { + "decode_latency_s": 0.06248221483838279, + "kv_live_bytes": 462274560, + "latency_s": 2.000260249711573, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71286784, + "prompt_kind": "short", + "t_relative_s": 3797.4099874170497 + }, + { + "decode_latency_s": 0.06791104166768491, + "kv_live_bytes": 462274560, + "latency_s": 2.1755267907865345, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72794112, + "prompt_kind": "long", + "t_relative_s": 3802.4200422503054 + }, + { + "decode_latency_s": 0.06058208075410221, + "kv_live_bytes": 462274560, + "latency_s": 1.9394077500328422, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72794112, + "prompt_kind": "short", + "t_relative_s": 3807.430107833352 + }, + { + "decode_latency_s": 0.06810017446696293, + "kv_live_bytes": 462274560, + "latency_s": 2.1807704577222466, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72859648, + "prompt_kind": "long", + "t_relative_s": 3812.440196292009 + }, + { + "decode_latency_s": 0.06028467578289565, + "kv_live_bytes": 462274560, + "latency_s": 1.9299808330833912, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73416704, + "prompt_kind": "short", + "t_relative_s": 3817.4502532500774 + }, + { + "decode_latency_s": 0.06846458720974624, + "kv_live_bytes": 462274560, + "latency_s": 2.193135042209178, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73433088, + "prompt_kind": "long", + "t_relative_s": 3822.460341833066 + }, + { + "decode_latency_s": 0.06062678777379915, + "kv_live_bytes": 462274560, + "latency_s": 1.9408882092684507, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73269248, + "prompt_kind": "short", + "t_relative_s": 3827.470418542158 + }, + { + "decode_latency_s": 0.06779798959905747, + "kv_live_bytes": 462274560, + "latency_s": 2.1719862911850214, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73367552, + "prompt_kind": "long", + "t_relative_s": 3832.480469833128 + }, + { + "decode_latency_s": 0.06035917706321925, + "kv_live_bytes": 462274560, + "latency_s": 1.932402249891311, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73285632, + "prompt_kind": "short", + "t_relative_s": 3837.4905490833335 + }, + { + "decode_latency_s": 0.06762599086505361, + "kv_live_bytes": 462274560, + "latency_s": 2.1656346251256764, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73302016, + "prompt_kind": "long", + "t_relative_s": 3842.5006217500195 + }, + { + "decode_latency_s": 0.06053358984354418, + "kv_live_bytes": 462274560, + "latency_s": 1.9380069160833955, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "short", + "t_relative_s": 3847.5106687080115 + }, + { + "decode_latency_s": 0.060912753906450234, + "kv_live_bytes": 462274560, + "latency_s": 1.9502072501927614, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73547776, + "prompt_kind": "long", + "t_relative_s": 3852.520746000111 + }, + { + "decode_latency_s": 0.060579660144867375, + "kv_live_bytes": 462274560, + "latency_s": 1.9393646251410246, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73007104, + "prompt_kind": "short", + "t_relative_s": 3857.5308380001225 + }, + { + "decode_latency_s": 0.06722516925947275, + "kv_live_bytes": 462274560, + "latency_s": 2.153846500441432, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73220096, + "prompt_kind": "long", + "t_relative_s": 3862.54089395795 + }, + { + "decode_latency_s": 0.060279345052549616, + "kv_live_bytes": 462274560, + "latency_s": 1.9298342079855502, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73285632, + "prompt_kind": "short", + "t_relative_s": 3867.550980000291 + }, + { + "decode_latency_s": 0.06804126431234181, + "kv_live_bytes": 462274560, + "latency_s": 2.179481790866703, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73203712, + "prompt_kind": "long", + "t_relative_s": 3872.561071000062 + }, + { + "decode_latency_s": 0.06040238149580546, + "kv_live_bytes": 462274560, + "latency_s": 1.9338055411353707, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73760768, + "prompt_kind": "short", + "t_relative_s": 3877.5614534579217 + }, + { + "decode_latency_s": 0.06875093359849416, + "kv_live_bytes": 462274560, + "latency_s": 2.2020011246204376, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66912256, + "prompt_kind": "long", + "t_relative_s": 3882.571531624999 + }, + { + "decode_latency_s": 0.060423347662435845, + "kv_live_bytes": 462274560, + "latency_s": 1.9345156671479344, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71172096, + "prompt_kind": "short", + "t_relative_s": 3887.5796765829436 + }, + { + "decode_latency_s": 0.06791532420902513, + "kv_live_bytes": 462274560, + "latency_s": 2.1754615837708116, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72351744, + "prompt_kind": "long", + "t_relative_s": 3892.5897477502003 + }, + { + "decode_latency_s": 0.060650621104286984, + "kv_live_bytes": 462274560, + "latency_s": 1.9418233749456704, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72499200, + "prompt_kind": "short", + "t_relative_s": 3897.59980416717 + }, + { + "decode_latency_s": 0.0682821314985631, + "kv_live_bytes": 462274560, + "latency_s": 2.18732316698879, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72564736, + "prompt_kind": "long", + "t_relative_s": 3902.606128375046 + }, + { + "decode_latency_s": 0.06051219010259956, + "kv_live_bytes": 462274560, + "latency_s": 1.9371555829420686, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73154560, + "prompt_kind": "short", + "t_relative_s": 3907.6162202083506 + }, + { + "decode_latency_s": 0.06811256772198249, + "kv_live_bytes": 462274560, + "latency_s": 2.1819567917846143, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72712192, + "prompt_kind": "long", + "t_relative_s": 3912.626251167152 + }, + { + "decode_latency_s": 0.06255858071381226, + "kv_live_bytes": 462274560, + "latency_s": 2.002702707890421, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72220672, + "prompt_kind": "short", + "t_relative_s": 3917.63632445829 + }, + { + "decode_latency_s": 0.06825240624311846, + "kv_live_bytes": 462274560, + "latency_s": 2.186641083098948, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72859648, + "prompt_kind": "long", + "t_relative_s": 3922.646401999984 + }, + { + "decode_latency_s": 0.06053844920825213, + "kv_live_bytes": 462274560, + "latency_s": 1.938037124928087, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73383936, + "prompt_kind": "short", + "t_relative_s": 3927.6564831673168 + }, + { + "decode_latency_s": 0.06794967578025535, + "kv_live_bytes": 462274560, + "latency_s": 2.176867707632482, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73302016, + "prompt_kind": "long", + "t_relative_s": 3932.6666627922095 + }, + { + "decode_latency_s": 0.060545639353222214, + "kv_live_bytes": 462274560, + "latency_s": 1.938222416676581, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73760768, + "prompt_kind": "short", + "t_relative_s": 3937.6767767919227 + }, + { + "decode_latency_s": 0.06342363153817132, + "kv_live_bytes": 462274560, + "latency_s": 2.0305602503940463, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68894720, + "prompt_kind": "long", + "t_relative_s": 3942.68682404235 + }, + { + "decode_latency_s": 0.060687490869895555, + "kv_live_bytes": 462274560, + "latency_s": 1.9433827078901231, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69517312, + "prompt_kind": "short", + "t_relative_s": 3947.696924542077 + }, + { + "decode_latency_s": 0.06316327211970929, + "kv_live_bytes": 462274560, + "latency_s": 2.0230176667682827, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70746112, + "prompt_kind": "long", + "t_relative_s": 3952.7069691251963 + }, + { + "decode_latency_s": 0.060492773438454606, + "kv_live_bytes": 462274560, + "latency_s": 1.936580709181726, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71237632, + "prompt_kind": "short", + "t_relative_s": 3957.717063332908 + }, + { + "decode_latency_s": 0.06345985157531686, + "kv_live_bytes": 462274560, + "latency_s": 2.032017458230257, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71319552, + "prompt_kind": "long", + "t_relative_s": 3962.720172292087 + }, + { + "decode_latency_s": 0.060472475248388946, + "kv_live_bytes": 462274560, + "latency_s": 1.9360606661066413, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71483392, + "prompt_kind": "short", + "t_relative_s": 3967.7302334173582 + }, + { + "decode_latency_s": 0.06836427471716888, + "kv_live_bytes": 462274560, + "latency_s": 2.1897036670707166, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72040448, + "prompt_kind": "long", + "t_relative_s": 3972.7403121669777 + }, + { + "decode_latency_s": 0.061220703122671694, + "kv_live_bytes": 462274560, + "latency_s": 1.9599759173579514, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68222976, + "prompt_kind": "short", + "t_relative_s": 3977.7490353332832 + }, + { + "decode_latency_s": 0.06794348437688313, + "kv_live_bytes": 462274560, + "latency_s": 2.176471082959324, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68796416, + "prompt_kind": "long", + "t_relative_s": 3982.7590874172747 + }, + { + "decode_latency_s": 0.060260688784183, + "kv_live_bytes": 462274560, + "latency_s": 1.9292601658962667, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69369856, + "prompt_kind": "short", + "t_relative_s": 3987.76916458318 + }, + { + "decode_latency_s": 0.0679630781232845, + "kv_live_bytes": 462274560, + "latency_s": 2.1764539587311447, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69500928, + "prompt_kind": "long", + "t_relative_s": 3992.7792428750545 + }, + { + "decode_latency_s": 0.06059671874390915, + "kv_live_bytes": 462274560, + "latency_s": 1.9399394998326898, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70074368, + "prompt_kind": "short", + "t_relative_s": 3997.789291750174 + }, + { + "decode_latency_s": 0.06762521353084594, + "kv_live_bytes": 462274560, + "latency_s": 2.1667886674404144, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70025216, + "prompt_kind": "long", + "t_relative_s": 4002.7993799173273 + }, + { + "decode_latency_s": 0.06056556379189715, + "kv_live_bytes": 462274560, + "latency_s": 1.93892833404243, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69566464, + "prompt_kind": "short", + "t_relative_s": 4007.8094511250965 + }, + { + "decode_latency_s": 0.06064263671578374, + "kv_live_bytes": 462274560, + "latency_s": 1.941553124692291, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69025792, + "prompt_kind": "long", + "t_relative_s": 4012.819499500096 + }, + { + "decode_latency_s": 0.06034044270927552, + "kv_live_bytes": 462274560, + "latency_s": 1.931819749996066, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69058560, + "prompt_kind": "short", + "t_relative_s": 4017.8296142923646 + }, + { + "decode_latency_s": 0.06824861590575892, + "kv_live_bytes": 462274560, + "latency_s": 2.1855755411088467, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69697536, + "prompt_kind": "long", + "t_relative_s": 4022.8396937083453 + }, + { + "decode_latency_s": 0.06710165756521747, + "kv_live_bytes": 462274560, + "latency_s": 2.148978707846254, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70270976, + "prompt_kind": "short", + "t_relative_s": 4027.8497550832108 + }, + { + "decode_latency_s": 0.06758616147271823, + "kv_live_bytes": 462274560, + "latency_s": 2.1650755829177797, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70402048, + "prompt_kind": "long", + "t_relative_s": 4032.859842750244 + }, + { + "decode_latency_s": 0.06117408331192564, + "kv_live_bytes": 462274560, + "latency_s": 1.958698958158493, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 64913408, + "prompt_kind": "short", + "t_relative_s": 4037.869948167354 + }, + { + "decode_latency_s": 0.06792352214688435, + "kv_live_bytes": 462274560, + "latency_s": 2.1752957082353532, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67502080, + "prompt_kind": "long", + "t_relative_s": 4042.8800487923436 + }, + { + "decode_latency_s": 0.06698737241094932, + "kv_live_bytes": 462274560, + "latency_s": 2.145467750262469, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69844992, + "prompt_kind": "short", + "t_relative_s": 4047.8900945419446 + }, + { + "decode_latency_s": 0.067749535155599, + "kv_live_bytes": 462274560, + "latency_s": 2.1702865827828646, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69926912, + "prompt_kind": "long", + "t_relative_s": 4052.9001832082868 + }, + { + "decode_latency_s": 0.06054736065561883, + "kv_live_bytes": 462274560, + "latency_s": 1.938283291645348, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70467584, + "prompt_kind": "short", + "t_relative_s": 4057.91027387511 + }, + { + "decode_latency_s": 0.06775684766762424, + "kv_live_bytes": 462274560, + "latency_s": 2.1704386668279767, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69238784, + "prompt_kind": "long", + "t_relative_s": 4062.92030183319 + }, + { + "decode_latency_s": 0.060428302094805986, + "kv_live_bytes": 462274560, + "latency_s": 1.9346277923323214, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69140480, + "prompt_kind": "short", + "t_relative_s": 4067.930375041906 + }, + { + "decode_latency_s": 0.06780017448181752, + "kv_live_bytes": 462274560, + "latency_s": 2.1714514158666134, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69861376, + "prompt_kind": "long", + "t_relative_s": 4072.940449083224 + }, + { + "decode_latency_s": 0.06052116534556262, + "kv_live_bytes": 462274560, + "latency_s": 1.9375864998437464, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70549504, + "prompt_kind": "short", + "t_relative_s": 4077.950146582909 + }, + { + "decode_latency_s": 0.06822515366366133, + "kv_live_bytes": 462274560, + "latency_s": 2.1853308747522533, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70582272, + "prompt_kind": "long", + "t_relative_s": 4082.9602424171753 + }, + { + "decode_latency_s": 0.060783934910432436, + "kv_live_bytes": 462274560, + "latency_s": 1.9459531656466424, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71172096, + "prompt_kind": "short", + "t_relative_s": 4087.9703230420128 + }, + { + "decode_latency_s": 0.06834057680680417, + "kv_live_bytes": 462274560, + "latency_s": 2.189270915929228, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70713344, + "prompt_kind": "long", + "t_relative_s": 4092.980375250336 + }, + { + "decode_latency_s": 0.0608038593782112, + "kv_live_bytes": 462274560, + "latency_s": 1.9465764998458326, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70270976, + "prompt_kind": "short", + "t_relative_s": 4097.990453124978 + }, + { + "decode_latency_s": 0.06875907942594495, + "kv_live_bytes": 462274560, + "latency_s": 2.201918624807149, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 64159744, + "prompt_kind": "long", + "t_relative_s": 4103.000535542145 + }, + { + "decode_latency_s": 0.0605662903690245, + "kv_live_bytes": 462274560, + "latency_s": 1.9390037921257317, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69828608, + "prompt_kind": "short", + "t_relative_s": 4108.010608917102 + }, + { + "decode_latency_s": 0.06045770311902743, + "kv_live_bytes": 462274560, + "latency_s": 1.9356391252949834, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70565888, + "prompt_kind": "long", + "t_relative_s": 4113.020661542192 + }, + { + "decode_latency_s": 0.06016007161815651, + "kv_live_bytes": 462274560, + "latency_s": 1.9260366247035563, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70877184, + "prompt_kind": "short", + "t_relative_s": 4118.030733792111 + }, + { + "decode_latency_s": 0.06754937890218571, + "kv_live_bytes": 462274560, + "latency_s": 2.1633440000005066, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71041024, + "prompt_kind": "long", + "t_relative_s": 4123.040815582965 + }, + { + "decode_latency_s": 0.06669795575726312, + "kv_live_bytes": 462274560, + "latency_s": 2.136203750036657, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71172096, + "prompt_kind": "short", + "t_relative_s": 4128.050886500161 + }, + { + "decode_latency_s": 0.06776955469103996, + "kv_live_bytes": 462274560, + "latency_s": 2.1708814576268196, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71303168, + "prompt_kind": "long", + "t_relative_s": 4133.060937125236 + }, + { + "decode_latency_s": 0.06049950000306126, + "kv_live_bytes": 462274560, + "latency_s": 1.9367447919212282, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71860224, + "prompt_kind": "short", + "t_relative_s": 4138.07103254227 + }, + { + "decode_latency_s": 0.06832872916129418, + "kv_live_bytes": 462274560, + "latency_s": 2.188623874913901, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71794688, + "prompt_kind": "long", + "t_relative_s": 4143.081087667029 + }, + { + "decode_latency_s": 0.06038501561852172, + "kv_live_bytes": 462274560, + "latency_s": 1.933244540821761, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71254016, + "prompt_kind": "short", + "t_relative_s": 4148.091154499911 + }, + { + "decode_latency_s": 0.0683900286530843, + "kv_live_bytes": 462274560, + "latency_s": 2.190195791888982, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71942144, + "prompt_kind": "long", + "t_relative_s": 4153.1012443332 + }, + { + "decode_latency_s": 0.060733173188054934, + "kv_live_bytes": 462274560, + "latency_s": 1.9443350001238286, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72007680, + "prompt_kind": "short", + "t_relative_s": 4158.111324542202 + }, + { + "decode_latency_s": 0.06050502083962783, + "kv_live_bytes": 462274560, + "latency_s": 1.9374538753181696, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72089600, + "prompt_kind": "long", + "t_relative_s": 4163.121386917308 + }, + { + "decode_latency_s": 0.06056126953626517, + "kv_live_bytes": 462274560, + "latency_s": 1.9387357500381768, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72630272, + "prompt_kind": "short", + "t_relative_s": 4168.131463458296 + }, + { + "decode_latency_s": 0.06790064585220534, + "kv_live_bytes": 462274560, + "latency_s": 2.175157249905169, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72548352, + "prompt_kind": "long", + "t_relative_s": 4173.141519207973 + }, + { + "decode_latency_s": 0.06024289193737786, + "kv_live_bytes": 462274560, + "latency_s": 1.9286755411885679, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72630272, + "prompt_kind": "short", + "t_relative_s": 4178.151583833154 + }, + { + "decode_latency_s": 0.06838235545728821, + "kv_live_bytes": 462274560, + "latency_s": 2.1895529995672405, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 63750144, + "prompt_kind": "long", + "t_relative_s": 4183.161653958261 + }, + { + "decode_latency_s": 0.061479433585191146, + "kv_live_bytes": 462274560, + "latency_s": 1.9691693750210106, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66912256, + "prompt_kind": "short", + "t_relative_s": 4188.171719292179 + }, + { + "decode_latency_s": 0.06813603515911382, + "kv_live_bytes": 462274560, + "latency_s": 2.182399165816605, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70746112, + "prompt_kind": "long", + "t_relative_s": 4193.181770833209 + }, + { + "decode_latency_s": 0.06038380469544791, + "kv_live_bytes": 462274560, + "latency_s": 1.9332510828971863, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70533120, + "prompt_kind": "short", + "t_relative_s": 4198.19183575036 + }, + { + "decode_latency_s": 0.060657394540612586, + "kv_live_bytes": 462274560, + "latency_s": 1.9419332090765238, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71188480, + "prompt_kind": "long", + "t_relative_s": 4203.201912167016 + }, + { + "decode_latency_s": 0.06035475652606692, + "kv_live_bytes": 462274560, + "latency_s": 1.932243041228503, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71729152, + "prompt_kind": "short", + "t_relative_s": 4208.211959083099 + }, + { + "decode_latency_s": 0.06096994399558753, + "kv_live_bytes": 462274560, + "latency_s": 1.952211958821863, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71778304, + "prompt_kind": "long", + "t_relative_s": 4213.222044917289 + }, + { + "decode_latency_s": 0.06358077343611512, + "kv_live_bytes": 462274560, + "latency_s": 2.035445415880531, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67092480, + "prompt_kind": "short", + "t_relative_s": 4218.232098875102 + }, + { + "decode_latency_s": 0.06796358594147023, + "kv_live_bytes": 462274560, + "latency_s": 2.176516542211175, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68091904, + "prompt_kind": "long", + "t_relative_s": 4224.506952208001 + }, + { + "decode_latency_s": 0.06078688151319511, + "kv_live_bytes": 462274560, + "latency_s": 1.946005207952112, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68747264, + "prompt_kind": "short", + "t_relative_s": 4229.5170046249405 + }, + { + "decode_latency_s": 0.06938032811740413, + "kv_live_bytes": 462274560, + "latency_s": 2.222579791676253, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68878336, + "prompt_kind": "long", + "t_relative_s": 4234.527086667251 + }, + { + "decode_latency_s": 0.06062459503300488, + "kv_live_bytes": 462274560, + "latency_s": 1.9408892076462507, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69402624, + "prompt_kind": "short", + "t_relative_s": 4239.537164083216 + }, + { + "decode_latency_s": 0.0680410664062947, + "kv_live_bytes": 462274560, + "latency_s": 2.1796668749302626, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66273280, + "prompt_kind": "long", + "t_relative_s": 4244.547228333075 + }, + { + "decode_latency_s": 0.06753829296212643, + "kv_live_bytes": 462274560, + "latency_s": 2.162589999847114, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67403776, + "prompt_kind": "short", + "t_relative_s": 4249.5573100419715 + }, + { + "decode_latency_s": 0.06591605990251992, + "kv_live_bytes": 462274560, + "latency_s": 2.111082249786705, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 55984128, + "prompt_kind": "long", + "t_relative_s": 4254.567390167154 + }, + { + "decode_latency_s": 0.06052566929429304, + "kv_live_bytes": 462274560, + "latency_s": 1.9377087918110192, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68501504, + "prompt_kind": "short", + "t_relative_s": 4259.577456166968 + }, + { + "decode_latency_s": 0.06852608331246302, + "kv_live_bytes": 462274560, + "latency_s": 2.1948546660132706, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69107712, + "prompt_kind": "long", + "t_relative_s": 4264.587508042343 + }, + { + "decode_latency_s": 0.060349276041961275, + "kv_live_bytes": 462274560, + "latency_s": 1.9321151250042021, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69107712, + "prompt_kind": "short", + "t_relative_s": 4269.59757958306 + }, + { + "decode_latency_s": 0.06897288412437774, + "kv_live_bytes": 462274560, + "latency_s": 2.2087374590337276, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67207168, + "prompt_kind": "long", + "t_relative_s": 4274.607655792031 + }, + { + "decode_latency_s": 0.06053803252871148, + "kv_live_bytes": 462274560, + "latency_s": 1.938073417171836, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68681728, + "prompt_kind": "short", + "t_relative_s": 4279.617725375108 + }, + { + "decode_latency_s": 0.06829715624917299, + "kv_live_bytes": 462274560, + "latency_s": 2.187232125084847, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68485120, + "prompt_kind": "long", + "t_relative_s": 4284.627799583133 + }, + { + "decode_latency_s": 0.06063508331135381, + "kv_live_bytes": 462274560, + "latency_s": 1.9411152079701424, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69173248, + "prompt_kind": "short", + "t_relative_s": 4289.637881624978 + }, + { + "decode_latency_s": 0.06824565235001501, + "kv_live_bytes": 462274560, + "latency_s": 2.1859375843778253, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69713920, + "prompt_kind": "long", + "t_relative_s": 4294.647916375194 + }, + { + "decode_latency_s": 0.0604976875038119, + "kv_live_bytes": 462274560, + "latency_s": 1.9368806667625904, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69812224, + "prompt_kind": "short", + "t_relative_s": 4299.657994417008 + }, + { + "decode_latency_s": 0.06823803514998872, + "kv_live_bytes": 462274560, + "latency_s": 2.185472166631371, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66355200, + "prompt_kind": "long", + "t_relative_s": 4304.6680698753335 + }, + { + "decode_latency_s": 0.06117926952720154, + "kv_live_bytes": 462274560, + "latency_s": 1.9586551669053733, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66224128, + "prompt_kind": "short", + "t_relative_s": 4309.67814195808 + }, + { + "decode_latency_s": 0.06846331640554126, + "kv_live_bytes": 462274560, + "latency_s": 2.1931947502307594, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69320704, + "prompt_kind": "long", + "t_relative_s": 4314.688212583307 + }, + { + "decode_latency_s": 0.060309967448120005, + "kv_live_bytes": 462274560, + "latency_s": 1.930869959294796, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68632576, + "prompt_kind": "short", + "t_relative_s": 4319.698306125123 + }, + { + "decode_latency_s": 0.06852156380773522, + "kv_live_bytes": 462274560, + "latency_s": 2.1952043338678777, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69566464, + "prompt_kind": "long", + "t_relative_s": 4324.708349042106 + }, + { + "decode_latency_s": 0.060624772115261294, + "kv_live_bytes": 462274560, + "latency_s": 1.940837583038956, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70238208, + "prompt_kind": "short", + "t_relative_s": 4329.7184326671995 + }, + { + "decode_latency_s": 0.06820776821405161, + "kv_live_bytes": 462274560, + "latency_s": 2.184940207749605, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67682304, + "prompt_kind": "long", + "t_relative_s": 4334.7192451669835 + }, + { + "decode_latency_s": 0.060326619786792435, + "kv_live_bytes": 462274560, + "latency_s": 1.9314792081713676, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68960256, + "prompt_kind": "short", + "t_relative_s": 4339.729330083355 + }, + { + "decode_latency_s": 0.06788668621447869, + "kv_live_bytes": 462274560, + "latency_s": 2.1739632501266897, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69746688, + "prompt_kind": "long", + "t_relative_s": 4344.739397374913 + }, + { + "decode_latency_s": 0.06091179166105576, + "kv_live_bytes": 462274560, + "latency_s": 1.9502930003218353, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69894144, + "prompt_kind": "short", + "t_relative_s": 4349.748921917286 + }, + { + "decode_latency_s": 0.0684197317896178, + "kv_live_bytes": 462274560, + "latency_s": 2.191493249963969, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70500352, + "prompt_kind": "long", + "t_relative_s": 4354.759009582922 + }, + { + "decode_latency_s": 0.06341457812231965, + "kv_live_bytes": 462274560, + "latency_s": 2.030197416432202, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69550080, + "prompt_kind": "short", + "t_relative_s": 4359.769092292059 + }, + { + "decode_latency_s": 0.09239275781146716, + "kv_live_bytes": 462274560, + "latency_s": 2.958213457837701, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67600384, + "prompt_kind": "long", + "t_relative_s": 4364.779165875167 + }, + { + "decode_latency_s": 0.0604607239656616, + "kv_live_bytes": 462274560, + "latency_s": 1.935711707919836, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67289088, + "prompt_kind": "short", + "t_relative_s": 4373.359112458304 + }, + { + "decode_latency_s": 0.24298020702553913, + "kv_live_bytes": 462274560, + "latency_s": 7.776230625342578, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 65847296, + "prompt_kind": "long", + "t_relative_s": 4378.3691513752565 + }, + { + "decode_latency_s": 0.06284499086905271, + "kv_live_bytes": 462274560, + "latency_s": 2.01195729104802, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 63455232, + "prompt_kind": "short", + "t_relative_s": 4392.363406875171 + }, + { + "decode_latency_s": 0.07005913149623666, + "kv_live_bytes": 462274560, + "latency_s": 2.2440097499638796, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66813952, + "prompt_kind": "long", + "t_relative_s": 4397.373488333076 + }, + { + "decode_latency_s": 0.061126451852032915, + "kv_live_bytes": 462274560, + "latency_s": 1.9569881660863757, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67633152, + "prompt_kind": "short", + "t_relative_s": 4402.3835432920605 + }, + { + "decode_latency_s": 0.064603037739289, + "kv_live_bytes": 462274560, + "latency_s": 2.068798207677901, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67469312, + "prompt_kind": "long", + "t_relative_s": 4407.393615167122 + }, + { + "decode_latency_s": 0.06115463409514632, + "kv_live_bytes": 462274560, + "latency_s": 1.9578917496837676, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67616768, + "prompt_kind": "short", + "t_relative_s": 4413.109373582993 + }, + { + "decode_latency_s": 0.06963876821100712, + "kv_live_bytes": 462274560, + "latency_s": 2.2305773328989744, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67616768, + "prompt_kind": "long", + "t_relative_s": 4418.119447917212 + }, + { + "decode_latency_s": 0.17310977083980106, + "kv_live_bytes": 462274560, + "latency_s": 5.540438541676849, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 63045632, + "prompt_kind": "short", + "t_relative_s": 4423.123319041915 + }, + { + "decode_latency_s": 0.061055473968735896, + "kv_live_bytes": 462274560, + "latency_s": 1.9548014998435974, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66338816, + "prompt_kind": "long", + "t_relative_s": 4434.236272375099 + }, + { + "decode_latency_s": 0.060609307285631076, + "kv_live_bytes": 462274560, + "latency_s": 1.940318957902491, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66994176, + "prompt_kind": "short", + "t_relative_s": 4439.246329166926 + }, + { + "decode_latency_s": 0.06795966146455612, + "kv_live_bytes": 462274560, + "latency_s": 2.177657708991319, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67649536, + "prompt_kind": "long", + "t_relative_s": 4444.256408541929 + }, + { + "decode_latency_s": 0.06060444012109656, + "kv_live_bytes": 462274560, + "latency_s": 1.9401070838794112, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68222976, + "prompt_kind": "short", + "t_relative_s": 4449.266499958001 + }, + { + "decode_latency_s": 0.06706480600405484, + "kv_live_bytes": 462274560, + "latency_s": 2.147910540923476, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68321280, + "prompt_kind": "long", + "t_relative_s": 4454.276554542128 + }, + { + "decode_latency_s": 0.060436285159084946, + "kv_live_bytes": 462274560, + "latency_s": 1.9348565000109375, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68943872, + "prompt_kind": "short", + "t_relative_s": 4459.2866402082145 + }, + { + "decode_latency_s": 0.06740111201361287, + "kv_live_bytes": 462274560, + "latency_s": 2.1590919997543097, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69042176, + "prompt_kind": "long", + "t_relative_s": 4464.296722417232 + }, + { + "decode_latency_s": 0.06049453516607173, + "kv_live_bytes": 462274560, + "latency_s": 1.9367648749612272, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69091328, + "prompt_kind": "short", + "t_relative_s": 4469.306793082971 + }, + { + "decode_latency_s": 0.06773041017004289, + "kv_live_bytes": 462274560, + "latency_s": 2.170031958259642, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69173248, + "prompt_kind": "long", + "t_relative_s": 4474.316861167084 + }, + { + "decode_latency_s": 0.0606467304751277, + "kv_live_bytes": 462274560, + "latency_s": 1.9414418330416083, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69222400, + "prompt_kind": "short", + "t_relative_s": 4479.326941583306 + }, + { + "decode_latency_s": 0.06514809114742093, + "kv_live_bytes": 462274560, + "latency_s": 2.0875124158337712, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 62717952, + "prompt_kind": "long", + "t_relative_s": 4484.336986250244 + }, + { + "decode_latency_s": 0.06044305078103207, + "kv_live_bytes": 462274560, + "latency_s": 1.9351804167963564, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66797568, + "prompt_kind": "short", + "t_relative_s": 4489.347065417096 + }, + { + "decode_latency_s": 0.06778677602414973, + "kv_live_bytes": 462274560, + "latency_s": 2.171178916003555, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68567040, + "prompt_kind": "long", + "t_relative_s": 4494.357146000024 + }, + { + "decode_latency_s": 0.060279522120254114, + "kv_live_bytes": 462274560, + "latency_s": 1.9298926671035588, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69238784, + "prompt_kind": "short", + "t_relative_s": 4499.367192458361 + }, + { + "decode_latency_s": 0.06042029298259877, + "kv_live_bytes": 462274560, + "latency_s": 1.9344202089123428, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69173248, + "prompt_kind": "long", + "t_relative_s": 4504.377269000281 + }, + { + "decode_latency_s": 0.06060829556372482, + "kv_live_bytes": 462274560, + "latency_s": 1.940320833120495, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69304320, + "prompt_kind": "short", + "t_relative_s": 4509.387341542169 + }, + { + "decode_latency_s": 0.06797465366253164, + "kv_live_bytes": 462274560, + "latency_s": 2.17733516683802, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68960256, + "prompt_kind": "long", + "t_relative_s": 4514.397388082929 + }, + { + "decode_latency_s": 0.0603776054631453, + "kv_live_bytes": 462274560, + "latency_s": 1.9329579998739064, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69533696, + "prompt_kind": "short", + "t_relative_s": 4519.407463292126 + }, + { + "decode_latency_s": 0.06805113413429353, + "kv_live_bytes": 462274560, + "latency_s": 2.1797797922044992, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69500928, + "prompt_kind": "long", + "t_relative_s": 4524.417542667128 + }, + { + "decode_latency_s": 0.06031271615938749, + "kv_live_bytes": 462274560, + "latency_s": 1.9309281660243869, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70107136, + "prompt_kind": "short", + "t_relative_s": 4529.427584375255 + }, + { + "decode_latency_s": 0.06871302083891351, + "kv_live_bytes": 462274560, + "latency_s": 2.201157667208463, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70156288, + "prompt_kind": "long", + "t_relative_s": 4534.437665417325 + }, + { + "decode_latency_s": 0.0671171354042599, + "kv_live_bytes": 462274560, + "latency_s": 2.1496181660331786, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70254592, + "prompt_kind": "short", + "t_relative_s": 4539.447776500136 + }, + { + "decode_latency_s": 0.13361867055937182, + "kv_live_bytes": 462274560, + "latency_s": 4.276869124732912, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67649536, + "prompt_kind": "long", + "t_relative_s": 4544.457805417012 + }, + { + "decode_latency_s": 0.061105222659534775, + "kv_live_bytes": 462274560, + "latency_s": 1.9567968752235174, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68370432, + "prompt_kind": "short", + "t_relative_s": 4551.509409375023 + }, + { + "decode_latency_s": 0.06805314715893473, + "kv_live_bytes": 462274560, + "latency_s": 2.1798327080905437, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68173824, + "prompt_kind": "long", + "t_relative_s": 4556.519503167365 + }, + { + "decode_latency_s": 0.060454320308053866, + "kv_live_bytes": 462274560, + "latency_s": 1.9355002921074629, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68173824, + "prompt_kind": "short", + "t_relative_s": 4561.52955300035 + }, + { + "decode_latency_s": 0.0682732239656616, + "kv_live_bytes": 462274560, + "latency_s": 2.186857624910772, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68927488, + "prompt_kind": "long", + "t_relative_s": 4566.539630792104 + }, + { + "decode_latency_s": 0.06057993878494017, + "kv_live_bytes": 462274560, + "latency_s": 1.9393163747154176, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68993024, + "prompt_kind": "short", + "t_relative_s": 4571.54969691718 + }, + { + "decode_latency_s": 0.0682582695299061, + "kv_live_bytes": 462274560, + "latency_s": 2.1867134999483824, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68435968, + "prompt_kind": "long", + "t_relative_s": 4576.5597512503155 + }, + { + "decode_latency_s": 0.06051299086539075, + "kv_live_bytes": 462274560, + "latency_s": 1.9371957089751959, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68993024, + "prompt_kind": "short", + "t_relative_s": 4581.569822667167 + }, + { + "decode_latency_s": 0.060844457024359144, + "kv_live_bytes": 462274560, + "latency_s": 1.9482400002889335, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68960256, + "prompt_kind": "long", + "t_relative_s": 4586.579881292302 + }, + { + "decode_latency_s": 0.06045249218004756, + "kv_live_bytes": 462274560, + "latency_s": 1.9354270417243242, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67993600, + "prompt_kind": "short", + "t_relative_s": 4591.589910292067 + }, + { + "decode_latency_s": 0.06835595965094399, + "kv_live_bytes": 462274560, + "latency_s": 2.189274291973561, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68829184, + "prompt_kind": "long", + "t_relative_s": 4596.599993250333 + }, + { + "decode_latency_s": 0.06059261977497954, + "kv_live_bytes": 462274560, + "latency_s": 1.939782917033881, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68861952, + "prompt_kind": "short", + "t_relative_s": 4601.610046749935 + }, + { + "decode_latency_s": 0.06824986197170801, + "kv_live_bytes": 462274560, + "latency_s": 2.1866881670430303, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68681728, + "prompt_kind": "long", + "t_relative_s": 4606.620137500111 + }, + { + "decode_latency_s": 0.0666193060023943, + "kv_live_bytes": 462274560, + "latency_s": 2.133578124921769, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69222400, + "prompt_kind": "short", + "t_relative_s": 4611.630212417338 + }, + { + "decode_latency_s": 0.06081228515540715, + "kv_live_bytes": 462274560, + "latency_s": 1.9474863749928772, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69287936, + "prompt_kind": "long", + "t_relative_s": 4616.640253541991 + }, + { + "decode_latency_s": 0.06041073828237131, + "kv_live_bytes": 462274560, + "latency_s": 1.9340131669305265, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69189632, + "prompt_kind": "short", + "t_relative_s": 4621.650332124904 + }, + { + "decode_latency_s": 0.06817592187144328, + "kv_live_bytes": 462274560, + "latency_s": 2.183285417035222, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69189632, + "prompt_kind": "long", + "t_relative_s": 4626.6603944581 + }, + { + "decode_latency_s": 0.06058508464775514, + "kv_live_bytes": 462274560, + "latency_s": 1.9396117078140378, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69287936, + "prompt_kind": "short", + "t_relative_s": 4631.670443207957 + }, + { + "decode_latency_s": 0.06769412763242144, + "kv_live_bytes": 462274560, + "latency_s": 2.168645082972944, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69255168, + "prompt_kind": "long", + "t_relative_s": 4636.680529708043 + }, + { + "decode_latency_s": 0.06076895572186913, + "kv_live_bytes": 462274560, + "latency_s": 1.945475374814123, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69844992, + "prompt_kind": "short", + "t_relative_s": 4641.6906420420855 + }, + { + "decode_latency_s": 0.06827145049464889, + "kv_live_bytes": 462274560, + "latency_s": 2.1871984591707587, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69910528, + "prompt_kind": "long", + "t_relative_s": 4646.700703167357 + }, + { + "decode_latency_s": 0.06037577733513899, + "kv_live_bytes": 462274560, + "latency_s": 1.9329420421272516, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70451200, + "prompt_kind": "short", + "t_relative_s": 4651.710790333338 + }, + { + "decode_latency_s": 0.06815442060178611, + "kv_live_bytes": 462274560, + "latency_s": 2.1830195407383144, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70369280, + "prompt_kind": "long", + "t_relative_s": 4656.720875957981 + }, + { + "decode_latency_s": 0.06114270049147308, + "kv_live_bytes": 462274560, + "latency_s": 1.9574724999256432, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70516736, + "prompt_kind": "short", + "t_relative_s": 4661.730963333044 + }, + { + "decode_latency_s": 0.06862086978799198, + "kv_live_bytes": 462274560, + "latency_s": 2.19819433381781, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70516736, + "prompt_kind": "long", + "t_relative_s": 4666.741033167113 + }, + { + "decode_latency_s": 0.06071917319786735, + "kv_live_bytes": 462274560, + "latency_s": 1.9438328328542411, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70057984, + "prompt_kind": "short", + "t_relative_s": 4671.751147292089 + }, + { + "decode_latency_s": 0.0681587630242575, + "kv_live_bytes": 462274560, + "latency_s": 2.18379120901227, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70598656, + "prompt_kind": "long", + "t_relative_s": 4676.761200750247 + }, + { + "decode_latency_s": 0.06038513020030223, + "kv_live_bytes": 462274560, + "latency_s": 1.9332144162617624, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70991872, + "prompt_kind": "short", + "t_relative_s": 4681.771285833325 + }, + { + "decode_latency_s": 0.06791740492917597, + "kv_live_bytes": 462274560, + "latency_s": 2.1755690001882613, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71090176, + "prompt_kind": "long", + "t_relative_s": 4686.781365333125 + }, + { + "decode_latency_s": 0.06687057031376753, + "kv_live_bytes": 462274560, + "latency_s": 2.1415487499907613, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71188480, + "prompt_kind": "short", + "t_relative_s": 4691.791419917252 + }, + { + "decode_latency_s": 0.06787540625373367, + "kv_live_bytes": 462274560, + "latency_s": 2.1743784160353243, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70664192, + "prompt_kind": "long", + "t_relative_s": 4696.801509708166 + }, + { + "decode_latency_s": 0.06059817709319759, + "kv_live_bytes": 462274560, + "latency_s": 1.9399295831099153, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71237632, + "prompt_kind": "short", + "t_relative_s": 4701.811595333274 + }, + { + "decode_latency_s": 0.06858668618951924, + "kv_live_bytes": 462274560, + "latency_s": 2.19701504195109, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71254016, + "prompt_kind": "long", + "t_relative_s": 4706.821649417281 + }, + { + "decode_latency_s": 0.06030108334380202, + "kv_live_bytes": 462274560, + "latency_s": 1.9305298337712884, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71778304, + "prompt_kind": "short", + "t_relative_s": 4711.83172116708 + }, + { + "decode_latency_s": 0.060404626303352416, + "kv_live_bytes": 462274560, + "latency_s": 1.9338352913036942, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71139328, + "prompt_kind": "long", + "t_relative_s": 4716.83742854232 + }, + { + "decode_latency_s": 0.06039355204848107, + "kv_live_bytes": 462274560, + "latency_s": 1.9335197499021888, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70565888, + "prompt_kind": "short", + "t_relative_s": 4721.847481958102 + }, + { + "decode_latency_s": 0.06056170574447606, + "kv_live_bytes": 462274560, + "latency_s": 1.9392929170280695, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70909952, + "prompt_kind": "long", + "t_relative_s": 4726.857548958156 + }, + { + "decode_latency_s": 0.06675348959106486, + "kv_live_bytes": 462274560, + "latency_s": 2.137522667180747, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71565312, + "prompt_kind": "short", + "t_relative_s": 4731.867609791923 + }, + { + "decode_latency_s": 0.06744516143226065, + "kv_live_bytes": 462274560, + "latency_s": 2.160501040983945, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71680000, + "prompt_kind": "long", + "t_relative_s": 4736.877668375149 + }, + { + "decode_latency_s": 0.0604398632858647, + "kv_live_bytes": 462274560, + "latency_s": 1.9348346251063049, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72171520, + "prompt_kind": "short", + "t_relative_s": 4741.881665583234 + }, + { + "decode_latency_s": 0.068340677084052, + "kv_live_bytes": 462274560, + "latency_s": 2.1887551662512124, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70615040, + "prompt_kind": "long", + "t_relative_s": 4746.884756166954 + }, + { + "decode_latency_s": 0.0602996875095414, + "kv_live_bytes": 462274560, + "latency_s": 1.9306397079490125, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70238208, + "prompt_kind": "short", + "t_relative_s": 4751.8948022080585 + }, + { + "decode_latency_s": 0.0685246224020375, + "kv_live_bytes": 462274560, + "latency_s": 2.1952765407040715, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71237632, + "prompt_kind": "long", + "t_relative_s": 4756.904886208009 + }, + { + "decode_latency_s": 0.06672046224412043, + "kv_live_bytes": 462274560, + "latency_s": 2.1367073338478804, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72220672, + "prompt_kind": "short", + "t_relative_s": 4761.914943417069 + }, + { + "decode_latency_s": 0.06781036978645716, + "kv_live_bytes": 462274560, + "latency_s": 2.1723663341253996, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72187904, + "prompt_kind": "long", + "t_relative_s": 4766.92501100013 + }, + { + "decode_latency_s": 0.06056901431293227, + "kv_live_bytes": 462274560, + "latency_s": 1.9390004999004304, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "short", + "t_relative_s": 4771.935098000336 + }, + { + "decode_latency_s": 0.0684082903462695, + "kv_live_bytes": 462274560, + "latency_s": 2.190709416754544, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72400896, + "prompt_kind": "long", + "t_relative_s": 4776.94516670797 + }, + { + "decode_latency_s": 0.06057929297094233, + "kv_live_bytes": 462274560, + "latency_s": 1.939426500350237, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72482816, + "prompt_kind": "short", + "t_relative_s": 4781.951034083031 + }, + { + "decode_latency_s": 0.06340114974591415, + "kv_live_bytes": 462274560, + "latency_s": 2.0298934588208795, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69844992, + "prompt_kind": "long", + "t_relative_s": 4786.961101583205 + }, + { + "decode_latency_s": 0.06038183852797374, + "kv_live_bytes": 462274560, + "latency_s": 1.93313662474975, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69926912, + "prompt_kind": "short", + "t_relative_s": 4793.065678250045 + }, + { + "decode_latency_s": 0.06804364843992516, + "kv_live_bytes": 462274560, + "latency_s": 2.1793563752435148, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69910528, + "prompt_kind": "long", + "t_relative_s": 4798.075747667346 + }, + { + "decode_latency_s": 0.0606545377522707, + "kv_live_bytes": 462274560, + "latency_s": 1.9418526249937713, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70483968, + "prompt_kind": "short", + "t_relative_s": 4803.0858117919415 + }, + { + "decode_latency_s": 0.06800217709678691, + "kv_live_bytes": 462274560, + "latency_s": 2.1781224170699716, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70516736, + "prompt_kind": "long", + "t_relative_s": 4808.095872667152 + }, + { + "decode_latency_s": 0.060612475252128206, + "kv_live_bytes": 462274560, + "latency_s": 1.9403772503137589, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70565888, + "prompt_kind": "short", + "t_relative_s": 4813.105968875345 + }, + { + "decode_latency_s": 0.06825047655729577, + "kv_live_bytes": 462274560, + "latency_s": 2.186089209280908, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70303744, + "prompt_kind": "long", + "t_relative_s": 4818.1160246669315 + }, + { + "decode_latency_s": 0.06032723959651776, + "kv_live_bytes": 462274560, + "latency_s": 1.9313323749229312, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70909952, + "prompt_kind": "short", + "t_relative_s": 4823.126103499904 + }, + { + "decode_latency_s": 0.06829401825962123, + "kv_live_bytes": 462274560, + "latency_s": 2.1871190830133855, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71024640, + "prompt_kind": "long", + "t_relative_s": 4828.136186833028 + }, + { + "decode_latency_s": 0.0606272135337349, + "kv_live_bytes": 462274560, + "latency_s": 1.9414509576745331, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71090176, + "prompt_kind": "short", + "t_relative_s": 4833.146240375005 + }, + { + "decode_latency_s": 0.06841318230726756, + "kv_live_bytes": 462274560, + "latency_s": 2.191872874740511, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71172096, + "prompt_kind": "long", + "t_relative_s": 4838.156334000174 + }, + { + "decode_latency_s": 0.0612777890637517, + "kv_live_bytes": 462274560, + "latency_s": 1.9617392080835998, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70303744, + "prompt_kind": "short", + "t_relative_s": 4843.166411207989 + }, + { + "decode_latency_s": 0.06822548827040009, + "kv_live_bytes": 462274560, + "latency_s": 2.185636874753982, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70172672, + "prompt_kind": "long", + "t_relative_s": 4848.176464875229 + }, + { + "decode_latency_s": 0.060646917976555414, + "kv_live_bytes": 462274560, + "latency_s": 1.9416298330761492, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70057984, + "prompt_kind": "short", + "t_relative_s": 4853.18653549999 + }, + { + "decode_latency_s": 0.06773027864983305, + "kv_live_bytes": 462274560, + "latency_s": 2.169249958358705, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70467584, + "prompt_kind": "long", + "t_relative_s": 4858.196608541999 + }, + { + "decode_latency_s": 0.06042998826887924, + "kv_live_bytes": 462274560, + "latency_s": 1.9354559159837663, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71139328, + "prompt_kind": "short", + "t_relative_s": 4863.206659000367 + }, + { + "decode_latency_s": 0.06749762628169265, + "kv_live_bytes": 462274560, + "latency_s": 2.1619292497634888, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71172096, + "prompt_kind": "long", + "t_relative_s": 4868.216749708168 + }, + { + "decode_latency_s": 0.06074832293961663, + "kv_live_bytes": 462274560, + "latency_s": 1.9447468328289688, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71303168, + "prompt_kind": "short", + "t_relative_s": 4873.226846167352 + }, + { + "decode_latency_s": 0.06836746483168099, + "kv_live_bytes": 462274560, + "latency_s": 2.19005358312279, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71270400, + "prompt_kind": "long", + "t_relative_s": 4878.2368795829825 + }, + { + "decode_latency_s": 0.06041995050327387, + "kv_live_bytes": 462274560, + "latency_s": 1.9343196251429617, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71876608, + "prompt_kind": "short", + "t_relative_s": 4883.246962750331 + }, + { + "decode_latency_s": 0.06062530599592719, + "kv_live_bytes": 462274560, + "latency_s": 1.9412900409661233, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71401472, + "prompt_kind": "long", + "t_relative_s": 4888.257039583288 + }, + { + "decode_latency_s": 0.06038472396903671, + "kv_live_bytes": 462274560, + "latency_s": 1.9331955411471426, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71974912, + "prompt_kind": "short", + "t_relative_s": 4893.267093792092 + }, + { + "decode_latency_s": 0.06827966016135179, + "kv_live_bytes": 462274560, + "latency_s": 2.186707832850516, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72089600, + "prompt_kind": "long", + "t_relative_s": 4898.277218125295 + }, + { + "decode_latency_s": 0.06080744792416226, + "kv_live_bytes": 462274560, + "latency_s": 1.9466319167986512, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72138752, + "prompt_kind": "short", + "t_relative_s": 4903.28731620824 + }, + { + "decode_latency_s": 0.06780675781192258, + "kv_live_bytes": 462274560, + "latency_s": 2.1724208327941597, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71712768, + "prompt_kind": "long", + "t_relative_s": 4908.297371292021 + }, + { + "decode_latency_s": 0.06046383985085413, + "kv_live_bytes": 462274560, + "latency_s": 1.935690165963024, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71827456, + "prompt_kind": "short", + "t_relative_s": 4913.307451167144 + }, + { + "decode_latency_s": 0.06764483853476122, + "kv_live_bytes": 462274560, + "latency_s": 2.1663628332316875, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71581696, + "prompt_kind": "long", + "t_relative_s": 4918.31753608305 + }, + { + "decode_latency_s": 0.06027386590722017, + "kv_live_bytes": 462274560, + "latency_s": 1.9296706253662705, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71614464, + "prompt_kind": "short", + "t_relative_s": 4923.3275832082145 + }, + { + "decode_latency_s": 0.06794851433369331, + "kv_live_bytes": 462274560, + "latency_s": 2.1761529999785125, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72237056, + "prompt_kind": "long", + "t_relative_s": 4928.337666708045 + }, + { + "decode_latency_s": 0.06042721224366687, + "kv_live_bytes": 462274560, + "latency_s": 1.9344794591888785, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72318976, + "prompt_kind": "short", + "t_relative_s": 4933.347723999992 + }, + { + "decode_latency_s": 0.06790122266102117, + "kv_live_bytes": 462274560, + "latency_s": 2.175185250118375, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71712768, + "prompt_kind": "long", + "t_relative_s": 4938.3577764579095 + }, + { + "decode_latency_s": 0.06041820443351753, + "kv_live_bytes": 462274560, + "latency_s": 1.9341715001501143, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71745536, + "prompt_kind": "short", + "t_relative_s": 4943.367850417271 + }, + { + "decode_latency_s": 0.0681297590926988, + "kv_live_bytes": 462274560, + "latency_s": 2.1817848328500986, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71401472, + "prompt_kind": "long", + "t_relative_s": 4948.3754688752815 + }, + { + "decode_latency_s": 0.06027451693080366, + "kv_live_bytes": 462274560, + "latency_s": 1.9297515000216663, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71876608, + "prompt_kind": "short", + "t_relative_s": 4953.385514582973 + }, + { + "decode_latency_s": 0.06818188281613402, + "kv_live_bytes": 462274560, + "latency_s": 2.184057124890387, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71991296, + "prompt_kind": "long", + "t_relative_s": 4958.395587624982 + }, + { + "decode_latency_s": 0.06785534633672796, + "kv_live_bytes": 462274560, + "latency_s": 2.173623625189066, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72056832, + "prompt_kind": "short", + "t_relative_s": 4963.405660542194 + }, + { + "decode_latency_s": 0.06767837499501184, + "kv_live_bytes": 462274560, + "latency_s": 2.1678038337267935, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72056832, + "prompt_kind": "long", + "t_relative_s": 4968.415730375331 + }, + { + "decode_latency_s": 0.06047050128108822, + "kv_live_bytes": 462274560, + "latency_s": 1.9358538752421737, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72007680, + "prompt_kind": "short", + "t_relative_s": 4973.425819000229 + }, + { + "decode_latency_s": 0.06789363543794025, + "kv_live_bytes": 462274560, + "latency_s": 2.174544000066817, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71909376, + "prompt_kind": "long", + "t_relative_s": 4978.43544233311 + }, + { + "decode_latency_s": 0.06162783724721521, + "kv_live_bytes": 462274560, + "latency_s": 1.9731065407395363, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68485120, + "prompt_kind": "short", + "t_relative_s": 4983.445497833192 + }, + { + "decode_latency_s": 0.06768397397536319, + "kv_live_bytes": 462274560, + "latency_s": 2.1679603750817478, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70582272, + "prompt_kind": "long", + "t_relative_s": 4988.4555446673185 + }, + { + "decode_latency_s": 0.06048193227616139, + "kv_live_bytes": 462274560, + "latency_s": 1.9361697086133063, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70729728, + "prompt_kind": "short", + "t_relative_s": 4993.4656403330155 + }, + { + "decode_latency_s": 0.06792391797353048, + "kv_live_bytes": 462274560, + "latency_s": 2.1753797912970185, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70074368, + "prompt_kind": "long", + "t_relative_s": 4998.4756803330965 + }, + { + "decode_latency_s": 0.0603189140529139, + "kv_live_bytes": 462274560, + "latency_s": 1.9311778331175447, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70385664, + "prompt_kind": "short", + "t_relative_s": 5003.485750208143 + }, + { + "decode_latency_s": 0.06819344009272754, + "kv_live_bytes": 462274560, + "latency_s": 2.1844648751430213, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71057408, + "prompt_kind": "long", + "t_relative_s": 5008.4958337503485 + }, + { + "decode_latency_s": 0.06647875520866364, + "kv_live_bytes": 462274560, + "latency_s": 2.129160459153354, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71630848, + "prompt_kind": "short", + "t_relative_s": 5013.505903083365 + }, + { + "decode_latency_s": 0.0680990026157815, + "kv_live_bytes": 462274560, + "latency_s": 2.1816036659292877, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71729152, + "prompt_kind": "long", + "t_relative_s": 5018.5159669169225 + }, + { + "decode_latency_s": 0.0633603671885794, + "kv_live_bytes": 462274560, + "latency_s": 2.0284154578112066, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69910528, + "prompt_kind": "short", + "t_relative_s": 5023.526051583234 + }, + { + "decode_latency_s": 0.06824391537520569, + "kv_live_bytes": 462274560, + "latency_s": 2.186329582706094, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70811648, + "prompt_kind": "long", + "t_relative_s": 5028.536135625094 + }, + { + "decode_latency_s": 0.060702839837176725, + "kv_live_bytes": 462274560, + "latency_s": 1.9432618748396635, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71352320, + "prompt_kind": "short", + "t_relative_s": 5033.546220000368 + }, + { + "decode_latency_s": 0.06064721875009127, + "kv_live_bytes": 462274560, + "latency_s": 1.941572249867022, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71401472, + "prompt_kind": "long", + "t_relative_s": 5038.55625441717 + }, + { + "decode_latency_s": 0.06045028124935925, + "kv_live_bytes": 462274560, + "latency_s": 1.9351682080887258, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71892992, + "prompt_kind": "short", + "t_relative_s": 5043.566330542322 + }, + { + "decode_latency_s": 0.068170985658071, + "kv_live_bytes": 462274560, + "latency_s": 2.183476416859776, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71942144, + "prompt_kind": "long", + "t_relative_s": 5048.576410542242 + }, + { + "decode_latency_s": 0.06040767187369056, + "kv_live_bytes": 462274560, + "latency_s": 1.9339162497781217, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72073216, + "prompt_kind": "short", + "t_relative_s": 5053.586466582958 + }, + { + "decode_latency_s": 0.06796346225019079, + "kv_live_bytes": 462274560, + "latency_s": 2.176752499770373, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72171520, + "prompt_kind": "long", + "t_relative_s": 5058.596545167267 + }, + { + "decode_latency_s": 0.06057231512386352, + "kv_live_bytes": 462274560, + "latency_s": 1.9391308748163283, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72269824, + "prompt_kind": "short", + "t_relative_s": 5063.606648458168 + }, + { + "decode_latency_s": 0.06789435936661903, + "kv_live_bytes": 462274560, + "latency_s": 2.175045750103891, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71729152, + "prompt_kind": "long", + "t_relative_s": 5068.616705375258 + }, + { + "decode_latency_s": 0.06037813150032889, + "kv_live_bytes": 462274560, + "latency_s": 1.9330028328113258, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72269824, + "prompt_kind": "short", + "t_relative_s": 5073.626786167268 + }, + { + "decode_latency_s": 0.06833160415408202, + "kv_live_bytes": 462274560, + "latency_s": 2.1881961659528315, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72368128, + "prompt_kind": "long", + "t_relative_s": 5078.6369097922 + }, + { + "decode_latency_s": 0.060807358066085726, + "kv_live_bytes": 462274560, + "latency_s": 1.946648959070444, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71745536, + "prompt_kind": "short", + "t_relative_s": 5083.646952250041 + }, + { + "decode_latency_s": 0.06064116668130737, + "kv_live_bytes": 462274560, + "latency_s": 1.942085084039718, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71925760, + "prompt_kind": "long", + "t_relative_s": 5088.657005625311 + }, + { + "decode_latency_s": 0.06038334374898113, + "kv_live_bytes": 462274560, + "latency_s": 1.9331630831584334, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72302592, + "prompt_kind": "short", + "t_relative_s": 5093.667072083335 + }, + { + "decode_latency_s": 0.0678113307512831, + "kv_live_bytes": 462274560, + "latency_s": 2.1716542500071228, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72351744, + "prompt_kind": "long", + "t_relative_s": 5098.677161999978 + }, + { + "decode_latency_s": 0.06059409637236968, + "kv_live_bytes": 462274560, + "latency_s": 1.9399039172567427, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72892416, + "prompt_kind": "short", + "t_relative_s": 5103.6872396250255 + }, + { + "decode_latency_s": 0.0675557044451125, + "kv_live_bytes": 462274560, + "latency_s": 2.1642337907105684, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72925184, + "prompt_kind": "long", + "t_relative_s": 5108.697334625293 + }, + { + "decode_latency_s": 0.06055227605975233, + "kv_live_bytes": 462274560, + "latency_s": 1.9384324587881565, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73007104, + "prompt_kind": "short", + "t_relative_s": 5113.707436542027 + }, + { + "decode_latency_s": 0.06811691797338426, + "kv_live_bytes": 462274560, + "latency_s": 2.181955875363201, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73007104, + "prompt_kind": "long", + "t_relative_s": 5118.717488416936 + }, + { + "decode_latency_s": 0.06038824869028758, + "kv_live_bytes": 462274560, + "latency_s": 1.9333183751441538, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72351744, + "prompt_kind": "short", + "t_relative_s": 5123.727576750331 + }, + { + "decode_latency_s": 0.06036850131931715, + "kv_live_bytes": 462274560, + "latency_s": 1.9327156245708466, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72351744, + "prompt_kind": "long", + "t_relative_s": 5128.737657958176 + }, + { + "decode_latency_s": 0.06082091406278778, + "kv_live_bytes": 462274560, + "latency_s": 1.9471612498164177, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72892416, + "prompt_kind": "short", + "t_relative_s": 5133.747712083161 + }, + { + "decode_latency_s": 0.06758523438475095, + "kv_live_bytes": 462274560, + "latency_s": 2.1648479159921408, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72564736, + "prompt_kind": "long", + "t_relative_s": 5138.757809875067 + }, + { + "decode_latency_s": 0.061882764304755256, + "kv_live_bytes": 462274560, + "latency_s": 1.9814810841344297, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71090176, + "prompt_kind": "short", + "t_relative_s": 5143.767896042205 + }, + { + "decode_latency_s": 0.0686498124996433, + "kv_live_bytes": 462274560, + "latency_s": 2.1987967076711357, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72138752, + "prompt_kind": "long", + "t_relative_s": 5148.7779948753305 + }, + { + "decode_latency_s": 0.0607852877874393, + "kv_live_bytes": 462274560, + "latency_s": 1.945929542183876, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72630272, + "prompt_kind": "short", + "t_relative_s": 5153.788041042164 + }, + { + "decode_latency_s": 0.06776922264543828, + "kv_live_bytes": 462274560, + "latency_s": 2.1710374588146806, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72728576, + "prompt_kind": "long", + "t_relative_s": 5158.7981252921745 + }, + { + "decode_latency_s": 0.06084985155030154, + "kv_live_bytes": 462274560, + "latency_s": 1.9480158751830459, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73236480, + "prompt_kind": "short", + "t_relative_s": 5163.808200458065 + }, + { + "decode_latency_s": 0.06736497003294062, + "kv_live_bytes": 462274560, + "latency_s": 2.1576606249436736, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72761344, + "prompt_kind": "long", + "t_relative_s": 5168.8182581672445 + }, + { + "decode_latency_s": 0.06049643881851807, + "kv_live_bytes": 462274560, + "latency_s": 1.9368297909386456, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72024064, + "prompt_kind": "short", + "t_relative_s": 5173.82831449993 + }, + { + "decode_latency_s": 0.06792703906830866, + "kv_live_bytes": 462274560, + "latency_s": 2.1753812078386545, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "long", + "t_relative_s": 5178.838405082934 + }, + { + "decode_latency_s": 0.06047511064389255, + "kv_live_bytes": 462274560, + "latency_s": 1.9360533328726888, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72810496, + "prompt_kind": "short", + "t_relative_s": 5183.848455999978 + }, + { + "decode_latency_s": 0.06779176952841226, + "kv_live_bytes": 462274560, + "latency_s": 2.1716402913443744, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72859648, + "prompt_kind": "long", + "t_relative_s": 5188.858518667053 + }, + { + "decode_latency_s": 0.060833471376099624, + "kv_live_bytes": 462274560, + "latency_s": 1.9474725839681923, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73498624, + "prompt_kind": "short", + "t_relative_s": 5193.86858775001 + }, + { + "decode_latency_s": 0.06817613540624734, + "kv_live_bytes": 462274560, + "latency_s": 2.184313707984984, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73596928, + "prompt_kind": "long", + "t_relative_s": 5198.878648458049 + }, + { + "decode_latency_s": 0.06037428906711284, + "kv_live_bytes": 462274560, + "latency_s": 1.9329004581086338, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72318976, + "prompt_kind": "short", + "t_relative_s": 5203.888722958043 + }, + { + "decode_latency_s": 0.06806583462457638, + "kv_live_bytes": 462274560, + "latency_s": 2.180670500267297, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72974336, + "prompt_kind": "long", + "t_relative_s": 5208.898758500349 + }, + { + "decode_latency_s": 0.060314485686831176, + "kv_live_bytes": 462274560, + "latency_s": 1.93087258329615, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73023488, + "prompt_kind": "short", + "t_relative_s": 5213.908831333276 + }, + { + "decode_latency_s": 0.06805656899814494, + "kv_live_bytes": 462274560, + "latency_s": 2.179716874845326, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72810496, + "prompt_kind": "long", + "t_relative_s": 5218.918903542217 + }, + { + "decode_latency_s": 0.06131038410239853, + "kv_live_bytes": 462274560, + "latency_s": 1.9627695833332837, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70025216, + "prompt_kind": "short", + "t_relative_s": 5223.928959167097 + }, + { + "decode_latency_s": 0.06689815752906725, + "kv_live_bytes": 462274560, + "latency_s": 2.143056540749967, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72876032, + "prompt_kind": "long", + "t_relative_s": 5228.939011916984 + }, + { + "decode_latency_s": 0.0602659687428968, + "kv_live_bytes": 462274560, + "latency_s": 1.929359540808946, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72761344, + "prompt_kind": "short", + "t_relative_s": 5233.949092749972 + }, + { + "decode_latency_s": 0.06789299218507949, + "kv_live_bytes": 462274560, + "latency_s": 2.17489912500605, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72450048, + "prompt_kind": "long", + "t_relative_s": 5238.959147625137 + }, + { + "decode_latency_s": 0.06037170052877627, + "kv_live_bytes": 462274560, + "latency_s": 1.9326664158143103, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73056256, + "prompt_kind": "short", + "t_relative_s": 5243.969221916981 + }, + { + "decode_latency_s": 0.06763759764726274, + "kv_live_bytes": 462274560, + "latency_s": 2.166039042174816, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73089024, + "prompt_kind": "long", + "t_relative_s": 5248.979266083334 + }, + { + "decode_latency_s": 0.06054453906835988, + "kv_live_bytes": 462274560, + "latency_s": 1.9382123341783881, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73154560, + "prompt_kind": "short", + "t_relative_s": 5253.989337208215 + }, + { + "decode_latency_s": 0.06804172655392904, + "kv_live_bytes": 462274560, + "latency_s": 2.1794967497698963, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73744384, + "prompt_kind": "long", + "t_relative_s": 5258.9993938333355 + }, + { + "decode_latency_s": 0.06086014321772382, + "kv_live_bytes": 462274560, + "latency_s": 1.948417709209025, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73809920, + "prompt_kind": "short", + "t_relative_s": 5264.00741745811 + }, + { + "decode_latency_s": 0.067548365885159, + "kv_live_bytes": 462274560, + "latency_s": 2.163176625035703, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73334784, + "prompt_kind": "long", + "t_relative_s": 5269.017503583338 + }, + { + "decode_latency_s": 0.060765792964957654, + "kv_live_bytes": 462274560, + "latency_s": 1.9453703751787543, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72957952, + "prompt_kind": "short", + "t_relative_s": 5274.0275592501275 + }, + { + "decode_latency_s": 0.06794223828183021, + "kv_live_bytes": 462274560, + "latency_s": 2.1759425830096006, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73580544, + "prompt_kind": "long", + "t_relative_s": 5279.037636208348 + }, + { + "decode_latency_s": 0.06081358203664422, + "kv_live_bytes": 462274560, + "latency_s": 1.9468350000679493, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74006528, + "prompt_kind": "short", + "t_relative_s": 5284.047742542345 + }, + { + "decode_latency_s": 0.06745507684536278, + "kv_live_bytes": 462274560, + "latency_s": 2.1607816251926124, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74186752, + "prompt_kind": "long", + "t_relative_s": 5289.057797833346 + }, + { + "decode_latency_s": 0.060605492180911824, + "kv_live_bytes": 462274560, + "latency_s": 1.9401220004074275, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73678848, + "prompt_kind": "short", + "t_relative_s": 5294.067877416965 + }, + { + "decode_latency_s": 0.06522247528482694, + "kv_live_bytes": 462274560, + "latency_s": 2.088503667153418, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69894144, + "prompt_kind": "long", + "t_relative_s": 5299.205730125308 + }, + { + "decode_latency_s": 0.06037222524173558, + "kv_live_bytes": 462274560, + "latency_s": 1.9328091670759022, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70025216, + "prompt_kind": "short", + "t_relative_s": 5305.65885675 + }, + { + "decode_latency_s": 0.06806552734633442, + "kv_live_bytes": 462274560, + "latency_s": 2.179945208132267, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70139904, + "prompt_kind": "long", + "t_relative_s": 5310.668921750039 + }, + { + "decode_latency_s": 0.06083634766400792, + "kv_live_bytes": 462274560, + "latency_s": 1.9476372092030942, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70516736, + "prompt_kind": "short", + "t_relative_s": 5315.678985208273 + }, + { + "decode_latency_s": 0.06797776820894796, + "kv_live_bytes": 462274560, + "latency_s": 2.177840624935925, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70615040, + "prompt_kind": "long", + "t_relative_s": 5320.689063082915 + }, + { + "decode_latency_s": 0.06076434765418526, + "kv_live_bytes": 462274560, + "latency_s": 1.9452499998733401, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70680576, + "prompt_kind": "short", + "t_relative_s": 5325.699155000038 + }, + { + "decode_latency_s": 0.06748307812085841, + "kv_live_bytes": 462274560, + "latency_s": 2.1613934580236673, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70172672, + "prompt_kind": "long", + "t_relative_s": 5330.709202458151 + }, + { + "decode_latency_s": 0.06048290233593434, + "kv_live_bytes": 462274560, + "latency_s": 1.9363846252672374, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69828608, + "prompt_kind": "short", + "t_relative_s": 5335.719256542157 + }, + { + "decode_latency_s": 0.06748781508940738, + "kv_live_bytes": 462274560, + "latency_s": 2.162140333093703, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70418432, + "prompt_kind": "long", + "t_relative_s": 5340.729300958104 + }, + { + "decode_latency_s": 0.06038838150561787, + "kv_live_bytes": 462274560, + "latency_s": 1.9332730420865119, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70959104, + "prompt_kind": "short", + "t_relative_s": 5345.73938725004 + }, + { + "decode_latency_s": 0.06798706122208387, + "kv_live_bytes": 462274560, + "latency_s": 2.177361374720931, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71008256, + "prompt_kind": "long", + "t_relative_s": 5350.749483500142 + }, + { + "decode_latency_s": 0.060692601575283334, + "kv_live_bytes": 462274560, + "latency_s": 1.9431758751161397, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70647808, + "prompt_kind": "short", + "t_relative_s": 5355.759537125006 + }, + { + "decode_latency_s": 0.06760408593981992, + "kv_live_bytes": 462274560, + "latency_s": 2.1654435000382364, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70746112, + "prompt_kind": "long", + "t_relative_s": 5360.769598292187 + }, + { + "decode_latency_s": 0.060482018219772726, + "kv_live_bytes": 462274560, + "latency_s": 1.9361868752166629, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71467008, + "prompt_kind": "short", + "t_relative_s": 5365.77967308322 + }, + { + "decode_latency_s": 0.0680831002391642, + "kv_live_bytes": 462274560, + "latency_s": 2.1812083330005407, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71434240, + "prompt_kind": "long", + "t_relative_s": 5370.789729792159 + }, + { + "decode_latency_s": 0.06050017577945255, + "kv_live_bytes": 462274560, + "latency_s": 1.9369919998571277, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71499776, + "prompt_kind": "short", + "t_relative_s": 5375.799800542183 + }, + { + "decode_latency_s": 0.06810191666590981, + "kv_live_bytes": 462274560, + "latency_s": 2.1809036671184003, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70303744, + "prompt_kind": "long", + "t_relative_s": 5380.8098741672 + }, + { + "decode_latency_s": 0.06343649477639701, + "kv_live_bytes": 462274560, + "latency_s": 2.030799083877355, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66125824, + "prompt_kind": "short", + "t_relative_s": 5385.8123172502965 + }, + { + "decode_latency_s": 0.0681252916547237, + "kv_live_bytes": 462274560, + "latency_s": 2.1820747079327703, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70795264, + "prompt_kind": "long", + "t_relative_s": 5390.8224103753455 + }, + { + "decode_latency_s": 0.06048014711996075, + "kv_live_bytes": 462274560, + "latency_s": 1.9361817087046802, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71417856, + "prompt_kind": "short", + "t_relative_s": 5395.832482667174 + }, + { + "decode_latency_s": 0.06782659243617672, + "kv_live_bytes": 462274560, + "latency_s": 2.172808540984988, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71467008, + "prompt_kind": "long", + "t_relative_s": 5400.842534500174 + }, + { + "decode_latency_s": 0.06082322915608529, + "kv_live_bytes": 462274560, + "latency_s": 1.9471597908996046, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71876608, + "prompt_kind": "short", + "t_relative_s": 5405.852609250229 + }, + { + "decode_latency_s": 0.06795397003588732, + "kv_live_bytes": 462274560, + "latency_s": 2.1762945423834026, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71401472, + "prompt_kind": "long", + "t_relative_s": 5410.862658792175 + }, + { + "decode_latency_s": 0.06097272525948938, + "kv_live_bytes": 462274560, + "latency_s": 1.952005541883409, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70746112, + "prompt_kind": "short", + "t_relative_s": 5415.872744458262 + }, + { + "decode_latency_s": 0.06782374999602325, + "kv_live_bytes": 462274560, + "latency_s": 2.1724480828270316, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71368704, + "prompt_kind": "long", + "t_relative_s": 5420.882795417216 + }, + { + "decode_latency_s": 0.06046809506369755, + "kv_live_bytes": 462274560, + "latency_s": 1.9358747922815382, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71401472, + "prompt_kind": "short", + "t_relative_s": 5425.892891542055 + }, + { + "decode_latency_s": 0.06887476301926654, + "kv_live_bytes": 462274560, + "latency_s": 2.2057210421189666, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69484544, + "prompt_kind": "long", + "t_relative_s": 5430.902987250127 + }, + { + "decode_latency_s": 0.0605760963371722, + "kv_live_bytes": 462274560, + "latency_s": 1.9393745423294604, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71073792, + "prompt_kind": "short", + "t_relative_s": 5435.913033707999 + }, + { + "decode_latency_s": 0.06869764193834271, + "kv_live_bytes": 462274560, + "latency_s": 2.2005172092467546, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71811072, + "prompt_kind": "long", + "t_relative_s": 5440.92310133297 + }, + { + "decode_latency_s": 0.06068565105670132, + "kv_live_bytes": 462274560, + "latency_s": 1.9427439579740167, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71892992, + "prompt_kind": "short", + "t_relative_s": 5445.933157166932 + }, + { + "decode_latency_s": 0.0626436510210624, + "kv_live_bytes": 462274560, + "latency_s": 2.006334625184536, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71385088, + "prompt_kind": "long", + "t_relative_s": 5450.943206042051 + }, + { + "decode_latency_s": 0.06504875000973698, + "kv_live_bytes": 462274560, + "latency_s": 2.083002416882664, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69337088, + "prompt_kind": "short", + "t_relative_s": 5455.95155287534 + }, + { + "decode_latency_s": 0.06754390624701045, + "kv_live_bytes": 462274560, + "latency_s": 2.163765999954194, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70320128, + "prompt_kind": "long", + "t_relative_s": 5460.961672374979 + }, + { + "decode_latency_s": 0.06778468750417233, + "kv_live_bytes": 462274560, + "latency_s": 2.170501542277634, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69386240, + "prompt_kind": "short", + "t_relative_s": 5465.971719833091 + }, + { + "decode_latency_s": 0.06809895571495872, + "kv_live_bytes": 462274560, + "latency_s": 2.1811808752827346, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70746112, + "prompt_kind": "long", + "t_relative_s": 5470.981811167207 + }, + { + "decode_latency_s": 0.06687271874397993, + "kv_live_bytes": 462274560, + "latency_s": 2.1416138340719044, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70926336, + "prompt_kind": "short", + "t_relative_s": 5475.991866874974 + }, + { + "decode_latency_s": 0.0678113789181225, + "kv_live_bytes": 462274560, + "latency_s": 2.1723600002005696, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70991872, + "prompt_kind": "long", + "t_relative_s": 5481.001979667228 + }, + { + "decode_latency_s": 0.060588087246287614, + "kv_live_bytes": 462274560, + "latency_s": 1.9395846673287451, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71630848, + "prompt_kind": "short", + "t_relative_s": 5486.012055667117 + }, + { + "decode_latency_s": 0.06814298697281629, + "kv_live_bytes": 462274560, + "latency_s": 2.1829429999925196, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71712768, + "prompt_kind": "long", + "t_relative_s": 5491.022108542267 + }, + { + "decode_latency_s": 0.06058556250354741, + "kv_live_bytes": 462274560, + "latency_s": 1.939576624892652, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72187904, + "prompt_kind": "short", + "t_relative_s": 5496.0321736251935 + }, + { + "decode_latency_s": 0.06857630077865906, + "kv_live_bytes": 462274560, + "latency_s": 2.19611837528646, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71221248, + "prompt_kind": "long", + "t_relative_s": 5501.042254874948 + }, + { + "decode_latency_s": 0.060632440116023645, + "kv_live_bytes": 462274560, + "latency_s": 1.9411457078531384, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71155712, + "prompt_kind": "short", + "t_relative_s": 5506.052305832971 + }, + { + "decode_latency_s": 0.06824071222217754, + "kv_live_bytes": 462274560, + "latency_s": 2.185823292005807, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71843840, + "prompt_kind": "long", + "t_relative_s": 5511.062390374951 + }, + { + "decode_latency_s": 0.06069001040305011, + "kv_live_bytes": 462274560, + "latency_s": 1.9428435829468071, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72384512, + "prompt_kind": "short", + "t_relative_s": 5516.072472792119 + }, + { + "decode_latency_s": 0.06525982031598687, + "kv_live_bytes": 462274560, + "latency_s": 2.090739833191037, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72302592, + "prompt_kind": "long", + "t_relative_s": 5521.082525667269 + }, + { + "decode_latency_s": 0.06117649868247099, + "kv_live_bytes": 462274560, + "latency_s": 1.958419790957123, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72318976, + "prompt_kind": "short", + "t_relative_s": 5526.0926021672785 + }, + { + "decode_latency_s": 0.06794289973913692, + "kv_live_bytes": 462274560, + "latency_s": 2.1759312078356743, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71696384, + "prompt_kind": "long", + "t_relative_s": 5531.099274000153 + }, + { + "decode_latency_s": 0.06047992059029639, + "kv_live_bytes": 462274560, + "latency_s": 1.936479666735977, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70254592, + "prompt_kind": "short", + "t_relative_s": 5536.109302250203 + }, + { + "decode_latency_s": 0.06812442706723232, + "kv_live_bytes": 462274560, + "latency_s": 2.1821375000290573, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72155136, + "prompt_kind": "long", + "t_relative_s": 5541.119377458002 + }, + { + "decode_latency_s": 0.06661527471442241, + "kv_live_bytes": 462274560, + "latency_s": 2.133393708150834, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72892416, + "prompt_kind": "short", + "t_relative_s": 5546.129453833215 + }, + { + "decode_latency_s": 0.06039210027665831, + "kv_live_bytes": 462274560, + "latency_s": 1.9335072920657694, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72908800, + "prompt_kind": "long", + "t_relative_s": 5551.139526083134 + }, + { + "decode_latency_s": 0.06081705207179766, + "kv_live_bytes": 462274560, + "latency_s": 1.9469068329781294, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73531392, + "prompt_kind": "short", + "t_relative_s": 5556.149596458301 + }, + { + "decode_latency_s": 0.0681757994752843, + "kv_live_bytes": 462274560, + "latency_s": 2.1836214996874332, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73351168, + "prompt_kind": "long", + "t_relative_s": 5561.159647458233 + }, + { + "decode_latency_s": 0.06053571743541397, + "kv_live_bytes": 462274560, + "latency_s": 1.9379719588905573, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73023488, + "prompt_kind": "short", + "t_relative_s": 5566.169730375055 + }, + { + "decode_latency_s": 0.06773890493786894, + "kv_live_bytes": 462274560, + "latency_s": 2.1701154173351824, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72810496, + "prompt_kind": "long", + "t_relative_s": 5571.179782667197 + }, + { + "decode_latency_s": 0.060279744779109024, + "kv_live_bytes": 462274560, + "latency_s": 1.9299211660400033, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72859648, + "prompt_kind": "short", + "t_relative_s": 5576.1898743333295 + }, + { + "decode_latency_s": 0.06758727997657843, + "kv_live_bytes": 462274560, + "latency_s": 2.164323542267084, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72843264, + "prompt_kind": "long", + "t_relative_s": 5581.1999528333545 + }, + { + "decode_latency_s": 0.06697456250549294, + "kv_live_bytes": 462274560, + "latency_s": 2.145015375223011, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73400320, + "prompt_kind": "short", + "t_relative_s": 5586.210003792308 + }, + { + "decode_latency_s": 0.06792639844934456, + "kv_live_bytes": 462274560, + "latency_s": 2.1761746662668884, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73498624, + "prompt_kind": "long", + "t_relative_s": 5591.220110583119 + }, + { + "decode_latency_s": 0.060787549504311755, + "kv_live_bytes": 462274560, + "latency_s": 1.9460160001181066, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72957952, + "prompt_kind": "short", + "t_relative_s": 5596.230195542332 + }, + { + "decode_latency_s": 0.06791008592699654, + "kv_live_bytes": 462274560, + "latency_s": 2.1754917092621326, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72974336, + "prompt_kind": "long", + "t_relative_s": 5601.240249708295 + }, + { + "decode_latency_s": 0.06040788804239128, + "kv_live_bytes": 462274560, + "latency_s": 1.9339602920226753, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "short", + "t_relative_s": 5606.2423578752205 + }, + { + "decode_latency_s": 0.06844254296447616, + "kv_live_bytes": 462274560, + "latency_s": 2.1917563751339912, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73465856, + "prompt_kind": "long", + "t_relative_s": 5611.252437457908 + }, + { + "decode_latency_s": 0.06698280728596728, + "kv_live_bytes": 462274560, + "latency_s": 2.145127291791141, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74137600, + "prompt_kind": "short", + "t_relative_s": 5616.262495333329 + }, + { + "decode_latency_s": 0.06802464193606284, + "kv_live_bytes": 462274560, + "latency_s": 2.1792359170503914, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74088448, + "prompt_kind": "long", + "t_relative_s": 5621.272609750275 + }, + { + "decode_latency_s": 0.0606346562562976, + "kv_live_bytes": 462274560, + "latency_s": 1.9410959999077022, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74039296, + "prompt_kind": "short", + "t_relative_s": 5626.2826888333075 + }, + { + "decode_latency_s": 0.06792924087494612, + "kv_live_bytes": 462274560, + "latency_s": 2.175788374617696, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73973760, + "prompt_kind": "long", + "t_relative_s": 5631.292742125224 + }, + { + "decode_latency_s": 0.06025821615185123, + "kv_live_bytes": 462274560, + "latency_s": 1.9291592920199037, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72744960, + "prompt_kind": "short", + "t_relative_s": 5636.302814458031 + }, + { + "decode_latency_s": 0.06806530077301431, + "kv_live_bytes": 462274560, + "latency_s": 2.1798598342575133, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73433088, + "prompt_kind": "long", + "t_relative_s": 5641.312896457966 + }, + { + "decode_latency_s": 0.06076919271436054, + "kv_live_bytes": 462274560, + "latency_s": 1.9454945409670472, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73957376, + "prompt_kind": "short", + "t_relative_s": 5646.313850583043 + }, + { + "decode_latency_s": 0.06799564715765882, + "kv_live_bytes": 462274560, + "latency_s": 2.178237958345562, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74104832, + "prompt_kind": "long", + "t_relative_s": 5651.323931792285 + }, + { + "decode_latency_s": 0.2484007604070939, + "kv_live_bytes": 462274560, + "latency_s": 7.949615667108446, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 197476352, + "prompt_kind": "short", + "t_relative_s": 5656.3340245829895 + }, + { + "decode_latency_s": 0.06817980730556883, + "kv_live_bytes": 462274560, + "latency_s": 2.184153167065233, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 198148096, + "prompt_kind": "long", + "t_relative_s": 5665.166657625232 + }, + { + "decode_latency_s": 0.06037077865039464, + "kv_live_bytes": 462274560, + "latency_s": 1.9328107498586178, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 197689344, + "prompt_kind": "short", + "t_relative_s": 5670.176732750144 + }, + { + "decode_latency_s": 0.06067856641311664, + "kv_live_bytes": 462274560, + "latency_s": 1.9425846668891609, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 197820416, + "prompt_kind": "long", + "t_relative_s": 5675.186814625282 + }, + { + "decode_latency_s": 0.06068708070961293, + "kv_live_bytes": 462274560, + "latency_s": 1.9427944589406252, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 197754880, + "prompt_kind": "short", + "t_relative_s": 5680.196896083187 + }, + { + "decode_latency_s": 0.0688575182139175, + "kv_live_bytes": 462274560, + "latency_s": 2.206577999982983, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 103628800, + "prompt_kind": "long", + "t_relative_s": 5685.206950458232 + }, + { + "decode_latency_s": 0.06079288934415672, + "kv_live_bytes": 462274560, + "latency_s": 1.9462702088057995, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 97419264, + "prompt_kind": "short", + "t_relative_s": 5690.217053417116 + }, + { + "decode_latency_s": 0.06762899218301754, + "kv_live_bytes": 462274560, + "latency_s": 2.166463583242148, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71614464, + "prompt_kind": "long", + "t_relative_s": 5695.227102207951 + }, + { + "decode_latency_s": 0.06065555990790017, + "kv_live_bytes": 462274560, + "latency_s": 1.941780417226255, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72335360, + "prompt_kind": "short", + "t_relative_s": 5700.237167542335 + }, + { + "decode_latency_s": 0.061032920581055805, + "kv_live_bytes": 462274560, + "latency_s": 1.954207916278392, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72466432, + "prompt_kind": "long", + "t_relative_s": 5705.24720324995 + }, + { + "decode_latency_s": 0.06044326821574941, + "kv_live_bytes": 462274560, + "latency_s": 1.9351019589230418, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73089024, + "prompt_kind": "short", + "t_relative_s": 5710.251912042033 + }, + { + "decode_latency_s": 0.06809791925479658, + "kv_live_bytes": 462274560, + "latency_s": 2.1809624172747135, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73170944, + "prompt_kind": "long", + "t_relative_s": 5715.261976667214 + }, + { + "decode_latency_s": 0.06050046875316184, + "kv_live_bytes": 462274560, + "latency_s": 1.9368598330765963, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73416704, + "prompt_kind": "short", + "t_relative_s": 5720.272041375283 + }, + { + "decode_latency_s": 0.06851379034924321, + "kv_live_bytes": 462274560, + "latency_s": 2.19506649998948, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72777728, + "prompt_kind": "long", + "t_relative_s": 5725.282108208165 + }, + { + "decode_latency_s": 0.06628958853252698, + "kv_live_bytes": 462274560, + "latency_s": 2.1234282911755145, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "short", + "t_relative_s": 5730.292178750038 + }, + { + "decode_latency_s": 0.0684919101477135, + "kv_live_bytes": 462274560, + "latency_s": 2.1941080410033464, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73498624, + "prompt_kind": "long", + "t_relative_s": 5735.302236041985 + }, + { + "decode_latency_s": 0.060270656264037825, + "kv_live_bytes": 462274560, + "latency_s": 1.9295859588310122, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74055680, + "prompt_kind": "short", + "t_relative_s": 5740.312309875153 + }, + { + "decode_latency_s": 0.06804388150339946, + "kv_live_bytes": 462274560, + "latency_s": 2.1795366248115897, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73269248, + "prompt_kind": "long", + "t_relative_s": 5745.322390458081 + }, + { + "decode_latency_s": 0.06059109764464665, + "kv_live_bytes": 462274560, + "latency_s": 1.9397494997829199, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73515008, + "prompt_kind": "short", + "t_relative_s": 5750.332448666915 + }, + { + "decode_latency_s": 0.06674873827432748, + "kv_live_bytes": 462274560, + "latency_s": 2.1379295829683542, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73678848, + "prompt_kind": "long", + "t_relative_s": 5755.342515999917 + }, + { + "decode_latency_s": 0.06601769920962397, + "kv_live_bytes": 462274560, + "latency_s": 2.114217000082135, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "short", + "t_relative_s": 5760.347380542196 + }, + { + "decode_latency_s": 0.06801001430721954, + "kv_live_bytes": 462274560, + "latency_s": 2.1791731668636203, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72515584, + "prompt_kind": "long", + "t_relative_s": 5765.357428708114 + }, + { + "decode_latency_s": 0.060416535154217854, + "kv_live_bytes": 462274560, + "latency_s": 1.934269833844155, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73072640, + "prompt_kind": "short", + "t_relative_s": 5770.367504124995 + }, + { + "decode_latency_s": 0.06762728774629068, + "kv_live_bytes": 462274560, + "latency_s": 2.1657128748483956, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71663616, + "prompt_kind": "long", + "t_relative_s": 5775.377575207967 + }, + { + "decode_latency_s": 0.060371364568709396, + "kv_live_bytes": 462274560, + "latency_s": 1.9328658329322934, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71827456, + "prompt_kind": "short", + "t_relative_s": 5780.387626416981 + }, + { + "decode_latency_s": 0.06809502863325179, + "kv_live_bytes": 462274560, + "latency_s": 2.1810887921601534, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "long", + "t_relative_s": 5785.3977007921785 + }, + { + "decode_latency_s": 0.0672395481960848, + "kv_live_bytes": 462274560, + "latency_s": 2.1535207917913795, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73203712, + "prompt_kind": "short", + "t_relative_s": 5790.407749083359 + }, + { + "decode_latency_s": 0.06824851952842437, + "kv_live_bytes": 462274560, + "latency_s": 2.186586208175868, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73236480, + "prompt_kind": "long", + "t_relative_s": 5795.417805583216 + }, + { + "decode_latency_s": 0.06034682680910919, + "kv_live_bytes": 462274560, + "latency_s": 1.9318410828709602, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73891840, + "prompt_kind": "short", + "t_relative_s": 5800.427868375089 + }, + { + "decode_latency_s": 0.0682612330710981, + "kv_live_bytes": 462274560, + "latency_s": 2.186104209162295, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73728000, + "prompt_kind": "long", + "t_relative_s": 5805.437949708197 + }, + { + "decode_latency_s": 0.060392700528609566, + "kv_live_bytes": 462274560, + "latency_s": 1.9335027909837663, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73236480, + "prompt_kind": "short", + "t_relative_s": 5810.447997292038 + }, + { + "decode_latency_s": 0.06798155469004996, + "kv_live_bytes": 462274560, + "latency_s": 2.177728542126715, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73777152, + "prompt_kind": "long", + "t_relative_s": 5815.458098417148 + }, + { + "decode_latency_s": 0.06682088281377219, + "kv_live_bytes": 462274560, + "latency_s": 2.139820707961917, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73891840, + "prompt_kind": "short", + "t_relative_s": 5820.468149292283 + }, + { + "decode_latency_s": 0.06831002475519199, + "kv_live_bytes": 462274560, + "latency_s": 2.188266792334616, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73383936, + "prompt_kind": "long", + "t_relative_s": 5825.4781993329525 + }, + { + "decode_latency_s": 0.060364595061400905, + "kv_live_bytes": 462274560, + "latency_s": 1.9325463329441845, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73940992, + "prompt_kind": "short", + "t_relative_s": 5830.4882920002565 + }, + { + "decode_latency_s": 0.06796566405682825, + "kv_live_bytes": 462274560, + "latency_s": 2.176494542043656, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72712192, + "prompt_kind": "long", + "t_relative_s": 5835.498366250191 + }, + { + "decode_latency_s": 0.060275053401710466, + "kv_live_bytes": 462274560, + "latency_s": 1.9297597911208868, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72957952, + "prompt_kind": "short", + "t_relative_s": 5840.508407500107 + }, + { + "decode_latency_s": 0.06775046484835912, + "kv_live_bytes": 462274560, + "latency_s": 2.1702853329479694, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73531392, + "prompt_kind": "long", + "t_relative_s": 5845.518484292086 + }, + { + "decode_latency_s": 0.06059787760023028, + "kv_live_bytes": 462274560, + "latency_s": 1.9399456670507789, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74072064, + "prompt_kind": "short", + "t_relative_s": 5850.528582167346 + }, + { + "decode_latency_s": 0.060538467470905744, + "kv_live_bytes": 462274560, + "latency_s": 1.938066417351365, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74104832, + "prompt_kind": "long", + "t_relative_s": 5855.538634375203 + }, + { + "decode_latency_s": 0.060385523436707444, + "kv_live_bytes": 462274560, + "latency_s": 1.9331875410862267, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74727424, + "prompt_kind": "short", + "t_relative_s": 5860.548719750252 + }, + { + "decode_latency_s": 0.06076521356590092, + "kv_live_bytes": 462274560, + "latency_s": 1.9455079999752343, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70320128, + "prompt_kind": "long", + "t_relative_s": 5865.558846917003 + }, + { + "decode_latency_s": 0.060465204442152753, + "kv_live_bytes": 462274560, + "latency_s": 1.9358712090179324, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69746688, + "prompt_kind": "short", + "t_relative_s": 5870.568903333042 + }, + { + "decode_latency_s": 0.06787632293708157, + "kv_live_bytes": 462274560, + "latency_s": 2.1741129588335752, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73580544, + "prompt_kind": "long", + "t_relative_s": 5875.578954375349 + }, + { + "decode_latency_s": 0.060380695314961486, + "kv_live_bytes": 462274560, + "latency_s": 1.9331278330646455, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73957376, + "prompt_kind": "short", + "t_relative_s": 5880.589056707919 + }, + { + "decode_latency_s": 0.0679703646019334, + "kv_live_bytes": 462274560, + "latency_s": 2.1772683328017592, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73728000, + "prompt_kind": "long", + "t_relative_s": 5885.599141791929 + }, + { + "decode_latency_s": 0.060549934903974645, + "kv_live_bytes": 462274560, + "latency_s": 1.9385314169339836, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74350592, + "prompt_kind": "short", + "t_relative_s": 5890.60918550007 + }, + { + "decode_latency_s": 0.0688144817831926, + "kv_live_bytes": 462274560, + "latency_s": 2.20452345861122, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74383360, + "prompt_kind": "long", + "t_relative_s": 5895.619269375224 + }, + { + "decode_latency_s": 0.060603151068789884, + "kv_live_bytes": 462274560, + "latency_s": 1.9400901249609888, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74514432, + "prompt_kind": "short", + "t_relative_s": 5900.623227042146 + }, + { + "decode_latency_s": 0.06049291665840428, + "kv_live_bytes": 462274560, + "latency_s": 1.936766792088747, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74579968, + "prompt_kind": "long", + "t_relative_s": 5905.628472167067 + }, + { + "decode_latency_s": 0.06033000780735165, + "kv_live_bytes": 462274560, + "latency_s": 1.9314641249366105, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74612736, + "prompt_kind": "short", + "t_relative_s": 5910.638543667272 + }, + { + "decode_latency_s": 0.06827424609218724, + "kv_live_bytes": 462274560, + "latency_s": 2.186509625054896, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74661888, + "prompt_kind": "long", + "t_relative_s": 5915.64862750005 + }, + { + "decode_latency_s": 0.06074355596501846, + "kv_live_bytes": 462274560, + "latency_s": 1.9447232079692185, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75235328, + "prompt_kind": "short", + "t_relative_s": 5920.658676417079 + }, + { + "decode_latency_s": 0.06814580209902488, + "kv_live_bytes": 462274560, + "latency_s": 2.1831902088597417, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75104256, + "prompt_kind": "long", + "t_relative_s": 5925.668749208096 + }, + { + "decode_latency_s": 0.06069794272480067, + "kv_live_bytes": 462274560, + "latency_s": 1.9431522912345827, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75317248, + "prompt_kind": "short", + "t_relative_s": 5930.67884333292 + }, + { + "decode_latency_s": 0.06757323700003326, + "kv_live_bytes": 462274560, + "latency_s": 2.164455291815102, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74825728, + "prompt_kind": "long", + "t_relative_s": 5935.68890075013 + }, + { + "decode_latency_s": 0.06034611980430782, + "kv_live_bytes": 462274560, + "latency_s": 1.9319432084448636, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75497472, + "prompt_kind": "short", + "t_relative_s": 5940.698984167073 + }, + { + "decode_latency_s": 0.06804254556482192, + "kv_live_bytes": 462274560, + "latency_s": 2.1789476671256125, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75464704, + "prompt_kind": "long", + "t_relative_s": 5945.709054875188 + }, + { + "decode_latency_s": 0.06058230208873283, + "kv_live_bytes": 462274560, + "latency_s": 1.93951016664505, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76021760, + "prompt_kind": "short", + "t_relative_s": 5950.716877458151 + }, + { + "decode_latency_s": 0.06807538410066627, + "kv_live_bytes": 462274560, + "latency_s": 2.180792292114347, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75759616, + "prompt_kind": "long", + "t_relative_s": 5955.726962625049 + }, + { + "decode_latency_s": 0.060526542976731434, + "kv_live_bytes": 462274560, + "latency_s": 1.9376769163645804, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75841536, + "prompt_kind": "short", + "t_relative_s": 5960.737068958115 + }, + { + "decode_latency_s": 0.06760344789654482, + "kv_live_bytes": 462274560, + "latency_s": 2.165592874865979, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75890688, + "prompt_kind": "long", + "t_relative_s": 5965.739824749995 + }, + { + "decode_latency_s": 0.06034560938132927, + "kv_live_bytes": 462274560, + "latency_s": 1.931951165664941, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76496896, + "prompt_kind": "short", + "t_relative_s": 5970.749919292051 + }, + { + "decode_latency_s": 0.0683147942472715, + "kv_live_bytes": 462274560, + "latency_s": 2.188046208117157, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76447744, + "prompt_kind": "long", + "t_relative_s": 5975.759996917099 + }, + { + "decode_latency_s": 0.06341628514928743, + "kv_live_bytes": 462274560, + "latency_s": 2.0303834578953683, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67698688, + "prompt_kind": "short", + "t_relative_s": 5980.770063625183 + }, + { + "decode_latency_s": 0.08076485805213451, + "kv_live_bytes": 462274560, + "latency_s": 2.585367249790579, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66699264, + "prompt_kind": "long", + "t_relative_s": 5985.780115167145 + }, + { + "decode_latency_s": 0.06036216275242623, + "kv_live_bytes": 462274560, + "latency_s": 1.9324978329241276, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67485696, + "prompt_kind": "short", + "t_relative_s": 5992.892627499998 + }, + { + "decode_latency_s": 0.06811161850055214, + "kv_live_bytes": 462274560, + "latency_s": 2.1820076247677207, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69763072, + "prompt_kind": "long", + "t_relative_s": 5997.902697792277 + }, + { + "decode_latency_s": 0.06673798436531797, + "kv_live_bytes": 462274560, + "latency_s": 2.1374402083456516, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70402048, + "prompt_kind": "short", + "t_relative_s": 6002.912769292016 + }, + { + "decode_latency_s": 0.06792404559382703, + "kv_live_bytes": 462274560, + "latency_s": 2.1758832498453557, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70893568, + "prompt_kind": "long", + "t_relative_s": 6007.922806375194 + }, + { + "decode_latency_s": 0.060282118487521075, + "kv_live_bytes": 462274560, + "latency_s": 1.9299999168142676, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71450624, + "prompt_kind": "short", + "t_relative_s": 6012.932914667297 + }, + { + "decode_latency_s": 0.06838226821855642, + "kv_live_bytes": 462274560, + "latency_s": 2.190319959074259, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71335936, + "prompt_kind": "long", + "t_relative_s": 6017.943001749925 + }, + { + "decode_latency_s": 0.06106709374580532, + "kv_live_bytes": 462274560, + "latency_s": 1.9550232919864357, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70975488, + "prompt_kind": "short", + "t_relative_s": 6022.953069167212 + }, + { + "decode_latency_s": 0.06816423175041564, + "kv_live_bytes": 462274560, + "latency_s": 2.1832820419222116, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70959104, + "prompt_kind": "long", + "t_relative_s": 6027.9631459172815 + }, + { + "decode_latency_s": 0.06094574868620839, + "kv_live_bytes": 462274560, + "latency_s": 1.9510905831120908, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71565312, + "prompt_kind": "short", + "t_relative_s": 6032.973260750063 + }, + { + "decode_latency_s": 0.06035583330958616, + "kv_live_bytes": 462274560, + "latency_s": 1.9321801252663136, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71581696, + "prompt_kind": "long", + "t_relative_s": 6037.9833146249875 + }, + { + "decode_latency_s": 0.06076848309021443, + "kv_live_bytes": 462274560, + "latency_s": 1.9454345419071615, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72187904, + "prompt_kind": "short", + "t_relative_s": 6042.993411000352 + }, + { + "decode_latency_s": 0.06811595443286933, + "kv_live_bytes": 462274560, + "latency_s": 2.181398832704872, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72220672, + "prompt_kind": "long", + "t_relative_s": 6048.003496792167 + }, + { + "decode_latency_s": 0.060632532549789175, + "kv_live_bytes": 462274560, + "latency_s": 1.9415180408395827, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72335360, + "prompt_kind": "short", + "t_relative_s": 6053.0135516249575 + }, + { + "decode_latency_s": 0.06759824868640862, + "kv_live_bytes": 462274560, + "latency_s": 2.1656367918476462, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72368128, + "prompt_kind": "long", + "t_relative_s": 6058.0236294581555 + }, + { + "decode_latency_s": 0.060611509121372364, + "kv_live_bytes": 462274560, + "latency_s": 1.9403555416502059, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72908800, + "prompt_kind": "short", + "t_relative_s": 6063.033719625324 + }, + { + "decode_latency_s": 0.06792745574784931, + "kv_live_bytes": 462274560, + "latency_s": 2.1760287918150425, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72974336, + "prompt_kind": "long", + "t_relative_s": 6068.043777292129 + }, + { + "decode_latency_s": 0.06044243881478906, + "kv_live_bytes": 462274560, + "latency_s": 1.9350382080301642, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72925184, + "prompt_kind": "short", + "t_relative_s": 6073.053849332966 + }, + { + "decode_latency_s": 0.06831476301886141, + "kv_live_bytes": 462274560, + "latency_s": 2.1877139159478247, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72220672, + "prompt_kind": "long", + "t_relative_s": 6078.06391841732 + }, + { + "decode_latency_s": 0.06052056509361137, + "kv_live_bytes": 462274560, + "latency_s": 1.9374940837733448, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72335360, + "prompt_kind": "short", + "t_relative_s": 6083.073969041929 + }, + { + "decode_latency_s": 0.06790236981760245, + "kv_live_bytes": 462274560, + "latency_s": 2.1754929996095598, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72450048, + "prompt_kind": "long", + "t_relative_s": 6088.084046707954 + }, + { + "decode_latency_s": 0.06055896746693179, + "kv_live_bytes": 462274560, + "latency_s": 1.9387224996462464, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72990720, + "prompt_kind": "short", + "t_relative_s": 6093.0941585423425 + }, + { + "decode_latency_s": 0.0678413801942952, + "kv_live_bytes": 462274560, + "latency_s": 2.1732647912576795, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73072640, + "prompt_kind": "long", + "t_relative_s": 6098.104213042185 + }, + { + "decode_latency_s": 0.06066399090923369, + "kv_live_bytes": 462274560, + "latency_s": 1.9421747080050409, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73646080, + "prompt_kind": "short", + "t_relative_s": 6103.114293208346 + }, + { + "decode_latency_s": 0.06784712240914814, + "kv_live_bytes": 462274560, + "latency_s": 2.1726531670428813, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72728576, + "prompt_kind": "long", + "t_relative_s": 6108.124360083137 + }, + { + "decode_latency_s": 0.06692025260417722, + "kv_live_bytes": 462274560, + "latency_s": 2.1431248327717185, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72957952, + "prompt_kind": "short", + "t_relative_s": 6113.134405583143 + }, + { + "decode_latency_s": 0.06778684115852229, + "kv_live_bytes": 462274560, + "latency_s": 2.1720040407963097, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73089024, + "prompt_kind": "long", + "t_relative_s": 6118.14450333314 + }, + { + "decode_latency_s": 0.06048155990720261, + "kv_live_bytes": 462274560, + "latency_s": 1.936155208852142, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73646080, + "prompt_kind": "short", + "t_relative_s": 6123.15460345801 + }, + { + "decode_latency_s": 0.06795627866813447, + "kv_live_bytes": 462274560, + "latency_s": 2.17698224959895, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73728000, + "prompt_kind": "long", + "t_relative_s": 6128.164652958047 + }, + { + "decode_latency_s": 0.06027522656950168, + "kv_live_bytes": 462274560, + "latency_s": 1.9296477087773383, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74268672, + "prompt_kind": "short", + "t_relative_s": 6133.1747208749875 + }, + { + "decode_latency_s": 0.06053515493113082, + "kv_live_bytes": 462274560, + "latency_s": 1.9380102083086967, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73072640, + "prompt_kind": "long", + "t_relative_s": 6138.184795042034 + }, + { + "decode_latency_s": 0.060267312510404736, + "kv_live_bytes": 462274560, + "latency_s": 1.9295053747482598, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73170944, + "prompt_kind": "short", + "t_relative_s": 6143.1948533752 + }, + { + "decode_latency_s": 0.06771073827985674, + "kv_live_bytes": 462274560, + "latency_s": 2.1690715420991182, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73711616, + "prompt_kind": "long", + "t_relative_s": 6148.204938125331 + }, + { + "decode_latency_s": 0.0604764231829904, + "kv_live_bytes": 462274560, + "latency_s": 1.9360253745689988, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73711616, + "prompt_kind": "short", + "t_relative_s": 6153.2150179580785 + }, + { + "decode_latency_s": 0.06802722266002093, + "kv_live_bytes": 462274560, + "latency_s": 2.1793033331632614, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73826304, + "prompt_kind": "long", + "t_relative_s": 6158.225099042058 + }, + { + "decode_latency_s": 0.060613296867813915, + "kv_live_bytes": 462274560, + "latency_s": 1.940498541109264, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74366976, + "prompt_kind": "short", + "t_relative_s": 6163.235182708129 + }, + { + "decode_latency_s": 0.06777385156601667, + "kv_live_bytes": 462274560, + "latency_s": 2.170665374957025, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74137600, + "prompt_kind": "long", + "t_relative_s": 6168.245259958319 + }, + { + "decode_latency_s": 0.06036099868651945, + "kv_live_bytes": 462274560, + "latency_s": 1.9325249157845974, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74203136, + "prompt_kind": "short", + "t_relative_s": 6173.255309500266 + }, + { + "decode_latency_s": 0.06025839973881375, + "kv_live_bytes": 462274560, + "latency_s": 1.9291702499613166, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74268672, + "prompt_kind": "long", + "t_relative_s": 6178.265390167013 + }, + { + "decode_latency_s": 0.06051263150584418, + "kv_live_bytes": 462274560, + "latency_s": 1.9371860423125327, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74842112, + "prompt_kind": "short", + "t_relative_s": 6183.2754810419865 + }, + { + "decode_latency_s": 0.06754006378469057, + "kv_live_bytes": 462274560, + "latency_s": 2.163632749579847, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72876032, + "prompt_kind": "long", + "t_relative_s": 6188.285534667317 + }, + { + "decode_latency_s": 0.06060721352696419, + "kv_live_bytes": 462274560, + "latency_s": 1.9403336672112346, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73515008, + "prompt_kind": "short", + "t_relative_s": 6193.295580583159 + }, + { + "decode_latency_s": 0.06815589976031333, + "kv_live_bytes": 462274560, + "latency_s": 2.183439166750759, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73695232, + "prompt_kind": "long", + "t_relative_s": 6198.30564900022 + }, + { + "decode_latency_s": 0.060493255223263986, + "kv_live_bytes": 462274560, + "latency_s": 1.9365654583089054, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73105408, + "prompt_kind": "short", + "t_relative_s": 6203.315735083073 + }, + { + "decode_latency_s": 0.06782836066849995, + "kv_live_bytes": 462274560, + "latency_s": 2.1725026671774685, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73089024, + "prompt_kind": "long", + "t_relative_s": 6208.325790833216 + }, + { + "decode_latency_s": 0.060291213550954126, + "kv_live_bytes": 462274560, + "latency_s": 1.9302142914384604, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73089024, + "prompt_kind": "short", + "t_relative_s": 6213.335861042142 + }, + { + "decode_latency_s": 0.06748441405943595, + "kv_live_bytes": 462274560, + "latency_s": 2.1618750002235174, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73547776, + "prompt_kind": "long", + "t_relative_s": 6218.3459210833535 + }, + { + "decode_latency_s": 0.06087006772577297, + "kv_live_bytes": 462274560, + "latency_s": 1.9485877500846982, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74121216, + "prompt_kind": "short", + "t_relative_s": 6223.356025542133 + }, + { + "decode_latency_s": 0.06813049738411792, + "kv_live_bytes": 462274560, + "latency_s": 2.181814916897565, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73629696, + "prompt_kind": "long", + "t_relative_s": 6228.36370645836 + }, + { + "decode_latency_s": 0.06024862889898941, + "kv_live_bytes": 462274560, + "latency_s": 1.9288696660660207, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73515008, + "prompt_kind": "short", + "t_relative_s": 6233.373754417058 + }, + { + "decode_latency_s": 0.06803939453675412, + "kv_live_bytes": 462274560, + "latency_s": 2.1796649578027427, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73728000, + "prompt_kind": "long", + "t_relative_s": 6238.383829625323 + }, + { + "decode_latency_s": 0.060686111959512345, + "kv_live_bytes": 462274560, + "latency_s": 1.9427461251616478, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74350592, + "prompt_kind": "short", + "t_relative_s": 6243.3938747919165 + }, + { + "decode_latency_s": 0.06733484765572939, + "kv_live_bytes": 462274560, + "latency_s": 2.157046042382717, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74465280, + "prompt_kind": "long", + "t_relative_s": 6248.403943000361 + }, + { + "decode_latency_s": 0.06065216015849728, + "kv_live_bytes": 462274560, + "latency_s": 1.9416377088055015, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75005952, + "prompt_kind": "short", + "t_relative_s": 6253.414016417228 + }, + { + "decode_latency_s": 0.06800465626292862, + "kv_live_bytes": 462274560, + "latency_s": 2.1779562919400632, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74989568, + "prompt_kind": "long", + "t_relative_s": 6258.414721542038 + }, + { + "decode_latency_s": 0.060390023441868834, + "kv_live_bytes": 462274560, + "latency_s": 1.9333529160358012, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74465280, + "prompt_kind": "short", + "t_relative_s": 6263.42479391722 + }, + { + "decode_latency_s": 0.06853612631675787, + "kv_live_bytes": 462274560, + "latency_s": 2.19510304229334, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74498048, + "prompt_kind": "long", + "t_relative_s": 6268.434871792328 + }, + { + "decode_latency_s": 0.06057882292952854, + "kv_live_bytes": 462274560, + "latency_s": 1.9394039581529796, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75022336, + "prompt_kind": "short", + "t_relative_s": 6273.444937208202 + }, + { + "decode_latency_s": 0.06068117187533062, + "kv_live_bytes": 462274560, + "latency_s": 1.942764916922897, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75104256, + "prompt_kind": "long", + "t_relative_s": 6278.455013333354 + }, + { + "decode_latency_s": 0.06255679296737071, + "kv_live_bytes": 462274560, + "latency_s": 2.0035589998587966, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75743232, + "prompt_kind": "short", + "t_relative_s": 6283.465098999906 + }, + { + "decode_latency_s": 0.06812428125704173, + "kv_live_bytes": 462274560, + "latency_s": 2.182148292195052, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75710464, + "prompt_kind": "long", + "t_relative_s": 6288.475154333282 + }, + { + "decode_latency_s": 0.060262059909291565, + "kv_live_bytes": 462274560, + "latency_s": 1.9292968749068677, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75137024, + "prompt_kind": "short", + "t_relative_s": 6293.485227708239 + }, + { + "decode_latency_s": 0.06825944922456983, + "kv_live_bytes": 462274560, + "latency_s": 2.18602008279413, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75284480, + "prompt_kind": "long", + "t_relative_s": 6298.495320708025 + }, + { + "decode_latency_s": 0.06048230468877591, + "kv_live_bytes": 462274560, + "latency_s": 1.9363155830651522, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75333632, + "prompt_kind": "short", + "t_relative_s": 6303.504505250137 + }, + { + "decode_latency_s": 0.06780761849950068, + "kv_live_bytes": 462274560, + "latency_s": 2.1720944168046117, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78397440, + "prompt_kind": "long", + "t_relative_s": 6308.514597374946 + }, + { + "decode_latency_s": 0.06054704816779122, + "kv_live_bytes": 462274560, + "latency_s": 1.9382908749394119, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78970880, + "prompt_kind": "short", + "t_relative_s": 6313.524680292234 + }, + { + "decode_latency_s": 0.06788160806172527, + "kv_live_bytes": 462274560, + "latency_s": 2.1731307078152895, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72056832, + "prompt_kind": "long", + "t_relative_s": 6318.534729250241 + }, + { + "decode_latency_s": 0.06046995572978631, + "kv_live_bytes": 462274560, + "latency_s": 1.9358299998566508, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73547776, + "prompt_kind": "short", + "t_relative_s": 6323.699963625055 + }, + { + "decode_latency_s": 0.06798473828530405, + "kv_live_bytes": 462274560, + "latency_s": 2.1781647498719394, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73564160, + "prompt_kind": "long", + "t_relative_s": 6328.71001070831 + }, + { + "decode_latency_s": 0.06036605077679269, + "kv_live_bytes": 462274560, + "latency_s": 1.9325609160587192, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73990144, + "prompt_kind": "short", + "t_relative_s": 6333.720090833027 + }, + { + "decode_latency_s": 0.06793391796236392, + "kv_live_bytes": 462274560, + "latency_s": 2.176781083922833, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74170368, + "prompt_kind": "long", + "t_relative_s": 6338.730148083065 + }, + { + "decode_latency_s": 0.06086500649689697, + "kv_live_bytes": 462274560, + "latency_s": 1.9485772079788148, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74547200, + "prompt_kind": "short", + "t_relative_s": 6343.740207375027 + }, + { + "decode_latency_s": 0.06808588672720362, + "kv_live_bytes": 462274560, + "latency_s": 2.1806753748096526, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73498624, + "prompt_kind": "long", + "t_relative_s": 6348.750249583274 + }, + { + "decode_latency_s": 0.06067173175688367, + "kv_live_bytes": 462274560, + "latency_s": 1.9424148327670991, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73351168, + "prompt_kind": "short", + "t_relative_s": 6353.760318083223 + }, + { + "decode_latency_s": 0.06817729557224084, + "kv_live_bytes": 462274560, + "latency_s": 2.1842388748191297, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73678848, + "prompt_kind": "long", + "t_relative_s": 6358.77039545821 + }, + { + "decode_latency_s": 0.06070319400168955, + "kv_live_bytes": 462274560, + "latency_s": 1.9433535831049085, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74317824, + "prompt_kind": "short", + "t_relative_s": 6363.780446417164 + }, + { + "decode_latency_s": 0.06715558202995453, + "kv_live_bytes": 462274560, + "latency_s": 2.1513521657325327, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74383360, + "prompt_kind": "long", + "t_relative_s": 6368.788684458006 + }, + { + "decode_latency_s": 0.060818348938482814, + "kv_live_bytes": 462274560, + "latency_s": 1.946981250308454, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74891264, + "prompt_kind": "short", + "t_relative_s": 6373.798754208256 + }, + { + "decode_latency_s": 0.06768316144007258, + "kv_live_bytes": 462274560, + "latency_s": 2.167874749749899, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74907648, + "prompt_kind": "long", + "t_relative_s": 6378.808810375165 + }, + { + "decode_latency_s": 0.06018417968880385, + "kv_live_bytes": 462274560, + "latency_s": 1.9269143748097122, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73809920, + "prompt_kind": "short", + "t_relative_s": 6383.818900083192 + }, + { + "decode_latency_s": 0.06798205077939201, + "kv_live_bytes": 462274560, + "latency_s": 2.177006166893989, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72728576, + "prompt_kind": "long", + "t_relative_s": 6388.8289670832455 + }, + { + "decode_latency_s": 0.06036805600160733, + "kv_live_bytes": 462274560, + "latency_s": 1.9327046247199178, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73302016, + "prompt_kind": "short", + "t_relative_s": 6393.836654208135 + }, + { + "decode_latency_s": 0.06788227996730711, + "kv_live_bytes": 462274560, + "latency_s": 2.1744661671109498, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73334784, + "prompt_kind": "long", + "t_relative_s": 6398.846722875256 + }, + { + "decode_latency_s": 0.06057142840290908, + "kv_live_bytes": 462274560, + "latency_s": 1.939124541822821, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73891840, + "prompt_kind": "short", + "t_relative_s": 6403.856791708153 + }, + { + "decode_latency_s": 0.06801296224875841, + "kv_live_bytes": 462274560, + "latency_s": 2.1787322079762816, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73695232, + "prompt_kind": "long", + "t_relative_s": 6408.866839167196 + }, + { + "decode_latency_s": 0.060300062512396835, + "kv_live_bytes": 462274560, + "latency_s": 1.930606042034924, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72499200, + "prompt_kind": "short", + "t_relative_s": 6413.8769162921235 + }, + { + "decode_latency_s": 0.06821711717930157, + "kv_live_bytes": 462274560, + "latency_s": 2.184833041857928, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73121792, + "prompt_kind": "long", + "t_relative_s": 6418.886999083217 + }, + { + "decode_latency_s": 0.08154847002879251, + "kv_live_bytes": 462274560, + "latency_s": 2.610342166852206, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68763648, + "prompt_kind": "short", + "t_relative_s": 6423.897054166999 + }, + { + "decode_latency_s": 0.06793871225090697, + "kv_live_bytes": 462274560, + "latency_s": 2.1765197906643152, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69419008, + "prompt_kind": "long", + "t_relative_s": 6431.324171083048 + }, + { + "decode_latency_s": 0.061036606784909964, + "kv_live_bytes": 462274560, + "latency_s": 1.9540103748440742, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69910528, + "prompt_kind": "short", + "t_relative_s": 6436.334249458276 + }, + { + "decode_latency_s": 0.06821274608955719, + "kv_live_bytes": 462274560, + "latency_s": 2.1849205000326037, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69304320, + "prompt_kind": "long", + "t_relative_s": 6441.344295124989 + }, + { + "decode_latency_s": 0.0602965325670084, + "kv_live_bytes": 462274560, + "latency_s": 1.9303709589876235, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69304320, + "prompt_kind": "short", + "t_relative_s": 6446.3543520001695 + }, + { + "decode_latency_s": 0.06766182029969059, + "kv_live_bytes": 462274560, + "latency_s": 2.1672371667809784, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69517312, + "prompt_kind": "long", + "t_relative_s": 6451.364424250089 + }, + { + "decode_latency_s": 0.06095165364968125, + "kv_live_bytes": 462274560, + "latency_s": 1.9512707497924566, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70156288, + "prompt_kind": "short", + "t_relative_s": 6456.374465500005 + }, + { + "decode_latency_s": 0.06883889452728909, + "kv_live_bytes": 462274560, + "latency_s": 2.2052392079494894, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70172672, + "prompt_kind": "long", + "t_relative_s": 6461.3845397080295 + }, + { + "decode_latency_s": 0.0624699830514146, + "kv_live_bytes": 462274560, + "latency_s": 2.000049250200391, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69009408, + "prompt_kind": "short", + "t_relative_s": 6466.3946235422045 + }, + { + "decode_latency_s": 0.06749764452979434, + "kv_live_bytes": 462274560, + "latency_s": 2.162504583131522, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70041600, + "prompt_kind": "long", + "t_relative_s": 6471.404692667071 + }, + { + "decode_latency_s": 0.06701876825536601, + "kv_live_bytes": 462274560, + "latency_s": 2.1462977919727564, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70123520, + "prompt_kind": "short", + "t_relative_s": 6476.407824625261 + }, + { + "decode_latency_s": 0.06775108985311817, + "kv_live_bytes": 462274560, + "latency_s": 2.1703587910160422, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70090752, + "prompt_kind": "long", + "t_relative_s": 6481.417894625105 + }, + { + "decode_latency_s": 0.06061944141401909, + "kv_live_bytes": 462274560, + "latency_s": 1.9405785826966166, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70729728, + "prompt_kind": "short", + "t_relative_s": 6486.427964916918 + }, + { + "decode_latency_s": 0.06836201433907263, + "kv_live_bytes": 462274560, + "latency_s": 2.18916904181242, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70762496, + "prompt_kind": "long", + "t_relative_s": 6491.438044750132 + }, + { + "decode_latency_s": 0.060534480478963815, + "kv_live_bytes": 462274560, + "latency_s": 1.9379963329993188, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71368704, + "prompt_kind": "short", + "t_relative_s": 6496.448094916996 + }, + { + "decode_latency_s": 0.06819571483356413, + "kv_live_bytes": 462274560, + "latency_s": 2.184683416970074, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71532544, + "prompt_kind": "long", + "t_relative_s": 6501.458182542119 + }, + { + "decode_latency_s": 0.06080438803473953, + "kv_live_bytes": 462274560, + "latency_s": 1.9465087498538196, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71680000, + "prompt_kind": "short", + "t_relative_s": 6506.468278875109 + }, + { + "decode_latency_s": 0.06787971094308887, + "kv_live_bytes": 462274560, + "latency_s": 2.1744786659255624, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71712768, + "prompt_kind": "long", + "t_relative_s": 6511.478328708094 + }, + { + "decode_latency_s": 0.060536717472132295, + "kv_live_bytes": 462274560, + "latency_s": 1.938100166618824, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72286208, + "prompt_kind": "short", + "t_relative_s": 6516.488406707998 + }, + { + "decode_latency_s": 0.06811385415494442, + "kv_live_bytes": 462274560, + "latency_s": 2.1815312090329826, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72237056, + "prompt_kind": "long", + "t_relative_s": 6521.49850779213 + }, + { + "decode_latency_s": 0.06703488022321835, + "kv_live_bytes": 462274560, + "latency_s": 2.146816208027303, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72302592, + "prompt_kind": "short", + "t_relative_s": 6526.508558917325 + }, + { + "decode_latency_s": 0.0677392968791537, + "kv_live_bytes": 462274560, + "latency_s": 2.170055333059281, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72335360, + "prompt_kind": "long", + "t_relative_s": 6531.518643625081 + }, + { + "decode_latency_s": 0.060572498725377955, + "kv_live_bytes": 462274560, + "latency_s": 1.9391660001128912, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72417280, + "prompt_kind": "short", + "t_relative_s": 6536.528701208066 + }, + { + "decode_latency_s": 0.06731744010176044, + "kv_live_bytes": 462274560, + "latency_s": 2.1565941656008363, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72450048, + "prompt_kind": "long", + "t_relative_s": 6541.538754791953 + }, + { + "decode_latency_s": 0.06042759766569361, + "kv_live_bytes": 462274560, + "latency_s": 1.9345664996653795, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72957952, + "prompt_kind": "short", + "t_relative_s": 6546.548836458009 + }, + { + "decode_latency_s": 0.06040541795664467, + "kv_live_bytes": 462274560, + "latency_s": 1.9342653332278132, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73023488, + "prompt_kind": "long", + "t_relative_s": 6551.558926917147 + }, + { + "decode_latency_s": 0.06703950518567581, + "kv_live_bytes": 462274560, + "latency_s": 2.1467580827884376, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73039872, + "prompt_kind": "short", + "t_relative_s": 6556.568980542012 + }, + { + "decode_latency_s": 0.06812583463033661, + "kv_live_bytes": 462274560, + "latency_s": 2.1823657918721437, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73138176, + "prompt_kind": "long", + "t_relative_s": 6561.579065208323 + }, + { + "decode_latency_s": 0.060623891913564876, + "kv_live_bytes": 462274560, + "latency_s": 1.9407445839606225, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73203712, + "prompt_kind": "short", + "t_relative_s": 6566.589173083194 + }, + { + "decode_latency_s": 0.06810564712213818, + "kv_live_bytes": 462274560, + "latency_s": 2.1814750423654914, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73236480, + "prompt_kind": "long", + "t_relative_s": 6571.599230791908 + }, + { + "decode_latency_s": 0.06048735024523921, + "kv_live_bytes": 462274560, + "latency_s": 1.93651287490502, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73695232, + "prompt_kind": "short", + "t_relative_s": 6576.609318125062 + }, + { + "decode_latency_s": 0.06798610938130878, + "kv_live_bytes": 462274560, + "latency_s": 2.177558584138751, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73777152, + "prompt_kind": "long", + "t_relative_s": 6581.619408333208 + }, + { + "decode_latency_s": 0.06129327733651735, + "kv_live_bytes": 462274560, + "latency_s": 1.9623019164428115, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73908224, + "prompt_kind": "short", + "t_relative_s": 6586.629457667004 + }, + { + "decode_latency_s": 0.06940429037786089, + "kv_live_bytes": 462274560, + "latency_s": 2.2234986247494817, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73908224, + "prompt_kind": "long", + "t_relative_s": 6591.6395622082055 + }, + { + "decode_latency_s": 0.06692745834880043, + "kv_live_bytes": 462274560, + "latency_s": 2.14324662508443, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72859648, + "prompt_kind": "short", + "t_relative_s": 6596.646474624984 + }, + { + "decode_latency_s": 0.06802198306831997, + "kv_live_bytes": 462274560, + "latency_s": 2.1794455838389695, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73367552, + "prompt_kind": "long", + "t_relative_s": 6601.656512792222 + }, + { + "decode_latency_s": 0.06040773828863166, + "kv_live_bytes": 462274560, + "latency_s": 1.9339251657947898, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73908224, + "prompt_kind": "short", + "t_relative_s": 6606.666601125151 + }, + { + "decode_latency_s": 0.06813787890132517, + "kv_live_bytes": 462274560, + "latency_s": 2.182209166698158, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73695232, + "prompt_kind": "long", + "t_relative_s": 6611.676691625267 + }, + { + "decode_latency_s": 0.06148922396823764, + "kv_live_bytes": 462274560, + "latency_s": 1.96876479126513, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73465856, + "prompt_kind": "short", + "t_relative_s": 6616.686742166989 + }, + { + "decode_latency_s": 0.0675954283651663, + "kv_live_bytes": 462274560, + "latency_s": 2.1656034174375236, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74137600, + "prompt_kind": "long", + "t_relative_s": 6621.696828042157 + }, + { + "decode_latency_s": 0.06073211328475736, + "kv_live_bytes": 462274560, + "latency_s": 1.9442346249707043, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74268672, + "prompt_kind": "short", + "t_relative_s": 6626.706761542242 + }, + { + "decode_latency_s": 0.060474348938441835, + "kv_live_bytes": 462274560, + "latency_s": 1.9359965417534113, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73629696, + "prompt_kind": "long", + "t_relative_s": 6631.716810292099 + }, + { + "decode_latency_s": 0.06064325390616432, + "kv_live_bytes": 462274560, + "latency_s": 1.9414317500777543, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74186752, + "prompt_kind": "short", + "t_relative_s": 6636.726899583358 + }, + { + "decode_latency_s": 0.06903509503172245, + "kv_live_bytes": 462274560, + "latency_s": 2.2117362916469574, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72531968, + "prompt_kind": "long", + "t_relative_s": 6641.736986082979 + }, + { + "decode_latency_s": 0.06737803386931773, + "kv_live_bytes": 462274560, + "latency_s": 2.1578103750944138, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72810496, + "prompt_kind": "short", + "t_relative_s": 6646.747040250339 + }, + { + "decode_latency_s": 0.06713513018621597, + "kv_live_bytes": 462274560, + "latency_s": 2.1502767908386886, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73105408, + "prompt_kind": "long", + "t_relative_s": 6651.749470625073 + }, + { + "decode_latency_s": 0.06220977606426459, + "kv_live_bytes": 462274560, + "latency_s": 1.9915132499299943, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69861376, + "prompt_kind": "short", + "t_relative_s": 6656.754676583223 + }, + { + "decode_latency_s": 0.06788123046862893, + "kv_live_bytes": 462274560, + "latency_s": 2.1749926251359284, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70320128, + "prompt_kind": "long", + "t_relative_s": 6663.129346167203 + }, + { + "decode_latency_s": 0.060928015620447695, + "kv_live_bytes": 462274560, + "latency_s": 1.950544792227447, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70270976, + "prompt_kind": "short", + "t_relative_s": 6668.139440916944 + }, + { + "decode_latency_s": 0.06824966143176425, + "kv_live_bytes": 462274560, + "latency_s": 2.1860095830634236, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70860800, + "prompt_kind": "long", + "t_relative_s": 6673.149533083197 + }, + { + "decode_latency_s": 0.060839850251795724, + "kv_live_bytes": 462274560, + "latency_s": 1.947775417007506, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71434240, + "prompt_kind": "short", + "t_relative_s": 6678.1595918331295 + }, + { + "decode_latency_s": 0.06779638802981935, + "kv_live_bytes": 462274560, + "latency_s": 2.1719587077386677, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71483392, + "prompt_kind": "long", + "t_relative_s": 6683.169666000176 + }, + { + "decode_latency_s": 0.06054126824892592, + "kv_live_bytes": 462274560, + "latency_s": 1.9385538334026933, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71565312, + "prompt_kind": "short", + "t_relative_s": 6688.179741625208 + }, + { + "decode_latency_s": 0.06754533464845736, + "kv_live_bytes": 462274560, + "latency_s": 2.163694041315466, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70909952, + "prompt_kind": "long", + "t_relative_s": 6693.189790083095 + }, + { + "decode_latency_s": 0.06029063022288028, + "kv_live_bytes": 462274560, + "latency_s": 1.9301736671477556, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70877184, + "prompt_kind": "short", + "t_relative_s": 6698.199855083134 + }, + { + "decode_latency_s": 0.06811471484252252, + "kv_live_bytes": 462274560, + "latency_s": 2.1819858751259744, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71417856, + "prompt_kind": "long", + "t_relative_s": 6703.209932292346 + }, + { + "decode_latency_s": 0.06067425649962388, + "kv_live_bytes": 462274560, + "latency_s": 1.9424776253290474, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72040448, + "prompt_kind": "short", + "t_relative_s": 6708.219996583182 + }, + { + "decode_latency_s": 0.06831242446787655, + "kv_live_bytes": 462274560, + "latency_s": 2.1885259998962283, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71942144, + "prompt_kind": "long", + "t_relative_s": 6713.230069833342 + }, + { + "decode_latency_s": 0.06063889975484926, + "kv_live_bytes": 462274560, + "latency_s": 1.9412361672148108, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72105984, + "prompt_kind": "short", + "t_relative_s": 6718.240160082933 + }, + { + "decode_latency_s": 0.06782830206793733, + "kv_live_bytes": 462274560, + "latency_s": 2.1726888748817146, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71614464, + "prompt_kind": "long", + "t_relative_s": 6723.24338775035 + }, + { + "decode_latency_s": 0.06024668618920259, + "kv_live_bytes": 462274560, + "latency_s": 1.9288125843741, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72253440, + "prompt_kind": "short", + "t_relative_s": 6728.253468375187 + }, + { + "decode_latency_s": 0.06790773960528895, + "kv_live_bytes": 462274560, + "latency_s": 2.174685582984239, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72318976, + "prompt_kind": "long", + "t_relative_s": 6733.26354425028 + }, + { + "decode_latency_s": 0.060611463559325784, + "kv_live_bytes": 462274560, + "latency_s": 1.9404754168353975, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72826880, + "prompt_kind": "short", + "t_relative_s": 6738.2736004581675 + }, + { + "decode_latency_s": 0.06825083075091243, + "kv_live_bytes": 462274560, + "latency_s": 2.1864239582791924, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72908800, + "prompt_kind": "long", + "t_relative_s": 6743.283681124914 + }, + { + "decode_latency_s": 0.06054655209300108, + "kv_live_bytes": 462274560, + "latency_s": 1.938289541285485, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72974336, + "prompt_kind": "short", + "t_relative_s": 6748.293751250021 + }, + { + "decode_latency_s": 0.06054599872732069, + "kv_live_bytes": 462274560, + "latency_s": 1.9387172916904092, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73089024, + "prompt_kind": "long", + "t_relative_s": 6753.303796750028 + }, + { + "decode_latency_s": 0.060258484387304634, + "kv_live_bytes": 462274560, + "latency_s": 1.9291852079331875, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73039872, + "prompt_kind": "short", + "t_relative_s": 6758.313880708069 + }, + { + "decode_latency_s": 0.06862893099605571, + "kv_live_bytes": 462274560, + "latency_s": 2.198017292190343, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73089024, + "prompt_kind": "long", + "t_relative_s": 6763.323949958198 + }, + { + "decode_latency_s": 0.0604484935029177, + "kv_live_bytes": 462274560, + "latency_s": 1.9353173752315342, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73154560, + "prompt_kind": "short", + "t_relative_s": 6768.3340231669135 + }, + { + "decode_latency_s": 0.06782393230241723, + "kv_live_bytes": 462274560, + "latency_s": 2.1728618750348687, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73203712, + "prompt_kind": "long", + "t_relative_s": 6773.34410912497 + }, + { + "decode_latency_s": 0.060575592462555505, + "kv_live_bytes": 462274560, + "latency_s": 1.939256208948791, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73203712, + "prompt_kind": "short", + "t_relative_s": 6778.354189957958 + }, + { + "decode_latency_s": 0.06071783594961744, + "kv_live_bytes": 462274560, + "latency_s": 1.9445243342779577, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73285632, + "prompt_kind": "long", + "t_relative_s": 6783.364238083363 + }, + { + "decode_latency_s": 0.06023632813594304, + "kv_live_bytes": 462274560, + "latency_s": 1.9285018327645957, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73154560, + "prompt_kind": "short", + "t_relative_s": 6788.371253417339 + }, + { + "decode_latency_s": 0.06855343749339227, + "kv_live_bytes": 462274560, + "latency_s": 2.1957504581660032, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73285632, + "prompt_kind": "long", + "t_relative_s": 6793.381337624975 + }, + { + "decode_latency_s": 0.06042363931192085, + "kv_live_bytes": 462274560, + "latency_s": 1.9344112081453204, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73285632, + "prompt_kind": "short", + "t_relative_s": 6798.391404417343 + }, + { + "decode_latency_s": 0.06811639711668249, + "kv_live_bytes": 462274560, + "latency_s": 2.1821763748303056, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73416704, + "prompt_kind": "long", + "t_relative_s": 6803.401466458105 + }, + { + "decode_latency_s": 0.0604990768188145, + "kv_live_bytes": 462274560, + "latency_s": 1.9367217076942325, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "short", + "t_relative_s": 6808.411543125287 + }, + { + "decode_latency_s": 0.06803874087927397, + "kv_live_bytes": 462274560, + "latency_s": 2.1797041250392795, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73269248, + "prompt_kind": "long", + "t_relative_s": 6813.421601792332 + }, + { + "decode_latency_s": 0.06034891927265562, + "kv_live_bytes": 462274560, + "latency_s": 1.9324356662109494, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "short", + "t_relative_s": 6818.431636957917 + }, + { + "decode_latency_s": 0.0686035781254759, + "kv_live_bytes": 462274560, + "latency_s": 2.197694499976933, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73482240, + "prompt_kind": "long", + "t_relative_s": 6823.441719749942 + }, + { + "decode_latency_s": 0.0666248072811868, + "kv_live_bytes": 462274560, + "latency_s": 2.1336878747679293, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74039296, + "prompt_kind": "short", + "t_relative_s": 6828.451781624928 + }, + { + "decode_latency_s": 0.06784063021768816, + "kv_live_bytes": 462274560, + "latency_s": 2.173622709233314, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74039296, + "prompt_kind": "long", + "t_relative_s": 6833.46185025014 + }, + { + "decode_latency_s": 0.06051137109170668, + "kv_live_bytes": 462274560, + "latency_s": 1.9371552909724414, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74072064, + "prompt_kind": "short", + "t_relative_s": 6838.471927457955 + }, + { + "decode_latency_s": 0.06790853777783923, + "kv_live_bytes": 462274560, + "latency_s": 2.1747387498617172, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73957376, + "prompt_kind": "long", + "t_relative_s": 6843.481952458154 + }, + { + "decode_latency_s": 0.06035283334495034, + "kv_live_bytes": 462274560, + "latency_s": 1.9323892495594919, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73383936, + "prompt_kind": "short", + "t_relative_s": 6848.490829417016 + }, + { + "decode_latency_s": 0.06807419141114224, + "kv_live_bytes": 462274560, + "latency_s": 2.180523042101413, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74072064, + "prompt_kind": "long", + "t_relative_s": 6853.500909542199 + }, + { + "decode_latency_s": 0.06059661981998943, + "kv_live_bytes": 462274560, + "latency_s": 1.939896292053163, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74645504, + "prompt_kind": "short", + "t_relative_s": 6858.510989542119 + }, + { + "decode_latency_s": 0.06783013543463312, + "kv_live_bytes": 462274560, + "latency_s": 2.1729437075555325, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74727424, + "prompt_kind": "long", + "t_relative_s": 6863.521061208099 + }, + { + "decode_latency_s": 0.060814214855781756, + "kv_live_bytes": 462274560, + "latency_s": 1.9468059167265892, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74809344, + "prompt_kind": "short", + "t_relative_s": 6868.531142792199 + }, + { + "decode_latency_s": 0.06752500653965399, + "kv_live_bytes": 462274560, + "latency_s": 2.163085165899247, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74694656, + "prompt_kind": "long", + "t_relative_s": 6873.541190833319 + }, + { + "decode_latency_s": 0.0603371875040466, + "kv_live_bytes": 462274560, + "latency_s": 1.9316887920722365, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75317248, + "prompt_kind": "short", + "t_relative_s": 6878.5512460833415 + }, + { + "decode_latency_s": 0.06514560937648639, + "kv_live_bytes": 462274560, + "latency_s": 2.086497541051358, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75399168, + "prompt_kind": "long", + "t_relative_s": 6883.561333416961 + }, + { + "decode_latency_s": 0.060483555993414484, + "kv_live_bytes": 462274560, + "latency_s": 1.9362960420548916, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75300864, + "prompt_kind": "short", + "t_relative_s": 6888.571395708248 + }, + { + "decode_latency_s": 0.06402310419071, + "kv_live_bytes": 462274560, + "latency_s": 2.050088125281036, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71254016, + "prompt_kind": "long", + "t_relative_s": 6893.581503957976 + }, + { + "decode_latency_s": 0.06044693231524434, + "kv_live_bytes": 462274560, + "latency_s": 1.9351775832474232, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71729152, + "prompt_kind": "short", + "t_relative_s": 6899.448747625109 + }, + { + "decode_latency_s": 0.06803389193373732, + "kv_live_bytes": 462274560, + "latency_s": 2.179498916026205, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71909376, + "prompt_kind": "long", + "t_relative_s": 6904.458802875131 + }, + { + "decode_latency_s": 0.06094789321650751, + "kv_live_bytes": 462274560, + "latency_s": 1.9512512078508735, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72433664, + "prompt_kind": "short", + "t_relative_s": 6909.468900416978 + }, + { + "decode_latency_s": 0.06874694272119086, + "kv_live_bytes": 462274560, + "latency_s": 2.2020337502472103, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71204864, + "prompt_kind": "long", + "t_relative_s": 6914.478955667 + }, + { + "decode_latency_s": 0.060417270855396055, + "kv_live_bytes": 462274560, + "latency_s": 1.9342564581893384, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69681152, + "prompt_kind": "short", + "t_relative_s": 6919.489029542077 + }, + { + "decode_latency_s": 0.06935764584341086, + "kv_live_bytes": 462274560, + "latency_s": 2.221222249791026, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70877184, + "prompt_kind": "long", + "t_relative_s": 6924.49910487514 + }, + { + "decode_latency_s": 0.060618161427555606, + "kv_live_bytes": 462274560, + "latency_s": 1.9405664158985019, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71008256, + "prompt_kind": "short", + "t_relative_s": 6929.509203458205 + }, + { + "decode_latency_s": 0.06815244010067545, + "kv_live_bytes": 462274560, + "latency_s": 2.183843707665801, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71434240, + "prompt_kind": "long", + "t_relative_s": 6934.519253833219 + }, + { + "decode_latency_s": 0.06029174737341236, + "kv_live_bytes": 462274560, + "latency_s": 1.930234500207007, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72105984, + "prompt_kind": "short", + "t_relative_s": 6939.529284083284 + }, + { + "decode_latency_s": 0.0682766302197706, + "kv_live_bytes": 462274560, + "latency_s": 2.1874222080223262, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72286208, + "prompt_kind": "long", + "t_relative_s": 6944.539351500105 + }, + { + "decode_latency_s": 0.06057329296891112, + "kv_live_bytes": 462274560, + "latency_s": 1.939168291632086, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72843264, + "prompt_kind": "short", + "t_relative_s": 6949.549409583211 + }, + { + "decode_latency_s": 0.06710867577930912, + "kv_live_bytes": 462274560, + "latency_s": 2.149984208866954, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76300288, + "prompt_kind": "long", + "t_relative_s": 6954.559464416932 + }, + { + "decode_latency_s": 0.06676679555675946, + "kv_live_bytes": 462274560, + "latency_s": 2.138285582885146, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76365824, + "prompt_kind": "short", + "t_relative_s": 6959.56953795813 + }, + { + "decode_latency_s": 0.06840181117877364, + "kv_live_bytes": 462274560, + "latency_s": 2.1913007921539247, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74334208, + "prompt_kind": "long", + "t_relative_s": 6964.579585167114 + }, + { + "decode_latency_s": 0.06022480990213808, + "kv_live_bytes": 462274560, + "latency_s": 1.9281031666323543, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74219520, + "prompt_kind": "short", + "t_relative_s": 6969.5896417503245 + }, + { + "decode_latency_s": 0.06844901044678409, + "kv_live_bytes": 462274560, + "latency_s": 2.192008042242378, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74711040, + "prompt_kind": "long", + "t_relative_s": 6974.599718875252 + }, + { + "decode_latency_s": 0.06681322134681977, + "kv_live_bytes": 462274560, + "latency_s": 2.1397442501038313, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75284480, + "prompt_kind": "short", + "t_relative_s": 6979.609759625047 + }, + { + "decode_latency_s": 0.067910436206148, + "kv_live_bytes": 462274560, + "latency_s": 2.1757638338021934, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75399168, + "prompt_kind": "long", + "t_relative_s": 6984.61983929202 + }, + { + "decode_latency_s": 0.060810083348769695, + "kv_live_bytes": 462274560, + "latency_s": 1.9467107919044793, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75202560, + "prompt_kind": "short", + "t_relative_s": 6989.62193437526 + }, + { + "decode_latency_s": 0.06825759897765238, + "kv_live_bytes": 462274560, + "latency_s": 2.1866439171135426, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75235328, + "prompt_kind": "long", + "t_relative_s": 6994.631985000335 + }, + { + "decode_latency_s": 0.06020677734341007, + "kv_live_bytes": 462274560, + "latency_s": 1.9275576672516763, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75055104, + "prompt_kind": "short", + "t_relative_s": 6999.642064624932 + }, + { + "decode_latency_s": 0.06822254559665453, + "kv_live_bytes": 462274560, + "latency_s": 2.184775499626994, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75169792, + "prompt_kind": "long", + "t_relative_s": 7004.652127292007 + }, + { + "decode_latency_s": 0.06065628906071652, + "kv_live_bytes": 462274560, + "latency_s": 1.941864333115518, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75710464, + "prompt_kind": "short", + "t_relative_s": 7009.662205750123 + }, + { + "decode_latency_s": 0.06769943359540775, + "kv_live_bytes": 462274560, + "latency_s": 2.1688099172897637, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75857920, + "prompt_kind": "long", + "t_relative_s": 7014.672271292191 + }, + { + "decode_latency_s": 0.06054500000027474, + "kv_live_bytes": 462274560, + "latency_s": 1.9382267920300364, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75874304, + "prompt_kind": "short", + "t_relative_s": 7019.682366208173 + }, + { + "decode_latency_s": 0.06840506900334731, + "kv_live_bytes": 462274560, + "latency_s": 2.191465000156313, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75317248, + "prompt_kind": "long", + "t_relative_s": 7024.69242304191 + }, + { + "decode_latency_s": 0.060321815093629993, + "kv_live_bytes": 462274560, + "latency_s": 1.931181417312473, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75202560, + "prompt_kind": "short", + "t_relative_s": 7029.694860457908 + }, + { + "decode_latency_s": 0.06788559375854675, + "kv_live_bytes": 462274560, + "latency_s": 2.1741069168783724, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75137024, + "prompt_kind": "long", + "t_relative_s": 7034.70493687503 + }, + { + "decode_latency_s": 0.060350222658598796, + "kv_live_bytes": 462274560, + "latency_s": 1.9319648332893848, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75857920, + "prompt_kind": "short", + "t_relative_s": 7039.714995208196 + }, + { + "decode_latency_s": 0.06792755337664858, + "kv_live_bytes": 462274560, + "latency_s": 2.176520749926567, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76316672, + "prompt_kind": "long", + "t_relative_s": 7044.7250861250795 + }, + { + "decode_latency_s": 0.060142270842334256, + "kv_live_bytes": 462274560, + "latency_s": 1.9258347498252988, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76464128, + "prompt_kind": "short", + "t_relative_s": 7049.735140417237 + }, + { + "decode_latency_s": 0.06812575132062193, + "kv_live_bytes": 462274560, + "latency_s": 2.182192667387426, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76513280, + "prompt_kind": "long", + "t_relative_s": 7054.745216458105 + }, + { + "decode_latency_s": 0.06045704166172072, + "kv_live_bytes": 462274560, + "latency_s": 1.93543391674757, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76529664, + "prompt_kind": "short", + "t_relative_s": 7059.755332082976 + }, + { + "decode_latency_s": 0.06781800130556803, + "kv_live_bytes": 462274560, + "latency_s": 2.172644999809563, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76627968, + "prompt_kind": "long", + "t_relative_s": 7064.76538245799 + }, + { + "decode_latency_s": 0.06033712369389832, + "kv_live_bytes": 462274560, + "latency_s": 1.9315768331289291, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76644352, + "prompt_kind": "short", + "t_relative_s": 7069.775458374992 + }, + { + "decode_latency_s": 0.06821641279384494, + "kv_live_bytes": 462274560, + "latency_s": 2.184670540969819, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76627968, + "prompt_kind": "long", + "t_relative_s": 7074.785539083183 + }, + { + "decode_latency_s": 0.06030397916038055, + "kv_live_bytes": 462274560, + "latency_s": 1.9306435002945364, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76693504, + "prompt_kind": "short", + "t_relative_s": 7079.795590917114 + }, + { + "decode_latency_s": 0.06778747524367645, + "kv_live_bytes": 462274560, + "latency_s": 2.1716428752988577, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76759040, + "prompt_kind": "long", + "t_relative_s": 7084.805678833276 + }, + { + "decode_latency_s": 0.06061357681755908, + "kv_live_bytes": 462274560, + "latency_s": 1.9404225829057395, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77332480, + "prompt_kind": "short", + "t_relative_s": 7089.815772417001 + }, + { + "decode_latency_s": 0.06762485025683418, + "kv_live_bytes": 462274560, + "latency_s": 2.166610708925873, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76808192, + "prompt_kind": "long", + "t_relative_s": 7094.825827583205 + }, + { + "decode_latency_s": 0.0603937018167926, + "kv_live_bytes": 462274560, + "latency_s": 1.9334585419856012, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77365248, + "prompt_kind": "short", + "t_relative_s": 7099.835907167289 + }, + { + "decode_latency_s": 0.06809217056434136, + "kv_live_bytes": 462274560, + "latency_s": 2.180301041342318, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77430784, + "prompt_kind": "long", + "t_relative_s": 7104.845986292232 + }, + { + "decode_latency_s": 0.0604689283791231, + "kv_live_bytes": 462274560, + "latency_s": 1.9358604582957923, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77479936, + "prompt_kind": "short", + "t_relative_s": 7109.856033083051 + }, + { + "decode_latency_s": 0.06834389193682, + "kv_live_bytes": 462274560, + "latency_s": 2.189312625210732, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77529088, + "prompt_kind": "long", + "t_relative_s": 7114.866124667227 + }, + { + "decode_latency_s": 0.06066801300039515, + "kv_live_bytes": 462274560, + "latency_s": 1.9422239172272384, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78069760, + "prompt_kind": "short", + "t_relative_s": 7119.876245541964 + }, + { + "decode_latency_s": 0.06779302474751603, + "kv_live_bytes": 462274560, + "latency_s": 2.1719817076809704, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78069760, + "prompt_kind": "long", + "t_relative_s": 7124.8849015422165 + }, + { + "decode_latency_s": 0.060217217440367676, + "kv_live_bytes": 462274560, + "latency_s": 1.9278416251763701, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78020608, + "prompt_kind": "short", + "t_relative_s": 7129.894979167264 + }, + { + "decode_latency_s": 0.06073010675027035, + "kv_live_bytes": 462274560, + "latency_s": 1.9449086668901145, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78118912, + "prompt_kind": "long", + "t_relative_s": 7134.905030833092 + }, + { + "decode_latency_s": 0.06042916275328025, + "kv_live_bytes": 462274560, + "latency_s": 1.9344858331605792, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78233600, + "prompt_kind": "short", + "t_relative_s": 7139.915117667057 + }, + { + "decode_latency_s": 0.06819904559233692, + "kv_live_bytes": 462274560, + "latency_s": 2.1842479580082, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77545472, + "prompt_kind": "long", + "t_relative_s": 7144.9252056670375 + }, + { + "decode_latency_s": 0.060347897160681896, + "kv_live_bytes": 462274560, + "latency_s": 1.9320002500899136, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77545472, + "prompt_kind": "short", + "t_relative_s": 7149.9352502920665 + }, + { + "decode_latency_s": 0.06833205859584268, + "kv_live_bytes": 462274560, + "latency_s": 2.1885101245716214, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77611008, + "prompt_kind": "long", + "t_relative_s": 7154.9453179999255 + }, + { + "decode_latency_s": 0.06668451562291011, + "kv_live_bytes": 462274560, + "latency_s": 2.1356006669811904, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78151680, + "prompt_kind": "short", + "t_relative_s": 7159.9554134579375 + }, + { + "decode_latency_s": 0.06796012369159143, + "kv_live_bytes": 462274560, + "latency_s": 2.1775870001874864, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78282752, + "prompt_kind": "long", + "t_relative_s": 7164.965476792306 + }, + { + "decode_latency_s": 0.060344161465764046, + "kv_live_bytes": 462274560, + "latency_s": 1.9319207081571221, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78348288, + "prompt_kind": "short", + "t_relative_s": 7169.975562416948 + }, + { + "decode_latency_s": 0.06809596093080472, + "kv_live_bytes": 462274560, + "latency_s": 2.1809667921625078, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76677120, + "prompt_kind": "long", + "t_relative_s": 7174.985653417185 + }, + { + "decode_latency_s": 0.06657560546591412, + "kv_live_bytes": 462274560, + "latency_s": 2.1319570830091834, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76709888, + "prompt_kind": "short", + "t_relative_s": 7179.995706958231 + }, + { + "decode_latency_s": 0.06459774999530055, + "kv_live_bytes": 462274560, + "latency_s": 2.0685719158500433, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75972608, + "prompt_kind": "long", + "t_relative_s": 7185.005796167068 + }, + { + "decode_latency_s": 0.06667969921545591, + "kv_live_bytes": 462274560, + "latency_s": 2.135548332706094, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76120064, + "prompt_kind": "short", + "t_relative_s": 7190.015862208325 + }, + { + "decode_latency_s": 0.06803118489915505, + "kv_live_bytes": 462274560, + "latency_s": 2.1794971246272326, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75890688, + "prompt_kind": "long", + "t_relative_s": 7195.025919750333 + }, + { + "decode_latency_s": 0.06049961849930696, + "kv_live_bytes": 462274560, + "latency_s": 1.936876333784312, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75415552, + "prompt_kind": "short", + "t_relative_s": 7200.035992917139 + }, + { + "decode_latency_s": 0.06881481381424237, + "kv_live_bytes": 462274560, + "latency_s": 2.204003875143826, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75710464, + "prompt_kind": "long", + "t_relative_s": 7205.04607662512 + }, + { + "decode_latency_s": 0.061233985688886605, + "kv_live_bytes": 462274560, + "latency_s": 1.9603989161550999, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75382784, + "prompt_kind": "short", + "t_relative_s": 7210.056131000165 + }, + { + "decode_latency_s": 0.06856543486355804, + "kv_live_bytes": 462274560, + "latency_s": 2.195921041071415, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75038720, + "prompt_kind": "long", + "t_relative_s": 7215.06621375028 + }, + { + "decode_latency_s": 0.06059744791127741, + "kv_live_bytes": 462274560, + "latency_s": 1.9402917078696191, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76382208, + "prompt_kind": "short", + "t_relative_s": 7220.071125958115 + }, + { + "decode_latency_s": 0.06097530078841373, + "kv_live_bytes": 462274560, + "latency_s": 1.95262208301574, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76496896, + "prompt_kind": "long", + "t_relative_s": 7225.081176124979 + }, + { + "decode_latency_s": 0.06033020179893356, + "kv_live_bytes": 462274560, + "latency_s": 1.9312975001521409, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76693504, + "prompt_kind": "short", + "t_relative_s": 7230.091256957967 + }, + { + "decode_latency_s": 0.060568524757400155, + "kv_live_bytes": 462274560, + "latency_s": 1.9390561250038445, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76021760, + "prompt_kind": "long", + "t_relative_s": 7235.101328582969 + }, + { + "decode_latency_s": 0.06045986847311724, + "kv_live_bytes": 462274560, + "latency_s": 1.9356704577803612, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "short", + "t_relative_s": 7240.111398125067 + }, + { + "decode_latency_s": 0.06815410546550993, + "kv_live_bytes": 462274560, + "latency_s": 2.1832765829749405, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75169792, + "prompt_kind": "long", + "t_relative_s": 7245.121443333104 + }, + { + "decode_latency_s": 0.06037955210194923, + "kv_live_bytes": 462274560, + "latency_s": 1.9330731253139675, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75530240, + "prompt_kind": "short", + "t_relative_s": 7250.131534541957 + }, + { + "decode_latency_s": 0.06405449999147095, + "kv_live_bytes": 462274560, + "latency_s": 2.050872541964054, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70090752, + "prompt_kind": "long", + "t_relative_s": 7255.141787083354 + }, + { + "decode_latency_s": 0.06038871612690855, + "kv_live_bytes": 462274560, + "latency_s": 1.9334346670657396, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70336512, + "prompt_kind": "short", + "t_relative_s": 7261.5923203751445 + }, + { + "decode_latency_s": 0.06800741665938403, + "kv_live_bytes": 462274560, + "latency_s": 2.177788292057812, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70664192, + "prompt_kind": "long", + "t_relative_s": 7266.602387832943 + }, + { + "decode_latency_s": 0.06057979818433523, + "kv_live_bytes": 462274560, + "latency_s": 1.9393331673927605, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71254016, + "prompt_kind": "short", + "t_relative_s": 7271.612444542348 + }, + { + "decode_latency_s": 0.0680844140733825, + "kv_live_bytes": 462274560, + "latency_s": 2.1815390000119805, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71172096, + "prompt_kind": "long", + "t_relative_s": 7276.622523583006 + }, + { + "decode_latency_s": 0.06065312889404595, + "kv_live_bytes": 462274560, + "latency_s": 1.9416884589008987, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71745536, + "prompt_kind": "short", + "t_relative_s": 7281.632620250341 + }, + { + "decode_latency_s": 0.06817869530641474, + "kv_live_bytes": 462274560, + "latency_s": 2.183564541861415, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71827456, + "prompt_kind": "long", + "t_relative_s": 7286.642662125174 + }, + { + "decode_latency_s": 0.060331419241265394, + "kv_live_bytes": 462274560, + "latency_s": 1.9315054579637945, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71385088, + "prompt_kind": "short", + "t_relative_s": 7291.652714082971 + }, + { + "decode_latency_s": 0.06786925780761521, + "kv_live_bytes": 462274560, + "latency_s": 2.1737910001538694, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71516160, + "prompt_kind": "long", + "t_relative_s": 7296.662800667342 + }, + { + "decode_latency_s": 0.06067643100686837, + "kv_live_bytes": 462274560, + "latency_s": 1.9425064590759575, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72105984, + "prompt_kind": "short", + "t_relative_s": 7301.672875457909 + }, + { + "decode_latency_s": 0.06920446225558408, + "kv_live_bytes": 462274560, + "latency_s": 2.2169441250152886, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71909376, + "prompt_kind": "long", + "t_relative_s": 7306.682934999932 + }, + { + "decode_latency_s": 0.06663027212198358, + "kv_live_bytes": 462274560, + "latency_s": 2.133940333034843, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72499200, + "prompt_kind": "short", + "t_relative_s": 7311.692994917277 + }, + { + "decode_latency_s": 0.06818807160016149, + "kv_live_bytes": 462274560, + "latency_s": 2.1843085000291467, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72531968, + "prompt_kind": "long", + "t_relative_s": 7316.70305208303 + }, + { + "decode_latency_s": 0.0605999244726263, + "kv_live_bytes": 462274560, + "latency_s": 1.9400837919674814, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71942144, + "prompt_kind": "short", + "t_relative_s": 7321.7131439582445 + }, + { + "decode_latency_s": 0.06836798306903802, + "kv_live_bytes": 462274560, + "latency_s": 2.189271583221853, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72056832, + "prompt_kind": "long", + "t_relative_s": 7326.723225000314 + }, + { + "decode_latency_s": 0.060364704419043846, + "kv_live_bytes": 462274560, + "latency_s": 1.9324877923354506, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "short", + "t_relative_s": 7331.733267208096 + }, + { + "decode_latency_s": 0.060567696593352593, + "kv_live_bytes": 462274560, + "latency_s": 1.9394711670465767, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72597504, + "prompt_kind": "long", + "t_relative_s": 7336.743318708148 + }, + { + "decode_latency_s": 0.06080362890497781, + "kv_live_bytes": 462274560, + "latency_s": 1.9465409996919334, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73187328, + "prompt_kind": "short", + "t_relative_s": 7341.753396542277 + }, + { + "decode_latency_s": 0.0677484583138721, + "kv_live_bytes": 462274560, + "latency_s": 2.1703937500715256, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73269248, + "prompt_kind": "long", + "t_relative_s": 7346.763445042074 + }, + { + "decode_latency_s": 0.060320829434203915, + "kv_live_bytes": 462274560, + "latency_s": 1.931175250094384, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73269248, + "prompt_kind": "short", + "t_relative_s": 7351.77350633312 + }, + { + "decode_latency_s": 0.06798711587907746, + "kv_live_bytes": 462274560, + "latency_s": 2.1772938752546906, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73334784, + "prompt_kind": "long", + "t_relative_s": 7356.783576000016 + }, + { + "decode_latency_s": 0.060703451847075485, + "kv_live_bytes": 462274560, + "latency_s": 1.9433642500080168, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73908224, + "prompt_kind": "short", + "t_relative_s": 7361.793634083122 + }, + { + "decode_latency_s": 0.06777945571229793, + "kv_live_bytes": 462274560, + "latency_s": 2.1712264167144895, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73940992, + "prompt_kind": "long", + "t_relative_s": 7366.803691625129 + }, + { + "decode_latency_s": 0.06058150000171736, + "kv_live_bytes": 462274560, + "latency_s": 1.939401624724269, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73973760, + "prompt_kind": "short", + "t_relative_s": 7371.8137663332745 + }, + { + "decode_latency_s": 0.06791937108209822, + "kv_live_bytes": 462274560, + "latency_s": 2.175516915973276, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73990144, + "prompt_kind": "long", + "t_relative_s": 7376.823821458034 + }, + { + "decode_latency_s": 0.06039214583870489, + "kv_live_bytes": 462274560, + "latency_s": 1.9334094170480967, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74022912, + "prompt_kind": "short", + "t_relative_s": 7381.833898042329 + }, + { + "decode_latency_s": 0.060280412741121836, + "kv_live_bytes": 462274560, + "latency_s": 1.929837082978338, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74072064, + "prompt_kind": "long", + "t_relative_s": 7386.843973125331 + }, + { + "decode_latency_s": 0.060482282569864765, + "kv_live_bytes": 462274560, + "latency_s": 1.9362763329409063, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74661888, + "prompt_kind": "short", + "t_relative_s": 7391.854016125202 + }, + { + "decode_latency_s": 0.06823273569170851, + "kv_live_bytes": 462274560, + "latency_s": 2.186238208319992, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74072064, + "prompt_kind": "long", + "t_relative_s": 7396.864078583196 + }, + { + "decode_latency_s": 0.06053659116150811, + "kv_live_bytes": 462274560, + "latency_s": 1.9380071656778455, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74645504, + "prompt_kind": "short", + "t_relative_s": 7401.874178833328 + }, + { + "decode_latency_s": 0.06782558462873567, + "kv_live_bytes": 462274560, + "latency_s": 2.172533041331917, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74711040, + "prompt_kind": "long", + "t_relative_s": 7406.884228667244 + }, + { + "decode_latency_s": 0.060414208346628584, + "kv_live_bytes": 462274560, + "latency_s": 1.934136999770999, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74825728, + "prompt_kind": "short", + "t_relative_s": 7411.894281167071 + }, + { + "decode_latency_s": 0.06830718491983134, + "kv_live_bytes": 462274560, + "latency_s": 2.1879371250979602, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74891264, + "prompt_kind": "long", + "t_relative_s": 7416.904368500225 + }, + { + "decode_latency_s": 0.06049879555939697, + "kv_live_bytes": 462274560, + "latency_s": 1.9367778752930462, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74907648, + "prompt_kind": "short", + "t_relative_s": 7421.914435875136 + }, + { + "decode_latency_s": 0.06068203515314963, + "kv_live_bytes": 462274560, + "latency_s": 1.94279395788908, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74924032, + "prompt_kind": "long", + "t_relative_s": 7426.924492083024 + }, + { + "decode_latency_s": 0.06690829688159283, + "kv_live_bytes": 462274560, + "latency_s": 2.1427332921884954, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74956800, + "prompt_kind": "short", + "t_relative_s": 7431.934571458027 + }, + { + "decode_latency_s": 0.06035788022563793, + "kv_live_bytes": 462274560, + "latency_s": 1.9325604159384966, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74956800, + "prompt_kind": "long", + "t_relative_s": 7436.944616000168 + }, + { + "decode_latency_s": 0.060384578129742295, + "kv_live_bytes": 462274560, + "latency_s": 1.933192207943648, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74809344, + "prompt_kind": "short", + "t_relative_s": 7441.9546777503565 + }, + { + "decode_latency_s": 0.06824153257184662, + "kv_live_bytes": 462274560, + "latency_s": 2.186123457737267, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74924032, + "prompt_kind": "long", + "t_relative_s": 7446.964758499991 + }, + { + "decode_latency_s": 0.06652402994222939, + "kv_live_bytes": 462274560, + "latency_s": 2.1304632909595966, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75481088, + "prompt_kind": "short", + "t_relative_s": 7451.974815333262 + }, + { + "decode_latency_s": 0.06768269008898642, + "kv_live_bytes": 462274560, + "latency_s": 2.168578375130892, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75513856, + "prompt_kind": "long", + "t_relative_s": 7456.98488041712 + }, + { + "decode_latency_s": 0.06052105731214397, + "kv_live_bytes": 462274560, + "latency_s": 1.9374624998308718, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75513856, + "prompt_kind": "short", + "t_relative_s": 7461.994970917236 + }, + { + "decode_latency_s": 0.06815102342807222, + "kv_live_bytes": 462274560, + "latency_s": 2.1828500418923795, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75399168, + "prompt_kind": "long", + "t_relative_s": 7467.005017917138 + }, + { + "decode_latency_s": 0.060446233095717616, + "kv_live_bytes": 462274560, + "latency_s": 1.9351708749309182, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75628544, + "prompt_kind": "short", + "t_relative_s": 7472.015065208077 + }, + { + "decode_latency_s": 0.06796266142919194, + "kv_live_bytes": 462274560, + "latency_s": 2.1768078338354826, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75694080, + "prompt_kind": "long", + "t_relative_s": 7477.025146208238 + }, + { + "decode_latency_s": 0.06088845184422098, + "kv_live_bytes": 462274560, + "latency_s": 1.9491943749599159, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76234752, + "prompt_kind": "short", + "t_relative_s": 7482.035235667136 + }, + { + "decode_latency_s": 0.06314969400409609, + "kv_live_bytes": 462274560, + "latency_s": 2.0219997498206794, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71401472, + "prompt_kind": "long", + "t_relative_s": 7487.035287207924 + }, + { + "decode_latency_s": 0.06032462891016621, + "kv_live_bytes": 462274560, + "latency_s": 1.9312961250543594, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71909376, + "prompt_kind": "short", + "t_relative_s": 7492.905891292263 + }, + { + "decode_latency_s": 0.06749759503873065, + "kv_live_bytes": 462274560, + "latency_s": 2.1621838328428566, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71991296, + "prompt_kind": "long", + "t_relative_s": 7497.915963458363 + }, + { + "decode_latency_s": 0.060747489595087245, + "kv_live_bytes": 462274560, + "latency_s": 1.9447706658393145, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72630272, + "prompt_kind": "short", + "t_relative_s": 7502.926042875275 + }, + { + "decode_latency_s": 0.06858877862396184, + "kv_live_bytes": 462274560, + "latency_s": 2.1972520002163947, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71368704, + "prompt_kind": "long", + "t_relative_s": 7507.9360987083055 + }, + { + "decode_latency_s": 0.06042635288031306, + "kv_live_bytes": 462274560, + "latency_s": 1.9345455411821604, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71090176, + "prompt_kind": "short", + "t_relative_s": 7512.946175666992 + }, + { + "decode_latency_s": 0.06835198700719047, + "kv_live_bytes": 462274560, + "latency_s": 2.1894545000977814, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71548928, + "prompt_kind": "long", + "t_relative_s": 7517.956252083182 + }, + { + "decode_latency_s": 0.06060058203001972, + "kv_live_bytes": 462274560, + "latency_s": 1.9401006251573563, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72089600, + "prompt_kind": "short", + "t_relative_s": 7522.966296874918 + }, + { + "decode_latency_s": 0.06805089712725021, + "kv_live_bytes": 462274560, + "latency_s": 2.180019082967192, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72155136, + "prompt_kind": "long", + "t_relative_s": 7527.976386208087 + }, + { + "decode_latency_s": 0.06699903385015205, + "kv_live_bytes": 462274560, + "latency_s": 2.145375957712531, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72253440, + "prompt_kind": "short", + "t_relative_s": 7532.986459041946 + }, + { + "decode_latency_s": 0.06833687370817643, + "kv_live_bytes": 462274560, + "latency_s": 2.1891484172083437, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72253440, + "prompt_kind": "long", + "t_relative_s": 7537.996516332962 + }, + { + "decode_latency_s": 0.060812951807747595, + "kv_live_bytes": 462274560, + "latency_s": 1.9469340411014855, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71991296, + "prompt_kind": "short", + "t_relative_s": 7542.999826667365 + }, + { + "decode_latency_s": 0.06793122657109052, + "kv_live_bytes": 462274560, + "latency_s": 2.1758836661465466, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71876608, + "prompt_kind": "long", + "t_relative_s": 7548.009909583256 + }, + { + "decode_latency_s": 0.0607969466218492, + "kv_live_bytes": 462274560, + "latency_s": 1.946375290863216, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72433664, + "prompt_kind": "short", + "t_relative_s": 7553.019959625322 + }, + { + "decode_latency_s": 0.06748826822149567, + "kv_live_bytes": 462274560, + "latency_s": 2.1619398752227426, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72499200, + "prompt_kind": "long", + "t_relative_s": 7558.029271374922 + }, + { + "decode_latency_s": 0.06109037759597413, + "kv_live_bytes": 462274560, + "latency_s": 1.9556813752278686, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72581120, + "prompt_kind": "short", + "t_relative_s": 7563.039374208078 + }, + { + "decode_latency_s": 0.06793339975411072, + "kv_live_bytes": 462274560, + "latency_s": 2.176313958130777, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72548352, + "prompt_kind": "long", + "t_relative_s": 7568.049426542129 + }, + { + "decode_latency_s": 0.060269205714575946, + "kv_live_bytes": 462274560, + "latency_s": 1.9295261250808835, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73138176, + "prompt_kind": "short", + "t_relative_s": 7573.059504833072 + }, + { + "decode_latency_s": 0.06729600390826818, + "kv_live_bytes": 462274560, + "latency_s": 2.1550620002672076, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72581120, + "prompt_kind": "long", + "t_relative_s": 7578.069583167322 + }, + { + "decode_latency_s": 0.06041502735752147, + "kv_live_bytes": 462274560, + "latency_s": 1.9342403332702816, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73154560, + "prompt_kind": "short", + "t_relative_s": 7583.079638000112 + }, + { + "decode_latency_s": 0.06800155078235548, + "kv_live_bytes": 462274560, + "latency_s": 2.1781423329375684, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73285632, + "prompt_kind": "long", + "t_relative_s": 7588.089749875013 + }, + { + "decode_latency_s": 0.06063200130301993, + "kv_live_bytes": 462274560, + "latency_s": 1.9410469997674227, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73367552, + "prompt_kind": "short", + "t_relative_s": 7593.099864624906 + }, + { + "decode_latency_s": 0.06785003647382837, + "kv_live_bytes": 462274560, + "latency_s": 2.1735819168388844, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "long", + "t_relative_s": 7598.109926542267 + }, + { + "decode_latency_s": 0.06074229556543287, + "kv_live_bytes": 462274560, + "latency_s": 1.944717874750495, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73990144, + "prompt_kind": "short", + "t_relative_s": 7603.120008542202 + }, + { + "decode_latency_s": 0.06819481639831793, + "kv_live_bytes": 462274560, + "latency_s": 2.1837957077659667, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73957376, + "prompt_kind": "long", + "t_relative_s": 7608.130096042063 + }, + { + "decode_latency_s": 0.06717539712553844, + "kv_live_bytes": 462274560, + "latency_s": 2.1513825422152877, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73957376, + "prompt_kind": "short", + "t_relative_s": 7613.140145167243 + }, + { + "decode_latency_s": 0.067998328115209, + "kv_live_bytes": 462274560, + "latency_s": 2.1786036249250174, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74055680, + "prompt_kind": "long", + "t_relative_s": 7618.150275708176 + }, + { + "decode_latency_s": 0.06056154947145842, + "kv_live_bytes": 462274560, + "latency_s": 1.9387181662023067, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74137600, + "prompt_kind": "short", + "t_relative_s": 7623.160357542336 + }, + { + "decode_latency_s": 0.06800824349920731, + "kv_live_bytes": 462274560, + "latency_s": 2.178863083012402, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74039296, + "prompt_kind": "long", + "t_relative_s": 7628.170404375065 + }, + { + "decode_latency_s": 0.06029634765582159, + "kv_live_bytes": 462274560, + "latency_s": 1.9303993340581656, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73596928, + "prompt_kind": "short", + "t_relative_s": 7633.18049970828 + }, + { + "decode_latency_s": 0.06765080337936524, + "kv_live_bytes": 462274560, + "latency_s": 2.166704291012138, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73547776, + "prompt_kind": "long", + "t_relative_s": 7638.190571417101 + }, + { + "decode_latency_s": 0.06658908724784851, + "kv_live_bytes": 462274560, + "latency_s": 2.13264033338055, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74121216, + "prompt_kind": "short", + "t_relative_s": 7643.200626958162 + }, + { + "decode_latency_s": 0.06790290234494023, + "kv_live_bytes": 462274560, + "latency_s": 2.176214958075434, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74203136, + "prompt_kind": "long", + "t_relative_s": 7648.210718958173 + }, + { + "decode_latency_s": 0.06050004165444989, + "kv_live_bytes": 462274560, + "latency_s": 1.936825416982174, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74268672, + "prompt_kind": "short", + "t_relative_s": 7653.220832750201 + }, + { + "decode_latency_s": 0.0678200221300358, + "kv_live_bytes": 462274560, + "latency_s": 2.172898749820888, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74416128, + "prompt_kind": "long", + "t_relative_s": 7658.230877708178 + }, + { + "decode_latency_s": 0.06074672655086033, + "kv_live_bytes": 462274560, + "latency_s": 1.944766249973327, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74399744, + "prompt_kind": "short", + "t_relative_s": 7663.240951917134 + }, + { + "decode_latency_s": 0.06798610936675686, + "kv_live_bytes": 462274560, + "latency_s": 2.1776407919824123, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73875456, + "prompt_kind": "long", + "t_relative_s": 7668.2510328749195 + }, + { + "decode_latency_s": 0.060470147160231136, + "kv_live_bytes": 462274560, + "latency_s": 1.9359576245769858, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74383360, + "prompt_kind": "short", + "t_relative_s": 7673.26108945813 + }, + { + "decode_latency_s": 0.06799275781668257, + "kv_live_bytes": 462274560, + "latency_s": 2.1778074582107365, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 81903616, + "prompt_kind": "long", + "t_relative_s": 7678.271176208276 + }, + { + "decode_latency_s": 0.060512173164170235, + "kv_live_bytes": 462274560, + "latency_s": 1.9371324582025409, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 82001920, + "prompt_kind": "short", + "t_relative_s": 7683.2812906671315 + }, + { + "decode_latency_s": 0.06052619403635617, + "kv_live_bytes": 462274560, + "latency_s": 1.937973499763757, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 82018304, + "prompt_kind": "long", + "t_relative_s": 7688.291341250297 + }, + { + "decode_latency_s": 0.06041463931614999, + "kv_live_bytes": 462274560, + "latency_s": 1.934253249783069, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 82345984, + "prompt_kind": "short", + "t_relative_s": 7693.301423999947 + }, + { + "decode_latency_s": 0.0680665625113761, + "kv_live_bytes": 462274560, + "latency_s": 2.179544833023101, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 80740352, + "prompt_kind": "long", + "t_relative_s": 7698.311509625055 + }, + { + "decode_latency_s": 0.060618592440732755, + "kv_live_bytes": 462274560, + "latency_s": 1.9406378329731524, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 81281024, + "prompt_kind": "short", + "t_relative_s": 7703.321575125214 + }, + { + "decode_latency_s": 0.06767777343338821, + "kv_live_bytes": 462274560, + "latency_s": 2.16796191688627, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 81379328, + "prompt_kind": "long", + "t_relative_s": 7708.3316618329845 + }, + { + "decode_latency_s": 0.06053177475405391, + "kv_live_bytes": 462274560, + "latency_s": 1.937835458200425, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 81379328, + "prompt_kind": "short", + "t_relative_s": 7713.333047500346 + }, + { + "decode_latency_s": 0.06833736460248474, + "kv_live_bytes": 462274560, + "latency_s": 2.188853583764285, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 81412096, + "prompt_kind": "long", + "t_relative_s": 7718.343099167105 + }, + { + "decode_latency_s": 0.06043743621557951, + "kv_live_bytes": 462274560, + "latency_s": 1.9349419581703842, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79249408, + "prompt_kind": "short", + "t_relative_s": 7723.35315779224 + }, + { + "decode_latency_s": 0.06799870965187438, + "kv_live_bytes": 462274560, + "latency_s": 2.178669084329158, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79282176, + "prompt_kind": "long", + "t_relative_s": 7728.36323300004 + }, + { + "decode_latency_s": 0.06047173179103993, + "kv_live_bytes": 462274560, + "latency_s": 1.9358994169160724, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79364096, + "prompt_kind": "short", + "t_relative_s": 7733.373281457927 + }, + { + "decode_latency_s": 0.0679380286601372, + "kv_live_bytes": 462274560, + "latency_s": 2.1766857081092894, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79396864, + "prompt_kind": "long", + "t_relative_s": 7738.383346083108 + }, + { + "decode_latency_s": 0.06049421353964135, + "kv_live_bytes": 462274560, + "latency_s": 1.9365792092867196, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79446016, + "prompt_kind": "short", + "t_relative_s": 7743.393428250216 + }, + { + "decode_latency_s": 0.06046458984201308, + "kv_live_bytes": 462274560, + "latency_s": 1.9361309171654284, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79446016, + "prompt_kind": "long", + "t_relative_s": 7748.403488458134 + }, + { + "decode_latency_s": 0.06023388540779706, + "kv_live_bytes": 462274560, + "latency_s": 1.928411084227264, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79904768, + "prompt_kind": "short", + "t_relative_s": 7753.4135560831055 + }, + { + "decode_latency_s": 0.06026119271700736, + "kv_live_bytes": 462274560, + "latency_s": 1.9292402910068631, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79446016, + "prompt_kind": "long", + "t_relative_s": 7758.4236435000785 + }, + { + "decode_latency_s": 0.06056269658438396, + "kv_live_bytes": 462274560, + "latency_s": 1.9388349582441151, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79986688, + "prompt_kind": "short", + "t_relative_s": 7763.433711207937 + }, + { + "decode_latency_s": 0.06808899086900055, + "kv_live_bytes": 462274560, + "latency_s": 2.181974791921675, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 80084992, + "prompt_kind": "long", + "t_relative_s": 7768.443798167165 + }, + { + "decode_latency_s": 0.060880261720740236, + "kv_live_bytes": 462274560, + "latency_s": 1.9489853330887854, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 80134144, + "prompt_kind": "short", + "t_relative_s": 7773.453885958064 + }, + { + "decode_latency_s": 0.06049989321036264, + "kv_live_bytes": 462274560, + "latency_s": 1.9369014999829233, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 80150528, + "prompt_kind": "long", + "t_relative_s": 7778.463934333064 + }, + { + "decode_latency_s": 0.061084128901711665, + "kv_live_bytes": 462274560, + "latency_s": 1.9555985420010984, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79249408, + "prompt_kind": "short", + "t_relative_s": 7783.473996125162 + }, + { + "decode_latency_s": 0.06429701171873603, + "kv_live_bytes": 462274560, + "latency_s": 2.0585994170978665, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74760192, + "prompt_kind": "long", + "t_relative_s": 7788.484068042133 + }, + { + "decode_latency_s": 0.060590294291614555, + "kv_live_bytes": 462274560, + "latency_s": 1.940071249846369, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75087872, + "prompt_kind": "short", + "t_relative_s": 7795.028333500028 + }, + { + "decode_latency_s": 0.06889944399881642, + "kv_live_bytes": 462274560, + "latency_s": 2.2072855001315475, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67960832, + "prompt_kind": "long", + "t_relative_s": 7800.038405207917 + }, + { + "decode_latency_s": 0.06025044531270396, + "kv_live_bytes": 462274560, + "latency_s": 1.9289866252802312, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67518464, + "prompt_kind": "short", + "t_relative_s": 7805.048451917246 + }, + { + "decode_latency_s": 0.06707122657098807, + "kv_live_bytes": 462274560, + "latency_s": 2.14834566693753, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70647808, + "prompt_kind": "long", + "t_relative_s": 7810.058532875031 + }, + { + "decode_latency_s": 0.06073130600270815, + "kv_live_bytes": 462274560, + "latency_s": 1.9441755833104253, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71122944, + "prompt_kind": "short", + "t_relative_s": 7815.06860166695 + }, + { + "decode_latency_s": 0.060432399724959396, + "kv_live_bytes": 462274560, + "latency_s": 1.935428540688008, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71204864, + "prompt_kind": "long", + "t_relative_s": 7820.075868792366 + }, + { + "decode_latency_s": 0.06049545049609151, + "kv_live_bytes": 462274560, + "latency_s": 1.9366557919420302, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71745536, + "prompt_kind": "short", + "t_relative_s": 7825.085953874979 + }, + { + "decode_latency_s": 0.06863233853073325, + "kv_live_bytes": 462274560, + "latency_s": 2.198310417123139, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71843840, + "prompt_kind": "long", + "t_relative_s": 7830.09600995807 + }, + { + "decode_latency_s": 0.06044345312693622, + "kv_live_bytes": 462274560, + "latency_s": 1.9351307908073068, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71892992, + "prompt_kind": "short", + "t_relative_s": 7835.1060968749225 + }, + { + "decode_latency_s": 0.06797323437058367, + "kv_live_bytes": 462274560, + "latency_s": 2.1767185418866575, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71909376, + "prompt_kind": "long", + "t_relative_s": 7840.116187292151 + }, + { + "decode_latency_s": 0.060581205718335696, + "kv_live_bytes": 462274560, + "latency_s": 1.9394103749655187, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71876608, + "prompt_kind": "short", + "t_relative_s": 7845.119569832925 + }, + { + "decode_latency_s": 0.06567297395667993, + "kv_live_bytes": 462274560, + "latency_s": 2.1030265828594565, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72024064, + "prompt_kind": "long", + "t_relative_s": 7850.129628832918 + }, + { + "decode_latency_s": 0.06118410546332598, + "kv_live_bytes": 462274560, + "latency_s": 1.9586349581368268, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72482816, + "prompt_kind": "short", + "t_relative_s": 7855.139703917317 + }, + { + "decode_latency_s": 0.06094087372184731, + "kv_live_bytes": 462274560, + "latency_s": 1.951093875337392, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72531968, + "prompt_kind": "long", + "t_relative_s": 7860.149739332963 + }, + { + "decode_latency_s": 0.06037838544580154, + "kv_live_bytes": 462274560, + "latency_s": 1.9330372917465866, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72155136, + "prompt_kind": "short", + "t_relative_s": 7865.15984262526 + }, + { + "decode_latency_s": 0.06827385028009303, + "kv_live_bytes": 462274560, + "latency_s": 2.186699707992375, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71974912, + "prompt_kind": "long", + "t_relative_s": 7870.1699400423095 + }, + { + "decode_latency_s": 0.060395355481887236, + "kv_live_bytes": 462274560, + "latency_s": 1.933584333397448, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71925760, + "prompt_kind": "short", + "t_relative_s": 7875.179996833205 + }, + { + "decode_latency_s": 0.067791740904795, + "kv_live_bytes": 462274560, + "latency_s": 2.1713739996775985, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72695808, + "prompt_kind": "long", + "t_relative_s": 7880.190052208025 + }, + { + "decode_latency_s": 0.06049935286864638, + "kv_live_bytes": 462274560, + "latency_s": 1.9368295408785343, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71876608, + "prompt_kind": "short", + "t_relative_s": 7885.200136333238 + }, + { + "decode_latency_s": 0.06805674089991953, + "kv_live_bytes": 462274560, + "latency_s": 2.1796172498725355, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72220672, + "prompt_kind": "long", + "t_relative_s": 7890.210221624933 + }, + { + "decode_latency_s": 0.06047913931251969, + "kv_live_bytes": 462274560, + "latency_s": 1.9362139999866486, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72417280, + "prompt_kind": "short", + "t_relative_s": 7895.220255000051 + }, + { + "decode_latency_s": 0.06791724868526217, + "kv_live_bytes": 462274560, + "latency_s": 2.175848958082497, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72482816, + "prompt_kind": "long", + "t_relative_s": 7900.230330542196 + }, + { + "decode_latency_s": 0.0606665638188133, + "kv_live_bytes": 462274560, + "latency_s": 1.9421230829320848, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72843264, + "prompt_kind": "short", + "t_relative_s": 7905.240431792103 + }, + { + "decode_latency_s": 0.13782751304097474, + "kv_live_bytes": 462274560, + "latency_s": 4.41132283443585, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69173248, + "prompt_kind": "long", + "t_relative_s": 7910.250487208366 + }, + { + "decode_latency_s": 0.06038067315239459, + "kv_live_bytes": 462274560, + "latency_s": 1.9330936251208186, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69271552, + "prompt_kind": "short", + "t_relative_s": 7919.277554542292 + }, + { + "decode_latency_s": 0.06845641929248814, + "kv_live_bytes": 462274560, + "latency_s": 2.192249875050038, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69877760, + "prompt_kind": "long", + "t_relative_s": 7924.287635750137 + }, + { + "decode_latency_s": 0.060412243474274874, + "kv_live_bytes": 462274560, + "latency_s": 1.93452825024724, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70647808, + "prompt_kind": "short", + "t_relative_s": 7929.297687042039 + }, + { + "decode_latency_s": 0.0680943880288396, + "kv_live_bytes": 462274560, + "latency_s": 2.1812799582257867, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70041600, + "prompt_kind": "long", + "t_relative_s": 7934.307768625207 + }, + { + "decode_latency_s": 0.060571166657609865, + "kv_live_bytes": 462274560, + "latency_s": 1.9390991660766304, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70598656, + "prompt_kind": "short", + "t_relative_s": 7939.317870375235 + }, + { + "decode_latency_s": 0.06787764062755741, + "kv_live_bytes": 462274560, + "latency_s": 2.1742301248013973, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70615040, + "prompt_kind": "long", + "t_relative_s": 7944.327917499933 + }, + { + "decode_latency_s": 0.06067382685432676, + "kv_live_bytes": 462274560, + "latency_s": 1.94249645806849, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71139328, + "prompt_kind": "short", + "t_relative_s": 7949.338001166936 + }, + { + "decode_latency_s": 0.06817901298927609, + "kv_live_bytes": 462274560, + "latency_s": 2.18364941701293, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71139328, + "prompt_kind": "long", + "t_relative_s": 7954.348075082991 + }, + { + "decode_latency_s": 0.06051036328426562, + "kv_live_bytes": 462274560, + "latency_s": 1.9372510421089828, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71172096, + "prompt_kind": "short", + "t_relative_s": 7959.358139375225 + }, + { + "decode_latency_s": 0.06795035937102512, + "kv_live_bytes": 462274560, + "latency_s": 2.177136749960482, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70615040, + "prompt_kind": "long", + "t_relative_s": 7964.368212375324 + }, + { + "decode_latency_s": 0.06054515756841283, + "kv_live_bytes": 462274560, + "latency_s": 1.9382467921823263, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71041024, + "prompt_kind": "short", + "t_relative_s": 7969.378282958176 + }, + { + "decode_latency_s": 0.06795243750093505, + "kv_live_bytes": 462274560, + "latency_s": 2.17648124974221, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71041024, + "prompt_kind": "long", + "t_relative_s": 7974.3883380419575 + }, + { + "decode_latency_s": 0.060358755217748694, + "kv_live_bytes": 462274560, + "latency_s": 1.9323791251517832, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71581696, + "prompt_kind": "short", + "t_relative_s": 7979.398409458343 + }, + { + "decode_latency_s": 0.06819762372469995, + "kv_live_bytes": 462274560, + "latency_s": 2.183954792097211, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71024640, + "prompt_kind": "long", + "t_relative_s": 7984.408494625241 + }, + { + "decode_latency_s": 0.06041945572360419, + "kv_live_bytes": 462274560, + "latency_s": 1.9345865421928465, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71106560, + "prompt_kind": "short", + "t_relative_s": 7989.418563708197 + }, + { + "decode_latency_s": 0.067802351564751, + "kv_live_bytes": 462274560, + "latency_s": 2.172213749960065, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71139328, + "prompt_kind": "long", + "t_relative_s": 7994.4286370002665 + }, + { + "decode_latency_s": 0.06650342709326651, + "kv_live_bytes": 462274560, + "latency_s": 2.1297885831445456, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71663616, + "prompt_kind": "short", + "t_relative_s": 7999.438718917314 + }, + { + "decode_latency_s": 0.06806469659204595, + "kv_live_bytes": 462274560, + "latency_s": 2.1801792080514133, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71548928, + "prompt_kind": "long", + "t_relative_s": 8004.448771500029 + }, + { + "decode_latency_s": 0.0602770755358506, + "kv_live_bytes": 462274560, + "latency_s": 1.9297712086699903, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72089600, + "prompt_kind": "short", + "t_relative_s": 8009.458868375048 + }, + { + "decode_latency_s": 0.06090348699945025, + "kv_live_bytes": 462274560, + "latency_s": 1.9501274591311812, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72073216, + "prompt_kind": "long", + "t_relative_s": 8014.468950625043 + }, + { + "decode_latency_s": 0.06656474999908824, + "kv_live_bytes": 462274560, + "latency_s": 2.1316438750363886, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72237056, + "prompt_kind": "short", + "t_relative_s": 8019.4790060832165 + }, + { + "decode_latency_s": 0.06799462500202935, + "kv_live_bytes": 462274560, + "latency_s": 2.1783810830675066, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72253440, + "prompt_kind": "long", + "t_relative_s": 8024.4891115003265 + }, + { + "decode_latency_s": 0.06681233465496916, + "kv_live_bytes": 462274560, + "latency_s": 2.1396749997511506, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72794112, + "prompt_kind": "short", + "t_relative_s": 8029.499191750307 + }, + { + "decode_latency_s": 0.06803398956253659, + "kv_live_bytes": 462274560, + "latency_s": 2.1793037080205977, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72810496, + "prompt_kind": "long", + "t_relative_s": 8034.509240750223 + }, + { + "decode_latency_s": 0.06021203125419561, + "kv_live_bytes": 462274560, + "latency_s": 1.927762750070542, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72679424, + "prompt_kind": "short", + "t_relative_s": 8039.51932995813 + }, + { + "decode_latency_s": 0.06818226953328121, + "kv_live_bytes": 462274560, + "latency_s": 2.184154541697353, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72728576, + "prompt_kind": "long", + "t_relative_s": 8044.529414417222 + }, + { + "decode_latency_s": 0.06101706769550219, + "kv_live_bytes": 462274560, + "latency_s": 1.953466999810189, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72220672, + "prompt_kind": "short", + "t_relative_s": 8049.539451166987 + }, + { + "decode_latency_s": 0.06817223175312392, + "kv_live_bytes": 462274560, + "latency_s": 2.183917374815792, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72171520, + "prompt_kind": "long", + "t_relative_s": 8054.549535833299 + }, + { + "decode_latency_s": 0.06073467187525239, + "kv_live_bytes": 462274560, + "latency_s": 1.9443256659433246, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72761344, + "prompt_kind": "short", + "t_relative_s": 8059.55962758325 + }, + { + "decode_latency_s": 0.06746205336821731, + "kv_live_bytes": 462274560, + "latency_s": 2.1614432078786194, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72777728, + "prompt_kind": "long", + "t_relative_s": 8064.5647905422375 + }, + { + "decode_latency_s": 0.06033393096004147, + "kv_live_bytes": 462274560, + "latency_s": 1.931588708423078, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73269248, + "prompt_kind": "short", + "t_relative_s": 8069.574871625286 + }, + { + "decode_latency_s": 0.06809265365882311, + "kv_live_bytes": 462274560, + "latency_s": 2.180839290842414, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73400320, + "prompt_kind": "long", + "t_relative_s": 8074.5849454579875 + }, + { + "decode_latency_s": 0.060937022120924667, + "kv_live_bytes": 462274560, + "latency_s": 1.9509436669759452, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "short", + "t_relative_s": 8079.594994333107 + }, + { + "decode_latency_s": 0.09673426431254484, + "kv_live_bytes": 462274560, + "latency_s": 3.096962624695152, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69222400, + "prompt_kind": "long", + "t_relative_s": 8084.597600375302 + }, + { + "decode_latency_s": 0.06122827602666803, + "kv_live_bytes": 462274560, + "latency_s": 1.9604332502931356, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70057984, + "prompt_kind": "short", + "t_relative_s": 8093.624274708331 + }, + { + "decode_latency_s": 0.06823580077616498, + "kv_live_bytes": 462274560, + "latency_s": 2.1861744159832597, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70107136, + "prompt_kind": "long", + "t_relative_s": 8098.6343231252395 + }, + { + "decode_latency_s": 0.060360002622473985, + "kv_live_bytes": 462274560, + "latency_s": 1.9324573748745024, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70565888, + "prompt_kind": "short", + "t_relative_s": 8103.644407042302 + }, + { + "decode_latency_s": 0.06804121221648529, + "kv_live_bytes": 462274560, + "latency_s": 2.178840709384531, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70631424, + "prompt_kind": "long", + "t_relative_s": 8108.654479625169 + }, + { + "decode_latency_s": 0.06046320703171659, + "kv_live_bytes": 462274560, + "latency_s": 1.935659249778837, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70647808, + "prompt_kind": "short", + "t_relative_s": 8113.664538625162 + }, + { + "decode_latency_s": 0.06747037630702835, + "kv_live_bytes": 462274560, + "latency_s": 2.161805334035307, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70107136, + "prompt_kind": "long", + "t_relative_s": 8118.674604458269 + }, + { + "decode_latency_s": 0.06052086850104388, + "kv_live_bytes": 462274560, + "latency_s": 1.9374380423687398, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70696960, + "prompt_kind": "short", + "t_relative_s": 8123.6846809582785 + }, + { + "decode_latency_s": 0.06845882290508598, + "kv_live_bytes": 462274560, + "latency_s": 2.192663208115846, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70565888, + "prompt_kind": "long", + "t_relative_s": 8128.694736250211 + }, + { + "decode_latency_s": 0.06032472528750077, + "kv_live_bytes": 462274560, + "latency_s": 1.9313374580815434, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70483968, + "prompt_kind": "short", + "t_relative_s": 8133.704805542249 + }, + { + "decode_latency_s": 0.06816847655863967, + "kv_live_bytes": 462274560, + "latency_s": 2.183418875094503, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71057408, + "prompt_kind": "long", + "t_relative_s": 8138.714887917042 + }, + { + "decode_latency_s": 0.0606751653685933, + "kv_live_bytes": 462274560, + "latency_s": 1.9424505410715938, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70565888, + "prompt_kind": "short", + "t_relative_s": 8143.724253542256 + }, + { + "decode_latency_s": 0.13867930209380575, + "kv_live_bytes": 462274560, + "latency_s": 4.440033750142902, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68861952, + "prompt_kind": "long", + "t_relative_s": 8148.733001917135 + }, + { + "decode_latency_s": 0.060484865884063765, + "kv_live_bytes": 462274560, + "latency_s": 1.9365692920982838, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69468160, + "prompt_kind": "short", + "t_relative_s": 8157.910600916948 + }, + { + "decode_latency_s": 0.0680285156267928, + "kv_live_bytes": 462274560, + "latency_s": 2.178443167358637, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70057984, + "prompt_kind": "long", + "t_relative_s": 8162.920691625215 + }, + { + "decode_latency_s": 0.0607202812389005, + "kv_live_bytes": 462274560, + "latency_s": 1.9438787498511374, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70615040, + "prompt_kind": "short", + "t_relative_s": 8167.930750458036 + }, + { + "decode_latency_s": 0.060664726566756144, + "kv_live_bytes": 462274560, + "latency_s": 1.9425597498193383, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70582272, + "prompt_kind": "long", + "t_relative_s": 8172.940872125328 + }, + { + "decode_latency_s": 0.06091751043277327, + "kv_live_bytes": 462274560, + "latency_s": 1.9502572501078248, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70713344, + "prompt_kind": "short", + "t_relative_s": 8177.950466041919 + }, + { + "decode_latency_s": 0.06830218358663842, + "kv_live_bytes": 462274560, + "latency_s": 2.187994208186865, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70795264, + "prompt_kind": "long", + "t_relative_s": 8182.960516917054 + }, + { + "decode_latency_s": 0.06039163544483017, + "kv_live_bytes": 462274560, + "latency_s": 1.9334478746168315, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70893568, + "prompt_kind": "short", + "t_relative_s": 8187.970594292041 + }, + { + "decode_latency_s": 0.06834077996609267, + "kv_live_bytes": 462274560, + "latency_s": 2.1886161249130964, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70975488, + "prompt_kind": "long", + "t_relative_s": 8192.980686792172 + }, + { + "decode_latency_s": 0.060535699216416106, + "kv_live_bytes": 462274560, + "latency_s": 1.9384385831654072, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71548928, + "prompt_kind": "short", + "t_relative_s": 8197.990743208211 + }, + { + "decode_latency_s": 0.06795954296831042, + "kv_live_bytes": 462274560, + "latency_s": 2.177028249949217, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71598080, + "prompt_kind": "long", + "t_relative_s": 8203.000833833124 + }, + { + "decode_latency_s": 0.06083050390589051, + "kv_live_bytes": 462274560, + "latency_s": 1.9473730837926269, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71663616, + "prompt_kind": "short", + "t_relative_s": 8208.01091687521 + }, + { + "decode_latency_s": 0.06789616537571419, + "kv_live_bytes": 462274560, + "latency_s": 2.1747124162502587, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71663616, + "prompt_kind": "long", + "t_relative_s": 8213.02096850006 + }, + { + "decode_latency_s": 0.060354152345098555, + "kv_live_bytes": 462274560, + "latency_s": 1.9322488331235945, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72171520, + "prompt_kind": "short", + "t_relative_s": 8218.031059917063 + }, + { + "decode_latency_s": 0.06841352212359197, + "kv_live_bytes": 462274560, + "latency_s": 2.191247916780412, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71647232, + "prompt_kind": "long", + "t_relative_s": 8223.034062167164 + }, + { + "decode_latency_s": 0.060621220065513626, + "kv_live_bytes": 462274560, + "latency_s": 1.9407467497512698, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72187904, + "prompt_kind": "short", + "t_relative_s": 8228.044128458016 + }, + { + "decode_latency_s": 0.06836829165695235, + "kv_live_bytes": 462274560, + "latency_s": 2.1903685838915408, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72220672, + "prompt_kind": "long", + "t_relative_s": 8233.054218542296 + }, + { + "decode_latency_s": 0.060563901031855494, + "kv_live_bytes": 462274560, + "latency_s": 1.9393640421330929, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72269824, + "prompt_kind": "short", + "t_relative_s": 8238.064291458111 + }, + { + "decode_latency_s": 0.0678738515562145, + "kv_live_bytes": 462274560, + "latency_s": 2.1739826248958707, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72302592, + "prompt_kind": "long", + "t_relative_s": 8243.074342958163 + }, + { + "decode_latency_s": 0.06047992449020967, + "kv_live_bytes": 462274560, + "latency_s": 1.9363095830194652, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72089600, + "prompt_kind": "short", + "t_relative_s": 8248.084428875241 + }, + { + "decode_latency_s": 0.06769566927687265, + "kv_live_bytes": 462274560, + "latency_s": 2.1685368749313056, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72155136, + "prompt_kind": "long", + "t_relative_s": 8253.094512917101 + }, + { + "decode_latency_s": 0.06062965888122562, + "kv_live_bytes": 462274560, + "latency_s": 1.9410345419310033, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72712192, + "prompt_kind": "short", + "t_relative_s": 8258.104562417138 + }, + { + "decode_latency_s": 0.06768497005396057, + "kv_live_bytes": 462274560, + "latency_s": 2.168649291153997, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72810496, + "prompt_kind": "long", + "t_relative_s": 8263.114631250035 + }, + { + "decode_latency_s": 0.06328573178325314, + "kv_live_bytes": 462274560, + "latency_s": 2.026108083780855, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70107136, + "prompt_kind": "short", + "t_relative_s": 8268.124731500167 + }, + { + "decode_latency_s": 0.06765979036572389, + "kv_live_bytes": 462274560, + "latency_s": 2.16738195810467, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72024064, + "prompt_kind": "long", + "t_relative_s": 8273.13481812505 + }, + { + "decode_latency_s": 0.060715468745911494, + "kv_live_bytes": 462274560, + "latency_s": 1.9437432498671114, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72613888, + "prompt_kind": "short", + "t_relative_s": 8278.144875457976 + }, + { + "decode_latency_s": 0.06776206768699922, + "kv_live_bytes": 462274560, + "latency_s": 2.170744334347546, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "long", + "t_relative_s": 8283.154923500028 + }, + { + "decode_latency_s": 0.06050607420911547, + "kv_live_bytes": 462274560, + "latency_s": 1.9379326249472797, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73220096, + "prompt_kind": "short", + "t_relative_s": 8288.164989542216 + }, + { + "decode_latency_s": 0.06804925650067162, + "kv_live_bytes": 462274560, + "latency_s": 2.1792804999276996, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73121792, + "prompt_kind": "long", + "t_relative_s": 8293.175065125339 + }, + { + "decode_latency_s": 0.06048960027692374, + "kv_live_bytes": 462274560, + "latency_s": 1.9365565003827214, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73023488, + "prompt_kind": "short", + "t_relative_s": 8298.185120542068 + }, + { + "decode_latency_s": 0.06753730731725227, + "kv_live_bytes": 462274560, + "latency_s": 2.1630190829746425, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73105408, + "prompt_kind": "long", + "t_relative_s": 8303.19520016713 + }, + { + "decode_latency_s": 0.06054709374438971, + "kv_live_bytes": 462274560, + "latency_s": 1.938312916085124, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73646080, + "prompt_kind": "short", + "t_relative_s": 8308.205280708149 + }, + { + "decode_latency_s": 0.060369542974513024, + "kv_live_bytes": 462274560, + "latency_s": 1.9327880409546196, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73678848, + "prompt_kind": "long", + "t_relative_s": 8313.215333207976 + }, + { + "decode_latency_s": 0.06053461068950128, + "kv_live_bytes": 462274560, + "latency_s": 1.9385634581558406, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73760768, + "prompt_kind": "short", + "t_relative_s": 8318.22539854236 + }, + { + "decode_latency_s": 0.06803999478870537, + "kv_live_bytes": 462274560, + "latency_s": 2.178991583175957, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73121792, + "prompt_kind": "long", + "t_relative_s": 8323.23548120819 + }, + { + "decode_latency_s": 0.06039005989441648, + "kv_live_bytes": 462274560, + "latency_s": 1.9333218336105347, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72531968, + "prompt_kind": "short", + "t_relative_s": 8328.245531542227 + }, + { + "decode_latency_s": 0.06789514713454992, + "kv_live_bytes": 462274560, + "latency_s": 2.174722250085324, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73203712, + "prompt_kind": "long", + "t_relative_s": 8333.255611458328 + }, + { + "decode_latency_s": 0.06413711978530046, + "kv_live_bytes": 462274560, + "latency_s": 2.0532800406217575, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72744960, + "prompt_kind": "short", + "t_relative_s": 8338.265666374937 + }, + { + "decode_latency_s": 0.06753101172216702, + "kv_live_bytes": 462274560, + "latency_s": 2.1631617075763643, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73072640, + "prompt_kind": "long", + "t_relative_s": 8343.275740833022 + }, + { + "decode_latency_s": 0.06037449087307323, + "kv_live_bytes": 462274560, + "latency_s": 1.9328967495821416, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73744384, + "prompt_kind": "short", + "t_relative_s": 8348.285790707916 + }, + { + "decode_latency_s": 0.0678766132768942, + "kv_live_bytes": 462274560, + "latency_s": 2.174216417130083, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73826304, + "prompt_kind": "long", + "t_relative_s": 8353.295872667339 + }, + { + "decode_latency_s": 0.06074533856008202, + "kv_live_bytes": 462274560, + "latency_s": 1.9446433749981225, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73170944, + "prompt_kind": "short", + "t_relative_s": 8358.305988583248 + }, + { + "decode_latency_s": 0.06800317317538429, + "kv_live_bytes": 462274560, + "latency_s": 2.1786476247943938, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73662464, + "prompt_kind": "long", + "t_relative_s": 8363.31604283303 + }, + { + "decode_latency_s": 0.0604612421739148, + "kv_live_bytes": 462274560, + "latency_s": 1.9356225421652198, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73744384, + "prompt_kind": "short", + "t_relative_s": 8368.326121792197 + }, + { + "decode_latency_s": 0.06780699477531016, + "kv_live_bytes": 462274560, + "latency_s": 2.17151654092595, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73678848, + "prompt_kind": "long", + "t_relative_s": 8373.336200000253 + }, + { + "decode_latency_s": 0.06049679817806464, + "kv_live_bytes": 462274560, + "latency_s": 1.9367919997312129, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74219520, + "prompt_kind": "short", + "t_relative_s": 8378.346251333132 + }, + { + "decode_latency_s": 0.06165771093219519, + "kv_live_bytes": 462274560, + "latency_s": 1.9740874161943793, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73105408, + "prompt_kind": "long", + "t_relative_s": 8383.356326542329 + }, + { + "decode_latency_s": 0.060822087223641574, + "kv_live_bytes": 462274560, + "latency_s": 1.947138000279665, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72974336, + "prompt_kind": "short", + "t_relative_s": 8388.360391250346 + }, + { + "decode_latency_s": 0.06806772135314532, + "kv_live_bytes": 462274560, + "latency_s": 2.1807074160315096, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73744384, + "prompt_kind": "long", + "t_relative_s": 8393.370443375316 + }, + { + "decode_latency_s": 0.06054451434465591, + "kv_live_bytes": 462274560, + "latency_s": 1.9382075420580804, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74301440, + "prompt_kind": "short", + "t_relative_s": 8398.37655858323 + }, + { + "decode_latency_s": 0.06826565884693991, + "kv_live_bytes": 462274560, + "latency_s": 2.186602917034179, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74235904, + "prompt_kind": "long", + "t_relative_s": 8403.386608958244 + }, + { + "decode_latency_s": 0.0602821250067791, + "kv_live_bytes": 462274560, + "latency_s": 1.9299305831082165, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74809344, + "prompt_kind": "short", + "t_relative_s": 8408.396659417078 + }, + { + "decode_latency_s": 0.06049435547902249, + "kv_live_bytes": 462274560, + "latency_s": 1.936731334310025, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74907648, + "prompt_kind": "long", + "t_relative_s": 8413.406746208202 + }, + { + "decode_latency_s": 0.06069669789576437, + "kv_live_bytes": 462274560, + "latency_s": 1.9430882916785777, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75022336, + "prompt_kind": "short", + "t_relative_s": 8418.416791874915 + }, + { + "decode_latency_s": 0.06778094659966882, + "kv_live_bytes": 462274560, + "latency_s": 2.1710820831358433, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75071488, + "prompt_kind": "long", + "t_relative_s": 8423.42683629226 + }, + { + "decode_latency_s": 0.06060481118038297, + "kv_live_bytes": 462274560, + "latency_s": 1.9410095838829875, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75595776, + "prompt_kind": "short", + "t_relative_s": 8428.436904042028 + }, + { + "decode_latency_s": 0.060327238286845386, + "kv_live_bytes": 462274560, + "latency_s": 1.9313815832138062, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75612160, + "prompt_kind": "long", + "t_relative_s": 8433.446952749975 + }, + { + "decode_latency_s": 0.06041054424713366, + "kv_live_bytes": 462274560, + "latency_s": 1.9340403750538826, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75612160, + "prompt_kind": "short", + "t_relative_s": 8438.45701083308 + }, + { + "decode_latency_s": 0.06869993881264236, + "kv_live_bytes": 462274560, + "latency_s": 2.200495250057429, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75677696, + "prompt_kind": "long", + "t_relative_s": 8443.467091292143 + }, + { + "decode_latency_s": 0.06082927083480172, + "kv_live_bytes": 462274560, + "latency_s": 1.9473651251755655, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75202560, + "prompt_kind": "short", + "t_relative_s": 8448.47715683328 + }, + { + "decode_latency_s": 0.06325447134440765, + "kv_live_bytes": 462274560, + "latency_s": 2.0255441670306027, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75169792, + "prompt_kind": "long", + "t_relative_s": 8453.487215583213 + }, + { + "decode_latency_s": 0.06669915234670043, + "kv_live_bytes": 462274560, + "latency_s": 2.1362388329580426, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75792384, + "prompt_kind": "short", + "t_relative_s": 8458.497303333133 + }, + { + "decode_latency_s": 0.06834405859990511, + "kv_live_bytes": 462274560, + "latency_s": 2.1888032909482718, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75595776, + "prompt_kind": "long", + "t_relative_s": 8463.507363833021 + }, + { + "decode_latency_s": 0.060347509119310416, + "kv_live_bytes": 462274560, + "latency_s": 1.931992833968252, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75677696, + "prompt_kind": "short", + "t_relative_s": 8468.51743341703 + }, + { + "decode_latency_s": 0.06816218487801962, + "kv_live_bytes": 462274560, + "latency_s": 2.1831147079356015, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76152832, + "prompt_kind": "long", + "t_relative_s": 8473.527507999912 + }, + { + "decode_latency_s": 0.06104916274489369, + "kv_live_bytes": 462274560, + "latency_s": 1.9544425839558244, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75644928, + "prompt_kind": "short", + "t_relative_s": 8478.537570333108 + }, + { + "decode_latency_s": 0.06826502474723384, + "kv_live_bytes": 462274560, + "latency_s": 2.1875905408523977, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75677696, + "prompt_kind": "long", + "t_relative_s": 8483.54764600005 + }, + { + "decode_latency_s": 0.060725225252099335, + "kv_live_bytes": 462274560, + "latency_s": 1.9440460000187159, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75710464, + "prompt_kind": "short", + "t_relative_s": 8488.557735417038 + }, + { + "decode_latency_s": 0.06722948310198262, + "kv_live_bytes": 462274560, + "latency_s": 2.153201666660607, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75563008, + "prompt_kind": "long", + "t_relative_s": 8493.567804791965 + }, + { + "decode_latency_s": 0.06043748959200457, + "kv_live_bytes": 462274560, + "latency_s": 1.9348694160580635, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75595776, + "prompt_kind": "short", + "t_relative_s": 8498.577876375057 + }, + { + "decode_latency_s": 0.06854333852243144, + "kv_live_bytes": 462274560, + "latency_s": 2.1953394170850515, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75530240, + "prompt_kind": "long", + "t_relative_s": 8503.587955917232 + }, + { + "decode_latency_s": 0.060505403656861745, + "kv_live_bytes": 462274560, + "latency_s": 1.9370564171113074, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75661312, + "prompt_kind": "short", + "t_relative_s": 8508.598002583254 + }, + { + "decode_latency_s": 0.06819016014924273, + "kv_live_bytes": 462274560, + "latency_s": 2.184612500015646, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75661312, + "prompt_kind": "long", + "t_relative_s": 8513.608093082905 + }, + { + "decode_latency_s": 0.06665089452872053, + "kv_live_bytes": 462274560, + "latency_s": 2.134750208351761, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75694080, + "prompt_kind": "short", + "t_relative_s": 8518.618168250192 + }, + { + "decode_latency_s": 0.1116988776193466, + "kv_live_bytes": 462274560, + "latency_s": 3.575291041750461, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 63668224, + "prompt_kind": "long", + "t_relative_s": 8523.622669707984 + }, + { + "decode_latency_s": 0.06080760678742081, + "kv_live_bytes": 462274560, + "latency_s": 1.9467477910220623, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66404352, + "prompt_kind": "short", + "t_relative_s": 8533.619501917157 + }, + { + "decode_latency_s": 0.0681567460851511, + "kv_live_bytes": 462274560, + "latency_s": 2.1827893746085465, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69795840, + "prompt_kind": "long", + "t_relative_s": 8538.629539625254 + }, + { + "decode_latency_s": 0.060327749990392476, + "kv_live_bytes": 462274560, + "latency_s": 1.9314213329926133, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69795840, + "prompt_kind": "short", + "t_relative_s": 8543.639589332975 + }, + { + "decode_latency_s": 0.060506355468533, + "kv_live_bytes": 462274560, + "latency_s": 1.9373628753237426, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69910528, + "prompt_kind": "long", + "t_relative_s": 8548.649662000127 + }, + { + "decode_latency_s": 0.06074310546682682, + "kv_live_bytes": 462274560, + "latency_s": 1.9446475417353213, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70615040, + "prompt_kind": "short", + "t_relative_s": 8553.659714375157 + }, + { + "decode_latency_s": 0.06782161846058443, + "kv_live_bytes": 462274560, + "latency_s": 2.1728416667319834, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70713344, + "prompt_kind": "long", + "t_relative_s": 8558.66977712512 + }, + { + "decode_latency_s": 0.06138978907256387, + "kv_live_bytes": 462274560, + "latency_s": 1.9652810832485557, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71303168, + "prompt_kind": "short", + "t_relative_s": 8563.679863000289 + }, + { + "decode_latency_s": 0.06790707549953368, + "kv_live_bytes": 462274560, + "latency_s": 2.175723375286907, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70680576, + "prompt_kind": "long", + "t_relative_s": 8568.689921125304 + }, + { + "decode_latency_s": 0.060393585939891636, + "kv_live_bytes": 462274560, + "latency_s": 1.933534708339721, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70582272, + "prompt_kind": "short", + "t_relative_s": 8573.699991708156 + }, + { + "decode_latency_s": 0.06034038022335153, + "kv_live_bytes": 462274560, + "latency_s": 1.9318391671404243, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70500352, + "prompt_kind": "long", + "t_relative_s": 8578.71006920794 + }, + { + "decode_latency_s": 0.06073480338091031, + "kv_live_bytes": 462274560, + "latency_s": 1.9444614578969777, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71090176, + "prompt_kind": "short", + "t_relative_s": 8583.720123542007 + }, + { + "decode_latency_s": 0.06784913540468551, + "kv_live_bytes": 462274560, + "latency_s": 2.173183666076511, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71057408, + "prompt_kind": "long", + "t_relative_s": 8588.730196875054 + }, + { + "decode_latency_s": 0.0606877760583302, + "kv_live_bytes": 462274560, + "latency_s": 1.9427752080373466, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71647232, + "prompt_kind": "short", + "t_relative_s": 8593.740277375095 + }, + { + "decode_latency_s": 0.06810818228404969, + "kv_live_bytes": 462274560, + "latency_s": 2.1819256246089935, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71581696, + "prompt_kind": "long", + "t_relative_s": 8598.750319499988 + }, + { + "decode_latency_s": 0.06048133593867533, + "kv_live_bytes": 462274560, + "latency_s": 1.9363515838049352, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71139328, + "prompt_kind": "short", + "t_relative_s": 8603.756394000258 + }, + { + "decode_latency_s": 0.06851033722341526, + "kv_live_bytes": 462274560, + "latency_s": 2.1943855830468237, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71155712, + "prompt_kind": "long", + "t_relative_s": 8608.766474792268 + }, + { + "decode_latency_s": 0.06064219401741866, + "kv_live_bytes": 462274560, + "latency_s": 1.9414806249551475, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71712768, + "prompt_kind": "short", + "t_relative_s": 8613.77652179217 + }, + { + "decode_latency_s": 0.06095402083883528, + "kv_live_bytes": 462274560, + "latency_s": 1.9515986670739949, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71155712, + "prompt_kind": "long", + "t_relative_s": 8618.786611082964 + }, + { + "decode_latency_s": 0.0606179361930117, + "kv_live_bytes": 462274560, + "latency_s": 1.9406247078441083, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70959104, + "prompt_kind": "short", + "t_relative_s": 8623.79665758321 + }, + { + "decode_latency_s": 0.06799052606220357, + "kv_live_bytes": 462274560, + "latency_s": 2.1775244157761335, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70418432, + "prompt_kind": "long", + "t_relative_s": 8628.80673733307 + }, + { + "decode_latency_s": 0.06696117318642791, + "kv_live_bytes": 462274560, + "latency_s": 2.1443134169094265, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71221248, + "prompt_kind": "short", + "t_relative_s": 8633.816777457949 + }, + { + "decode_latency_s": 0.06827120180241764, + "kv_live_bytes": 462274560, + "latency_s": 2.186854540836066, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71745536, + "prompt_kind": "long", + "t_relative_s": 8638.826860208064 + }, + { + "decode_latency_s": 0.06052709897630848, + "kv_live_bytes": 462274560, + "latency_s": 1.9376454171724617, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72335360, + "prompt_kind": "short", + "t_relative_s": 8643.836949625053 + }, + { + "decode_latency_s": 0.0680368945322698, + "kv_live_bytes": 462274560, + "latency_s": 2.179319457616657, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72400896, + "prompt_kind": "long", + "t_relative_s": 8648.847007791977 + }, + { + "decode_latency_s": 0.06038997393625323, + "kv_live_bytes": 462274560, + "latency_s": 1.9333949168212712, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72237056, + "prompt_kind": "short", + "t_relative_s": 8653.857069374993 + }, + { + "decode_latency_s": 0.06794967578025535, + "kv_live_bytes": 462274560, + "latency_s": 2.176081624813378, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72302592, + "prompt_kind": "long", + "t_relative_s": 8658.867130000144 + }, + { + "decode_latency_s": 0.06721401953836903, + "kv_live_bytes": 462274560, + "latency_s": 2.152751292102039, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72417280, + "prompt_kind": "short", + "t_relative_s": 8663.87719587516 + }, + { + "decode_latency_s": 0.06774355209199712, + "kv_live_bytes": 462274560, + "latency_s": 2.1702105002477765, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72482816, + "prompt_kind": "long", + "t_relative_s": 8668.887263750192 + }, + { + "decode_latency_s": 0.060533014315296896, + "kv_live_bytes": 462274560, + "latency_s": 1.9378969580866396, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73023488, + "prompt_kind": "short", + "t_relative_s": 8673.897365666926 + }, + { + "decode_latency_s": 0.06782767969707493, + "kv_live_bytes": 462274560, + "latency_s": 2.1728578340262175, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72990720, + "prompt_kind": "long", + "t_relative_s": 8678.907417792361 + }, + { + "decode_latency_s": 0.0637326926807873, + "kv_live_bytes": 462274560, + "latency_s": 2.0407245829701424, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69812224, + "prompt_kind": "short", + "t_relative_s": 8683.917497167364 + }, + { + "decode_latency_s": 0.06801529166114051, + "kv_live_bytes": 462274560, + "latency_s": 2.1789779998362064, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72630272, + "prompt_kind": "long", + "t_relative_s": 8688.927570917178 + }, + { + "decode_latency_s": 0.060522031257278286, + "kv_live_bytes": 462274560, + "latency_s": 1.937463500071317, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72679424, + "prompt_kind": "short", + "t_relative_s": 8693.937673208304 + }, + { + "decode_latency_s": 0.06791585808969103, + "kv_live_bytes": 462274560, + "latency_s": 2.1755359582602978, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72761344, + "prompt_kind": "long", + "t_relative_s": 8698.947723249905 + }, + { + "decode_latency_s": 0.06036620441591367, + "kv_live_bytes": 462274560, + "latency_s": 1.932653000112623, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72744960, + "prompt_kind": "short", + "t_relative_s": 8703.95778341731 + }, + { + "decode_latency_s": 0.06826430989895016, + "kv_live_bytes": 462274560, + "latency_s": 2.1869747089222074, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73334784, + "prompt_kind": "long", + "t_relative_s": 8708.967857500073 + }, + { + "decode_latency_s": 0.06063365102454554, + "kv_live_bytes": 462274560, + "latency_s": 1.9411051669158041, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73957376, + "prompt_kind": "short", + "t_relative_s": 8713.97791529214 + }, + { + "decode_latency_s": 0.06833312239905354, + "kv_live_bytes": 462274560, + "latency_s": 2.188863333314657, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74022912, + "prompt_kind": "long", + "t_relative_s": 8718.987746917177 + }, + { + "decode_latency_s": 0.06056473568605725, + "kv_live_bytes": 462274560, + "latency_s": 1.9388889172114432, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74072064, + "prompt_kind": "short", + "t_relative_s": 8723.99782570824 + }, + { + "decode_latency_s": 0.0680312395998044, + "kv_live_bytes": 462274560, + "latency_s": 2.178698708768934, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73531392, + "prompt_kind": "long", + "t_relative_s": 8729.007870458066 + }, + { + "decode_latency_s": 0.060596028648433276, + "kv_live_bytes": 462274560, + "latency_s": 1.9399442500434816, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73498624, + "prompt_kind": "short", + "t_relative_s": 8734.0179101252 + }, + { + "decode_latency_s": 0.06919274087704252, + "kv_live_bytes": 462274560, + "latency_s": 2.216787583194673, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73940992, + "prompt_kind": "long", + "t_relative_s": 8739.027964542154 + }, + { + "decode_latency_s": 0.06309192709159106, + "kv_live_bytes": 462274560, + "latency_s": 2.019834291189909, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73056256, + "prompt_kind": "short", + "t_relative_s": 8744.038048750255 + }, + { + "decode_latency_s": 0.0677800703124376, + "kv_live_bytes": 462274560, + "latency_s": 2.1711312090046704, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74137600, + "prompt_kind": "long", + "t_relative_s": 8749.04811520828 + }, + { + "decode_latency_s": 0.06301746352983173, + "kv_live_bytes": 462274560, + "latency_s": 2.0179609581828117, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 37535744, + "prompt_kind": "short", + "t_relative_s": 8754.058195750229 + }, + { + "decode_latency_s": 0.06767724736710079, + "kv_live_bytes": 462274560, + "latency_s": 2.168446875177324, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "long", + "t_relative_s": 8759.068251542281 + }, + { + "decode_latency_s": 0.060397524721338414, + "kv_live_bytes": 462274560, + "latency_s": 1.9336392083205283, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73695232, + "prompt_kind": "short", + "t_relative_s": 8764.078322542366 + }, + { + "decode_latency_s": 0.06793117840425111, + "kv_live_bytes": 462274560, + "latency_s": 2.175332374870777, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73908224, + "prompt_kind": "long", + "t_relative_s": 8769.088396375068 + }, + { + "decode_latency_s": 0.0604411315289326, + "kv_live_bytes": 462274560, + "latency_s": 1.9349941248074174, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73891840, + "prompt_kind": "short", + "t_relative_s": 8774.09177750023 + }, + { + "decode_latency_s": 0.06852447136770934, + "kv_live_bytes": 462274560, + "latency_s": 2.1948566250503063, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73531392, + "prompt_kind": "long", + "t_relative_s": 8779.101856250316 + }, + { + "decode_latency_s": 0.06073101299989503, + "kv_live_bytes": 462274560, + "latency_s": 1.944239666685462, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73580544, + "prompt_kind": "short", + "t_relative_s": 8784.111910042353 + }, + { + "decode_latency_s": 0.0682250143145211, + "kv_live_bytes": 462274560, + "latency_s": 2.1856607496738434, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73711616, + "prompt_kind": "long", + "t_relative_s": 8789.121969917323 + }, + { + "decode_latency_s": 0.06064376822905615, + "kv_live_bytes": 462274560, + "latency_s": 1.941401000134647, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74285056, + "prompt_kind": "short", + "t_relative_s": 8794.132043333258 + }, + { + "decode_latency_s": 0.06798675390018616, + "kv_live_bytes": 462274560, + "latency_s": 2.1771217500790954, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74383360, + "prompt_kind": "long", + "t_relative_s": 8799.142102083191 + }, + { + "decode_latency_s": 0.06180107162799686, + "kv_live_bytes": 462274560, + "latency_s": 1.9784362497739494, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74350592, + "prompt_kind": "short", + "t_relative_s": 8804.152176083066 + }, + { + "decode_latency_s": 0.0673347812553402, + "kv_live_bytes": 462274560, + "latency_s": 2.1569827082566917, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74956800, + "prompt_kind": "long", + "t_relative_s": 8809.162229792215 + }, + { + "decode_latency_s": 0.060505604167701676, + "kv_live_bytes": 462274560, + "latency_s": 1.936975124757737, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75022336, + "prompt_kind": "short", + "t_relative_s": 8814.172303417232 + }, + { + "decode_latency_s": 0.06801467968034558, + "kv_live_bytes": 462274560, + "latency_s": 2.1785592497326434, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73924608, + "prompt_kind": "long", + "t_relative_s": 8819.182404167019 + }, + { + "decode_latency_s": 0.060272549468209036, + "kv_live_bytes": 462274560, + "latency_s": 1.9297020831145346, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74366976, + "prompt_kind": "short", + "t_relative_s": 8824.192460542079 + }, + { + "decode_latency_s": 0.06772245180036407, + "kv_live_bytes": 462274560, + "latency_s": 2.1686204159632325, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74973184, + "prompt_kind": "long", + "t_relative_s": 8829.20253720833 + }, + { + "decode_latency_s": 0.06053095443348866, + "kv_live_bytes": 462274560, + "latency_s": 1.937910791952163, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75513856, + "prompt_kind": "short", + "t_relative_s": 8834.212594000157 + }, + { + "decode_latency_s": 0.06821589716128074, + "kv_live_bytes": 462274560, + "latency_s": 2.1852466254495084, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75038720, + "prompt_kind": "long", + "t_relative_s": 8839.222643833142 + }, + { + "decode_latency_s": 0.060535338532645255, + "kv_live_bytes": 462274560, + "latency_s": 1.9378928747028112, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75071488, + "prompt_kind": "short", + "t_relative_s": 8844.23272600025 + }, + { + "decode_latency_s": 0.060583080718060955, + "kv_live_bytes": 462274560, + "latency_s": 1.9399138749577105, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74465280, + "prompt_kind": "long", + "t_relative_s": 8849.242771500256 + }, + { + "decode_latency_s": 0.06074260674358811, + "kv_live_bytes": 462274560, + "latency_s": 1.9447540417313576, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70041600, + "prompt_kind": "short", + "t_relative_s": 8854.25284849992 + }, + { + "decode_latency_s": 0.06745818100171164, + "kv_live_bytes": 462274560, + "latency_s": 2.1603909581899643, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74022912, + "prompt_kind": "long", + "t_relative_s": 8859.26292704232 + }, + { + "decode_latency_s": 0.06055219272093382, + "kv_live_bytes": 462274560, + "latency_s": 1.938580417074263, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74661888, + "prompt_kind": "short", + "t_relative_s": 8864.273000125308 + }, + { + "decode_latency_s": 0.06491708462999668, + "kv_live_bytes": 462274560, + "latency_s": 2.0782854170538485, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73400320, + "prompt_kind": "long", + "t_relative_s": 8869.283068249933 + }, + { + "decode_latency_s": 0.06300598305824678, + "kv_live_bytes": 462274560, + "latency_s": 2.017277750186622, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68698112, + "prompt_kind": "short", + "t_relative_s": 8874.293147667311 + }, + { + "decode_latency_s": 0.06821177602978423, + "kv_live_bytes": 462274560, + "latency_s": 2.1854515420272946, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73252864, + "prompt_kind": "long", + "t_relative_s": 8879.303246208001 + }, + { + "decode_latency_s": 0.06071849608269986, + "kv_live_bytes": 462274560, + "latency_s": 1.9438389167189598, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72941568, + "prompt_kind": "short", + "t_relative_s": 8884.31332425028 + }, + { + "decode_latency_s": 0.06884665496181697, + "kv_live_bytes": 462274560, + "latency_s": 2.205154957715422, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73383936, + "prompt_kind": "long", + "t_relative_s": 8889.323378083296 + }, + { + "decode_latency_s": 0.06046047915879171, + "kv_live_bytes": 462274560, + "latency_s": 1.9356745830737054, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73940992, + "prompt_kind": "short", + "t_relative_s": 8894.33345287526 + }, + { + "decode_latency_s": 0.06402303646609653, + "kv_live_bytes": 462274560, + "latency_s": 2.050802582874894, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70336512, + "prompt_kind": "long", + "t_relative_s": 8899.343512208201 + }, + { + "decode_latency_s": 0.06097684244741686, + "kv_live_bytes": 462274560, + "latency_s": 1.9521777499467134, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71106560, + "prompt_kind": "short", + "t_relative_s": 8905.491083958186 + }, + { + "decode_latency_s": 0.06850515624682885, + "kv_live_bytes": 462274560, + "latency_s": 2.1941823749803007, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71221248, + "prompt_kind": "long", + "t_relative_s": 8910.501173208002 + }, + { + "decode_latency_s": 0.060823602878372185, + "kv_live_bytes": 462274560, + "latency_s": 1.9471560842357576, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71794688, + "prompt_kind": "short", + "t_relative_s": 8915.511260792147 + }, + { + "decode_latency_s": 0.06792969268281013, + "kv_live_bytes": 462274560, + "latency_s": 2.1758302920497954, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71892992, + "prompt_kind": "long", + "t_relative_s": 8920.521307874937 + }, + { + "decode_latency_s": 0.06073774218384642, + "kv_live_bytes": 462274560, + "latency_s": 1.9443649579770863, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72417280, + "prompt_kind": "short", + "t_relative_s": 8925.531375292223 + }, + { + "decode_latency_s": 0.0680632564908592, + "kv_live_bytes": 462274560, + "latency_s": 2.1796655422076583, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72499200, + "prompt_kind": "long", + "t_relative_s": 8930.54145487491 + }, + { + "decode_latency_s": 0.060317531242617406, + "kv_live_bytes": 462274560, + "latency_s": 1.931067958008498, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72564736, + "prompt_kind": "short", + "t_relative_s": 8935.551489292178 + }, + { + "decode_latency_s": 0.06771092188137118, + "kv_live_bytes": 462274560, + "latency_s": 2.169043666217476, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "long", + "t_relative_s": 8940.561579792295 + }, + { + "decode_latency_s": 0.060521324223373085, + "kv_live_bytes": 462274560, + "latency_s": 1.9374551251530647, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73187328, + "prompt_kind": "short", + "t_relative_s": 8945.571651583072 + }, + { + "decode_latency_s": 0.06040774872235488, + "kv_live_bytes": 462274560, + "latency_s": 1.9347097496502101, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73220096, + "prompt_kind": "long", + "t_relative_s": 8950.581699542236 + }, + { + "decode_latency_s": 0.06053189843078144, + "kv_live_bytes": 462274560, + "latency_s": 1.9379799580201507, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73285632, + "prompt_kind": "short", + "t_relative_s": 8955.591775750276 + }, + { + "decode_latency_s": 0.06048424218897708, + "kv_live_bytes": 462274560, + "latency_s": 1.9364127502776682, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73367552, + "prompt_kind": "long", + "t_relative_s": 8960.601851208135 + }, + { + "decode_latency_s": 0.06027536069450434, + "kv_live_bytes": 462274560, + "latency_s": 1.9296790002845228, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "short", + "t_relative_s": 8965.611904042307 + }, + { + "decode_latency_s": 0.06775979687517975, + "kv_live_bytes": 462274560, + "latency_s": 2.1706132078543305, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73465856, + "prompt_kind": "long", + "t_relative_s": 8970.621988167055 + }, + { + "decode_latency_s": 0.06090552997193299, + "kv_live_bytes": 462274560, + "latency_s": 1.9496766249649227, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73990144, + "prompt_kind": "short", + "t_relative_s": 8975.632063041907 + }, + { + "decode_latency_s": 0.06781045052048285, + "kv_live_bytes": 462274560, + "latency_s": 2.172975040972233, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74055680, + "prompt_kind": "long", + "t_relative_s": 8980.64211799996 + }, + { + "decode_latency_s": 0.060409111974877305, + "kv_live_bytes": 462274560, + "latency_s": 1.9339992920868099, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74055680, + "prompt_kind": "short", + "t_relative_s": 8985.652178042103 + }, + { + "decode_latency_s": 0.06694165234512184, + "kv_live_bytes": 462274560, + "latency_s": 2.1439077081158757, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 65978368, + "prompt_kind": "long", + "t_relative_s": 8990.662244332954 + }, + { + "decode_latency_s": 0.0607876549474895, + "kv_live_bytes": 462274560, + "latency_s": 1.9461464169435203, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70467584, + "prompt_kind": "short", + "t_relative_s": 8995.672321625054 + }, + { + "decode_latency_s": 0.06743762371479534, + "kv_live_bytes": 462274560, + "latency_s": 2.1603376250714064, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72974336, + "prompt_kind": "long", + "t_relative_s": 9000.682382042054 + }, + { + "decode_latency_s": 0.060634571593254805, + "kv_live_bytes": 462274560, + "latency_s": 1.941194707993418, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73285632, + "prompt_kind": "short", + "t_relative_s": 9005.69244466722 + }, + { + "decode_latency_s": 0.06780964975769166, + "kv_live_bytes": 462274560, + "latency_s": 2.171483499929309, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73351168, + "prompt_kind": "long", + "t_relative_s": 9010.702525583096 + }, + { + "decode_latency_s": 0.06060421746224165, + "kv_live_bytes": 462274560, + "latency_s": 1.940200749784708, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73973760, + "prompt_kind": "short", + "t_relative_s": 9015.712575333193 + }, + { + "decode_latency_s": 0.06788979559496511, + "kv_live_bytes": 462274560, + "latency_s": 2.1747916657477617, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74153984, + "prompt_kind": "long", + "t_relative_s": 9020.722654667217 + }, + { + "decode_latency_s": 0.06648079687147401, + "kv_live_bytes": 462274560, + "latency_s": 2.1290982919745147, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74268672, + "prompt_kind": "short", + "t_relative_s": 9025.732764542103 + }, + { + "decode_latency_s": 0.06832660936925095, + "kv_live_bytes": 462274560, + "latency_s": 2.188561666291207, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74137600, + "prompt_kind": "long", + "t_relative_s": 9030.742819332983 + }, + { + "decode_latency_s": 0.060453195314039476, + "kv_live_bytes": 462274560, + "latency_s": 1.9353621248155832, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74072064, + "prompt_kind": "short", + "t_relative_s": 9035.752900208347 + }, + { + "decode_latency_s": 0.06784740624425467, + "kv_live_bytes": 462274560, + "latency_s": 2.1728655421175063, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74252288, + "prompt_kind": "long", + "t_relative_s": 9040.76296620816 + }, + { + "decode_latency_s": 0.06700635678134859, + "kv_live_bytes": 462274560, + "latency_s": 2.145736124832183, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74874880, + "prompt_kind": "short", + "t_relative_s": 9045.773025000002 + }, + { + "decode_latency_s": 0.0671588606928708, + "kv_live_bytes": 462274560, + "latency_s": 2.150950583163649, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74842112, + "prompt_kind": "long", + "t_relative_s": 9050.783116792329 + }, + { + "decode_latency_s": 0.06606869140523486, + "kv_live_bytes": 462274560, + "latency_s": 2.115923209115863, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74989568, + "prompt_kind": "short", + "t_relative_s": 9055.793182957917 + }, + { + "decode_latency_s": 0.06772060805815272, + "kv_live_bytes": 462274560, + "latency_s": 2.1700042500160635, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74940416, + "prompt_kind": "long", + "t_relative_s": 9060.803229582962 + }, + { + "decode_latency_s": 0.06049445184180513, + "kv_live_bytes": 462274560, + "latency_s": 1.9367032912559807, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75202560, + "prompt_kind": "short", + "t_relative_s": 9065.81334408326 + }, + { + "decode_latency_s": 0.06091388281492982, + "kv_live_bytes": 462274560, + "latency_s": 1.9504820420406759, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74547200, + "prompt_kind": "long", + "t_relative_s": 9070.823413208127 + }, + { + "decode_latency_s": 0.060525118504301645, + "kv_live_bytes": 462274560, + "latency_s": 1.937755708117038, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74596352, + "prompt_kind": "short", + "t_relative_s": 9075.833463000134 + }, + { + "decode_latency_s": 0.0680617734324187, + "kv_live_bytes": 462274560, + "latency_s": 2.1807733750902116, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75235328, + "prompt_kind": "long", + "t_relative_s": 9080.843549333047 + }, + { + "decode_latency_s": 0.0607366601616377, + "kv_live_bytes": 462274560, + "latency_s": 1.9443773748353124, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75431936, + "prompt_kind": "short", + "t_relative_s": 9085.853649500292 + }, + { + "decode_latency_s": 0.0678208814933896, + "kv_live_bytes": 462274560, + "latency_s": 2.173624916933477, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75579392, + "prompt_kind": "long", + "t_relative_s": 9090.863711792044 + }, + { + "decode_latency_s": 0.060520257815369405, + "kv_live_bytes": 462274560, + "latency_s": 1.9374919580295682, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75661312, + "prompt_kind": "short", + "t_relative_s": 9095.873813542072 + }, + { + "decode_latency_s": 0.06813572657119948, + "kv_live_bytes": 462274560, + "latency_s": 2.1829580408520997, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74416128, + "prompt_kind": "long", + "t_relative_s": 9100.883900958113 + }, + { + "decode_latency_s": 0.06066863281012047, + "kv_live_bytes": 462274560, + "latency_s": 1.9422544999979436, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75104256, + "prompt_kind": "short", + "t_relative_s": 9105.893984083086 + }, + { + "decode_latency_s": 0.06886635937553365, + "kv_live_bytes": 462274560, + "latency_s": 2.2062189583666623, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75431936, + "prompt_kind": "long", + "t_relative_s": 9110.904031666927 + }, + { + "decode_latency_s": 0.060272785151028074, + "kv_live_bytes": 462274560, + "latency_s": 1.9297472909092903, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74809344, + "prompt_kind": "short", + "t_relative_s": 9115.914102375042 + }, + { + "decode_latency_s": 0.06783394400554243, + "kv_live_bytes": 462274560, + "latency_s": 2.173052624799311, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74891264, + "prompt_kind": "long", + "t_relative_s": 9120.924186292104 + }, + { + "decode_latency_s": 0.06698399347078521, + "kv_live_bytes": 462274560, + "latency_s": 2.145289999898523, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75481088, + "prompt_kind": "short", + "t_relative_s": 9125.934237292036 + }, + { + "decode_latency_s": 0.0675167903827969, + "kv_live_bytes": 462274560, + "latency_s": 2.1630049580708146, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75563008, + "prompt_kind": "long", + "t_relative_s": 9130.944310875144 + }, + { + "decode_latency_s": 0.06089839975174982, + "kv_live_bytes": 462274560, + "latency_s": 1.9495169171132147, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76152832, + "prompt_kind": "short", + "t_relative_s": 9135.954391000327 + }, + { + "decode_latency_s": 0.06779149218345992, + "kv_live_bytes": 462274560, + "latency_s": 2.1716235000640154, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76054528, + "prompt_kind": "long", + "t_relative_s": 9140.964441708289 + }, + { + "decode_latency_s": 0.060393460938939825, + "kv_live_bytes": 462274560, + "latency_s": 1.9335162080824375, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75235328, + "prompt_kind": "short", + "t_relative_s": 9145.974503458012 + }, + { + "decode_latency_s": 0.06828545313328505, + "kv_live_bytes": 462274560, + "latency_s": 2.1871182080358267, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75317248, + "prompt_kind": "long", + "t_relative_s": 9150.984594583046 + }, + { + "decode_latency_s": 0.060806497363955714, + "kv_live_bytes": 462274560, + "latency_s": 1.9466765830293298, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75431936, + "prompt_kind": "short", + "t_relative_s": 9155.994654917158 + }, + { + "decode_latency_s": 0.0680529140663566, + "kv_live_bytes": 462274560, + "latency_s": 2.1802960000932217, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75481088, + "prompt_kind": "long", + "t_relative_s": 9161.004723750055 + }, + { + "decode_latency_s": 0.06671669399656821, + "kv_live_bytes": 462274560, + "latency_s": 2.1367303328588605, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75366400, + "prompt_kind": "short", + "t_relative_s": 9166.014796667267 + }, + { + "decode_latency_s": 0.06771039453451522, + "kv_live_bytes": 462274560, + "latency_s": 2.1689282502047718, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75382784, + "prompt_kind": "long", + "t_relative_s": 9171.024856000207 + }, + { + "decode_latency_s": 0.06064581511600409, + "kv_live_bytes": 462274560, + "latency_s": 1.9415915827266872, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74825728, + "prompt_kind": "short", + "t_relative_s": 9176.034940292127 + }, + { + "decode_latency_s": 0.06726530859305058, + "kv_live_bytes": 462274560, + "latency_s": 2.1545755406841636, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75399168, + "prompt_kind": "long", + "t_relative_s": 9181.045012833085 + }, + { + "decode_latency_s": 0.0607361731963465, + "kv_live_bytes": 462274560, + "latency_s": 1.9444714160636067, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76005376, + "prompt_kind": "short", + "t_relative_s": 9186.055066000205 + }, + { + "decode_latency_s": 0.06768599087081384, + "kv_live_bytes": 462274560, + "latency_s": 2.168330792337656, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76218368, + "prompt_kind": "long", + "t_relative_s": 9191.065145791974 + }, + { + "decode_latency_s": 0.06083925912389532, + "kv_live_bytes": 462274560, + "latency_s": 1.9476677500642836, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76775424, + "prompt_kind": "short", + "t_relative_s": 9196.075226624962 + }, + { + "decode_latency_s": 0.06786182681389619, + "kv_live_bytes": 462274560, + "latency_s": 2.1741126249544322, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76808192, + "prompt_kind": "long", + "t_relative_s": 9201.085273708217 + }, + { + "decode_latency_s": 0.0605369817931205, + "kv_live_bytes": 462274560, + "latency_s": 1.9380905842408538, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76709888, + "prompt_kind": "short", + "t_relative_s": 9206.09535362525 + }, + { + "decode_latency_s": 0.06033743097214028, + "kv_live_bytes": 462274560, + "latency_s": 1.9317395417019725, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76234752, + "prompt_kind": "long", + "t_relative_s": 9211.105438208207 + }, + { + "decode_latency_s": 0.06076571875018999, + "kv_live_bytes": 462274560, + "latency_s": 1.9453827082179487, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76251136, + "prompt_kind": "short", + "t_relative_s": 9216.115492417011 + }, + { + "decode_latency_s": 0.060613540379563347, + "kv_live_bytes": 462274560, + "latency_s": 1.9413735000416636, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76431360, + "prompt_kind": "long", + "t_relative_s": 9221.125584042165 + }, + { + "decode_latency_s": 0.06696347397519276, + "kv_live_bytes": 462274560, + "latency_s": 2.144568667281419, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76906496, + "prompt_kind": "short", + "t_relative_s": 9226.135669375304 + }, + { + "decode_latency_s": 0.06774325127480552, + "kv_live_bytes": 462274560, + "latency_s": 2.169894667342305, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76775424, + "prompt_kind": "long", + "t_relative_s": 9231.145719042048 + }, + { + "decode_latency_s": 0.06033219271921553, + "kv_live_bytes": 462274560, + "latency_s": 1.9315564162097871, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76218368, + "prompt_kind": "short", + "t_relative_s": 9236.155805250164 + }, + { + "decode_latency_s": 0.06744172528851777, + "kv_live_bytes": 462274560, + "latency_s": 2.1598627092316747, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76218368, + "prompt_kind": "long", + "t_relative_s": 9241.16587566724 + }, + { + "decode_latency_s": 0.06697017708211206, + "kv_live_bytes": 462274560, + "latency_s": 2.1448520408011973, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76267520, + "prompt_kind": "short", + "t_relative_s": 9246.175930083264 + }, + { + "decode_latency_s": 0.06776298697513994, + "kv_live_bytes": 462274560, + "latency_s": 2.1709255832247436, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76349440, + "prompt_kind": "long", + "t_relative_s": 9251.186046542134 + }, + { + "decode_latency_s": 0.060695611959090456, + "kv_live_bytes": 462274560, + "latency_s": 1.943020333070308, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76644352, + "prompt_kind": "short", + "t_relative_s": 9256.19615712529 + }, + { + "decode_latency_s": 0.06824043359665666, + "kv_live_bytes": 462274560, + "latency_s": 2.185782916843891, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76693504, + "prompt_kind": "long", + "t_relative_s": 9261.206213749945 + }, + { + "decode_latency_s": 0.060394397121854126, + "kv_live_bytes": 462274560, + "latency_s": 1.933516625314951, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76677120, + "prompt_kind": "short", + "t_relative_s": 9266.216302208137 + }, + { + "decode_latency_s": 0.06789934897096828, + "kv_live_bytes": 462274560, + "latency_s": 2.174520082771778, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76709888, + "prompt_kind": "long", + "t_relative_s": 9271.226391583215 + }, + { + "decode_latency_s": 0.06065126562316436, + "kv_live_bytes": 462274560, + "latency_s": 1.9416938750073314, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76775424, + "prompt_kind": "short", + "t_relative_s": 9276.236449375283 + }, + { + "decode_latency_s": 0.06799774087266997, + "kv_live_bytes": 462274560, + "latency_s": 2.17829104186967, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76857344, + "prompt_kind": "long", + "t_relative_s": 9281.246522833128 + }, + { + "decode_latency_s": 0.060709023440722376, + "kv_live_bytes": 462274560, + "latency_s": 1.9437852501869202, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72318976, + "prompt_kind": "short", + "t_relative_s": 9286.256632833276 + }, + { + "decode_latency_s": 0.06815992968040518, + "kv_live_bytes": 462274560, + "latency_s": 2.1832378329709172, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75923456, + "prompt_kind": "long", + "t_relative_s": 9291.266707000323 + }, + { + "decode_latency_s": 0.060686735683702864, + "kv_live_bytes": 462274560, + "latency_s": 1.942838708870113, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75366400, + "prompt_kind": "short", + "t_relative_s": 9296.276767666917 + }, + { + "decode_latency_s": 0.06822001822001766, + "kv_live_bytes": 462274560, + "latency_s": 2.185594040900469, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74956800, + "prompt_kind": "long", + "t_relative_s": 9301.286834333092 + }, + { + "decode_latency_s": 0.0607518059987342, + "kv_live_bytes": 462274560, + "latency_s": 1.9448811248876154, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75022336, + "prompt_kind": "short", + "t_relative_s": 9306.296901250258 + }, + { + "decode_latency_s": 0.06824149868043605, + "kv_live_bytes": 462274560, + "latency_s": 2.1866890001110733, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73465856, + "prompt_kind": "long", + "t_relative_s": 9311.306978750043 + }, + { + "decode_latency_s": 0.06029413543001283, + "kv_live_bytes": 462274560, + "latency_s": 1.930280499625951, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74629120, + "prompt_kind": "short", + "t_relative_s": 9316.317038042005 + }, + { + "decode_latency_s": 0.0679226914071478, + "kv_live_bytes": 462274560, + "latency_s": 2.1751989163458347, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74891264, + "prompt_kind": "long", + "t_relative_s": 9321.327109333128 + }, + { + "decode_latency_s": 0.0604177174391225, + "kv_live_bytes": 462274560, + "latency_s": 1.9342620419338346, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74956800, + "prompt_kind": "short", + "t_relative_s": 9326.337180375122 + }, + { + "decode_latency_s": 0.06825960677815601, + "kv_live_bytes": 462274560, + "latency_s": 2.1869760840199888, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75005952, + "prompt_kind": "long", + "t_relative_s": 9331.347264125012 + }, + { + "decode_latency_s": 0.060624443998676725, + "kv_live_bytes": 462274560, + "latency_s": 1.94077012501657, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75104256, + "prompt_kind": "short", + "t_relative_s": 9336.357347917277 + }, + { + "decode_latency_s": 0.06824826430238318, + "kv_live_bytes": 462274560, + "latency_s": 2.1860711248591542, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75235328, + "prompt_kind": "long", + "t_relative_s": 9341.367400208022 + }, + { + "decode_latency_s": 0.06026567058870569, + "kv_live_bytes": 462274560, + "latency_s": 1.9293937920592725, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75792384, + "prompt_kind": "short", + "t_relative_s": 9346.37746229209 + }, + { + "decode_latency_s": 0.06791585155588109, + "kv_live_bytes": 462274560, + "latency_s": 2.1750486250966787, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75284480, + "prompt_kind": "long", + "t_relative_s": 9351.387528917287 + }, + { + "decode_latency_s": 0.06575966274249367, + "kv_live_bytes": 462274560, + "latency_s": 2.1060167918913066, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75333632, + "prompt_kind": "short", + "t_relative_s": 9356.397588416934 + }, + { + "decode_latency_s": 0.0677021510200575, + "kv_live_bytes": 462274560, + "latency_s": 2.168929624836892, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75448320, + "prompt_kind": "long", + "t_relative_s": 9361.407702458091 + }, + { + "decode_latency_s": 0.06680506639531814, + "kv_live_bytes": 462274560, + "latency_s": 2.1395035409368575, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75431936, + "prompt_kind": "short", + "t_relative_s": 9366.417788917199 + }, + { + "decode_latency_s": 0.06222752343455795, + "kv_live_bytes": 462274560, + "latency_s": 1.9926808751188219, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75448320, + "prompt_kind": "long", + "t_relative_s": 9371.427865874954 + }, + { + "decode_latency_s": 0.06079390626109671, + "kv_live_bytes": 462274560, + "latency_s": 1.9462752919644117, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74825728, + "prompt_kind": "short", + "t_relative_s": 9376.43796870811 + }, + { + "decode_latency_s": 0.06794067578448448, + "kv_live_bytes": 462274560, + "latency_s": 2.1758503327146173, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75087872, + "prompt_kind": "long", + "t_relative_s": 9381.448060917202 + }, + { + "decode_latency_s": 0.06071338930632919, + "kv_live_bytes": 462274560, + "latency_s": 1.9438160830177367, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75137024, + "prompt_kind": "short", + "t_relative_s": 9386.458107292186 + }, + { + "decode_latency_s": 0.06815260025905445, + "kv_live_bytes": 462274560, + "latency_s": 2.1832719170488417, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75235328, + "prompt_kind": "long", + "t_relative_s": 9391.46820945805 + }, + { + "decode_latency_s": 0.0667324570385972, + "kv_live_bytes": 462274560, + "latency_s": 2.137153666932136, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75284480, + "prompt_kind": "short", + "t_relative_s": 9396.478302417323 + }, + { + "decode_latency_s": 0.0683263359387638, + "kv_live_bytes": 462274560, + "latency_s": 2.1883402089588344, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75284480, + "prompt_kind": "long", + "t_relative_s": 9401.48834658321 + }, + { + "decode_latency_s": 0.060829850248410366, + "kv_live_bytes": 462274560, + "latency_s": 1.9474652921780944, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73711616, + "prompt_kind": "short", + "t_relative_s": 9406.498447916936 + }, + { + "decode_latency_s": 0.06770594923000317, + "kv_live_bytes": 462274560, + "latency_s": 2.16830737516284, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75563008, + "prompt_kind": "long", + "t_relative_s": 9411.508530625142 + }, + { + "decode_latency_s": 0.06073744271998294, + "kv_live_bytes": 462274560, + "latency_s": 1.9444999159313738, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75677696, + "prompt_kind": "short", + "t_relative_s": 9416.51858333312 + }, + { + "decode_latency_s": 0.06771098566241562, + "kv_live_bytes": 462274560, + "latency_s": 2.1690643751062453, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75726848, + "prompt_kind": "long", + "t_relative_s": 9421.528670458123 + }, + { + "decode_latency_s": 0.06681634244159795, + "kv_live_bytes": 462274560, + "latency_s": 2.139723500236869, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75759616, + "prompt_kind": "short", + "t_relative_s": 9426.537982082926 + }, + { + "decode_latency_s": 0.0679177460988285, + "kv_live_bytes": 462274560, + "latency_s": 2.17536941729486, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75382784, + "prompt_kind": "long", + "t_relative_s": 9431.5480184583 + }, + { + "decode_latency_s": 0.06026388281316031, + "kv_live_bytes": 462274560, + "latency_s": 1.9293398750014603, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74694656, + "prompt_kind": "short", + "t_relative_s": 9436.558099499904 + }, + { + "decode_latency_s": 0.06830114584590774, + "kv_live_bytes": 462274560, + "latency_s": 2.187325374688953, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75350016, + "prompt_kind": "long", + "t_relative_s": 9441.56819345802 + }, + { + "decode_latency_s": 0.06052491665468551, + "kv_live_bytes": 462274560, + "latency_s": 1.937700083013624, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75448320, + "prompt_kind": "short", + "t_relative_s": 9446.578244583216 + }, + { + "decode_latency_s": 0.06728167450637557, + "kv_live_bytes": 462274560, + "latency_s": 2.1554314168170094, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75530240, + "prompt_kind": "long", + "t_relative_s": 9451.588321249932 + }, + { + "decode_latency_s": 0.06058010156266391, + "kv_live_bytes": 462274560, + "latency_s": 1.9393632910214365, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75563008, + "prompt_kind": "short", + "t_relative_s": 9456.598429583013 + }, + { + "decode_latency_s": 0.06790387761429884, + "kv_live_bytes": 462274560, + "latency_s": 2.1749545000493526, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75431936, + "prompt_kind": "long", + "t_relative_s": 9461.60848091729 + }, + { + "decode_latency_s": 0.06045158593042288, + "kv_live_bytes": 462274560, + "latency_s": 1.9353502499870956, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75431936, + "prompt_kind": "short", + "t_relative_s": 9466.618563167285 + }, + { + "decode_latency_s": 0.06815836587338708, + "kv_live_bytes": 462274560, + "latency_s": 2.1828174591064453, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76054528, + "prompt_kind": "long", + "t_relative_s": 9471.628642333206 + }, + { + "decode_latency_s": 0.0605138294340577, + "kv_live_bytes": 462274560, + "latency_s": 1.9373732502572238, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76218368, + "prompt_kind": "short", + "t_relative_s": 9476.638702417258 + }, + { + "decode_latency_s": 0.06738259896519594, + "kv_live_bytes": 462274560, + "latency_s": 2.158578457776457, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76365824, + "prompt_kind": "long", + "t_relative_s": 9481.648782667238 + }, + { + "decode_latency_s": 0.060635201807599515, + "kv_live_bytes": 462274560, + "latency_s": 1.941111583262682, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76414976, + "prompt_kind": "short", + "t_relative_s": 9486.658888542093 + }, + { + "decode_latency_s": 0.06040422135265544, + "kv_live_bytes": 462274560, + "latency_s": 1.9339819997549057, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76447744, + "prompt_kind": "long", + "t_relative_s": 9491.668931417167 + }, + { + "decode_latency_s": 0.06069433984521311, + "kv_live_bytes": 462274560, + "latency_s": 1.9431346249766648, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73711616, + "prompt_kind": "short", + "t_relative_s": 9496.679014916997 + }, + { + "decode_latency_s": 0.06828447006409988, + "kv_live_bytes": 462274560, + "latency_s": 2.187406832817942, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75382784, + "prompt_kind": "long", + "t_relative_s": 9501.689084333368 + }, + { + "decode_latency_s": 0.06037682943861, + "kv_live_bytes": 462274560, + "latency_s": 1.9329471252858639, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75546624, + "prompt_kind": "short", + "t_relative_s": 9506.699129958171 + }, + { + "decode_latency_s": 0.06811341016145889, + "kv_live_bytes": 462274560, + "latency_s": 2.1818949161097407, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75612160, + "prompt_kind": "long", + "t_relative_s": 9511.709217917174 + }, + { + "decode_latency_s": 0.06054125649097841, + "kv_live_bytes": 462274560, + "latency_s": 1.9381121252663434, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75628544, + "prompt_kind": "short", + "t_relative_s": 9516.719314292073 + }, + { + "decode_latency_s": 0.06828496875823475, + "kv_live_bytes": 462274560, + "latency_s": 2.1870622923597693, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75874304, + "prompt_kind": "long", + "t_relative_s": 9521.72937183315 + }, + { + "decode_latency_s": 0.06037841796933208, + "kv_live_bytes": 462274560, + "latency_s": 1.9330162908881903, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76283904, + "prompt_kind": "short", + "t_relative_s": 9526.739454207942 + }, + { + "decode_latency_s": 0.06805843622714747, + "kv_live_bytes": 462274560, + "latency_s": 2.1796924169175327, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76300288, + "prompt_kind": "long", + "t_relative_s": 9531.749546166975 + }, + { + "decode_latency_s": 0.06651087368663866, + "kv_live_bytes": 462274560, + "latency_s": 2.1302935839630663, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76513280, + "prompt_kind": "short", + "t_relative_s": 9536.759601583239 + }, + { + "decode_latency_s": 0.06808749999618158, + "kv_live_bytes": 462274560, + "latency_s": 2.180907292291522, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76562432, + "prompt_kind": "long", + "t_relative_s": 9541.769703042228 + }, + { + "decode_latency_s": 0.060643317687208764, + "kv_live_bytes": 462274560, + "latency_s": 1.941364916972816, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76595200, + "prompt_kind": "short", + "t_relative_s": 9546.779782833066 + }, + { + "decode_latency_s": 0.06819510286732111, + "kv_live_bytes": 462274560, + "latency_s": 2.1847334587946534, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75628544, + "prompt_kind": "long", + "t_relative_s": 9551.789837958291 + }, + { + "decode_latency_s": 0.0624046380398795, + "kv_live_bytes": 462274560, + "latency_s": 1.9977325000800192, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72466432, + "prompt_kind": "short", + "t_relative_s": 9556.799893542193 + }, + { + "decode_latency_s": 0.06814590885187499, + "kv_live_bytes": 462274560, + "latency_s": 2.1831662501208484, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75415552, + "prompt_kind": "long", + "t_relative_s": 9561.809957083315 + }, + { + "decode_latency_s": 0.0606035247183172, + "kv_live_bytes": 462274560, + "latency_s": 1.9401198751293123, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75464704, + "prompt_kind": "short", + "t_relative_s": 9566.82003991725 + }, + { + "decode_latency_s": 0.06807094531541225, + "kv_live_bytes": 462274560, + "latency_s": 2.180344333872199, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74743808, + "prompt_kind": "long", + "t_relative_s": 9571.830072875135 + }, + { + "decode_latency_s": 0.06068572527146898, + "kv_live_bytes": 462274560, + "latency_s": 1.9427685420960188, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72613888, + "prompt_kind": "short", + "t_relative_s": 9576.840159958228 + }, + { + "decode_latency_s": 0.06778775913699064, + "kv_live_bytes": 462274560, + "latency_s": 2.1707237916998565, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73859072, + "prompt_kind": "long", + "t_relative_s": 9581.850206542294 + }, + { + "decode_latency_s": 0.060723016897100024, + "kv_live_bytes": 462274560, + "latency_s": 1.9439484160393476, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73695232, + "prompt_kind": "short", + "t_relative_s": 9586.860255500302 + }, + { + "decode_latency_s": 0.06780910675297491, + "kv_live_bytes": 462274560, + "latency_s": 2.171333166770637, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75530240, + "prompt_kind": "long", + "t_relative_s": 9591.868515999988 + }, + { + "decode_latency_s": 0.060318514340906404, + "kv_live_bytes": 462274560, + "latency_s": 1.931088542100042, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75595776, + "prompt_kind": "short", + "t_relative_s": 9596.869015583303 + }, + { + "decode_latency_s": 0.06772771483520046, + "kv_live_bytes": 462274560, + "latency_s": 2.1695184581913054, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75808768, + "prompt_kind": "long", + "t_relative_s": 9601.879110082984 + }, + { + "decode_latency_s": 0.06059711199486628, + "kv_live_bytes": 462274560, + "latency_s": 1.93992062471807, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75841536, + "prompt_kind": "short", + "t_relative_s": 9606.889154667035 + }, + { + "decode_latency_s": 0.06835158984176815, + "kv_live_bytes": 462274560, + "latency_s": 2.1897840830497444, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75923456, + "prompt_kind": "long", + "t_relative_s": 9611.899199374951 + }, + { + "decode_latency_s": 0.06260151561582461, + "kv_live_bytes": 462274560, + "latency_s": 2.0042178332805634, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75317248, + "prompt_kind": "short", + "t_relative_s": 9616.90928570833 + }, + { + "decode_latency_s": 0.06780091796827037, + "kv_live_bytes": 462274560, + "latency_s": 2.171704832930118, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75956224, + "prompt_kind": "long", + "t_relative_s": 9621.91935420828 + }, + { + "decode_latency_s": 0.06756031382246874, + "kv_live_bytes": 462274560, + "latency_s": 2.1636087084189057, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75317248, + "prompt_kind": "short", + "t_relative_s": 9626.92946966691 + }, + { + "decode_latency_s": 0.06769167841412127, + "kv_live_bytes": 462274560, + "latency_s": 2.168541958089918, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75218944, + "prompt_kind": "long", + "t_relative_s": 9631.939508500043 + }, + { + "decode_latency_s": 0.06047025522275362, + "kv_live_bytes": 462274560, + "latency_s": 1.9359723753295839, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75235328, + "prompt_kind": "short", + "t_relative_s": 9636.94961479213 + }, + { + "decode_latency_s": 0.06851094921876211, + "kv_live_bytes": 462274560, + "latency_s": 2.1939241248182952, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72499200, + "prompt_kind": "long", + "t_relative_s": 9641.959697416984 + }, + { + "decode_latency_s": 0.06664452735276427, + "kv_live_bytes": 462274560, + "latency_s": 2.1349327908828855, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74596352, + "prompt_kind": "short", + "t_relative_s": 9646.96979141701 + }, + { + "decode_latency_s": 0.06557664844149258, + "kv_live_bytes": 462274560, + "latency_s": 2.1005362919531763, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75153408, + "prompt_kind": "long", + "t_relative_s": 9651.979846208356 + }, + { + "decode_latency_s": 0.06042671615432482, + "kv_live_bytes": 462274560, + "latency_s": 1.9345801249146461, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74301440, + "prompt_kind": "short", + "t_relative_s": 9656.98992375005 + }, + { + "decode_latency_s": 0.06555982813006267, + "kv_live_bytes": 462274560, + "latency_s": 2.099562041927129, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69795840, + "prompt_kind": "long", + "t_relative_s": 9661.999976708088 + }, + { + "decode_latency_s": 0.06081181380432099, + "kv_live_bytes": 462274560, + "latency_s": 1.946898290887475, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73596928, + "prompt_kind": "short", + "t_relative_s": 9667.010047625285 + }, + { + "decode_latency_s": 0.06064314712421037, + "kv_live_bytes": 462274560, + "latency_s": 1.941627792082727, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72843264, + "prompt_kind": "long", + "t_relative_s": 9672.020082333125 + }, + { + "decode_latency_s": 0.06238199608924333, + "kv_live_bytes": 462274560, + "latency_s": 1.9970148750580847, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72073216, + "prompt_kind": "short", + "t_relative_s": 9677.030130042229 + }, + { + "decode_latency_s": 0.06820717579103075, + "kv_live_bytes": 462274560, + "latency_s": 2.1849121251143515, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74727424, + "prompt_kind": "long", + "t_relative_s": 9682.040179625154 + }, + { + "decode_latency_s": 0.060641877600573935, + "kv_live_bytes": 462274560, + "latency_s": 1.9412925001233816, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74842112, + "prompt_kind": "short", + "t_relative_s": 9687.050265708007 + }, + { + "decode_latency_s": 0.060633113273070194, + "kv_live_bytes": 462274560, + "latency_s": 1.941096667200327, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75284480, + "prompt_kind": "long", + "t_relative_s": 9692.060342042241 + }, + { + "decode_latency_s": 0.060484230460133404, + "kv_live_bytes": 462274560, + "latency_s": 1.9364958750084043, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74940416, + "prompt_kind": "short", + "t_relative_s": 9697.070395042188 + }, + { + "decode_latency_s": 0.06878479165607132, + "kv_live_bytes": 462274560, + "latency_s": 2.2030757078900933, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75104256, + "prompt_kind": "long", + "t_relative_s": 9702.080469958019 + }, + { + "decode_latency_s": 0.0612303567759227, + "kv_live_bytes": 462274560, + "latency_s": 1.9602850838564336, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74989568, + "prompt_kind": "short", + "t_relative_s": 9707.090516000055 + }, + { + "decode_latency_s": 0.06791855210030917, + "kv_live_bytes": 462274560, + "latency_s": 2.174893250223249, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73400320, + "prompt_kind": "long", + "t_relative_s": 9712.10062158294 + }, + { + "decode_latency_s": 0.06031438543868717, + "kv_live_bytes": 462274560, + "latency_s": 1.9309337921440601, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74678272, + "prompt_kind": "short", + "t_relative_s": 9717.11067495821 + }, + { + "decode_latency_s": 0.0679314322915161, + "kv_live_bytes": 462274560, + "latency_s": 2.1757936673238873, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75284480, + "prompt_kind": "long", + "t_relative_s": 9722.120742625091 + }, + { + "decode_latency_s": 0.06073309505882207, + "kv_live_bytes": 462274560, + "latency_s": 1.944253166206181, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75317248, + "prompt_kind": "short", + "t_relative_s": 9727.130807792302 + }, + { + "decode_latency_s": 0.0683636627800297, + "kv_live_bytes": 462274560, + "latency_s": 2.190413041971624, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75120640, + "prompt_kind": "long", + "t_relative_s": 9732.140865750145 + }, + { + "decode_latency_s": 0.062881442689104, + "kv_live_bytes": 462274560, + "latency_s": 2.0131461252458394, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74661888, + "prompt_kind": "short", + "t_relative_s": 9737.150960708037 + }, + { + "decode_latency_s": 0.06804323568940163, + "kv_live_bytes": 462274560, + "latency_s": 2.179894792381674, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75694080, + "prompt_kind": "long", + "t_relative_s": 9742.161033832934 + }, + { + "decode_latency_s": 0.06248862236679997, + "kv_live_bytes": 462274560, + "latency_s": 2.0004605003632605, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90193920, + "prompt_kind": "short", + "t_relative_s": 9747.171111875214 + }, + { + "decode_latency_s": 0.06801878774422221, + "kv_live_bytes": 462274560, + "latency_s": 2.1791366669349372, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90210304, + "prompt_kind": "long", + "t_relative_s": 9752.181156042032 + }, + { + "decode_latency_s": 0.06051574740558863, + "kv_live_bytes": 462274560, + "latency_s": 1.9374216659925878, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90112000, + "prompt_kind": "short", + "t_relative_s": 9757.191227250267 + }, + { + "decode_latency_s": 0.06859125000482891, + "kv_live_bytes": 462274560, + "latency_s": 2.1966065410524607, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88539136, + "prompt_kind": "long", + "t_relative_s": 9762.201305625029 + }, + { + "decode_latency_s": 0.06761985806224402, + "kv_live_bytes": 462274560, + "latency_s": 2.165699750185013, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 89276416, + "prompt_kind": "short", + "t_relative_s": 9767.211361750029 + }, + { + "decode_latency_s": 0.06765882813488133, + "kv_live_bytes": 462274560, + "latency_s": 2.167114249896258, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 89948160, + "prompt_kind": "long", + "t_relative_s": 9772.221469750162 + }, + { + "decode_latency_s": 0.06060289063316304, + "kv_live_bytes": 462274560, + "latency_s": 1.9401282076723874, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 89964544, + "prompt_kind": "short", + "t_relative_s": 9777.231577167287 + }, + { + "decode_latency_s": 0.06789551433757879, + "kv_live_bytes": 462274560, + "latency_s": 2.1749971667304635, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 89964544, + "prompt_kind": "long", + "t_relative_s": 9782.241635249928 + }, + { + "decode_latency_s": 0.06032345835410524, + "kv_live_bytes": 462274560, + "latency_s": 1.9310989580117166, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 89964544, + "prompt_kind": "short", + "t_relative_s": 9787.251724125352 + }, + { + "decode_latency_s": 0.06805143489327747, + "kv_live_bytes": 462274560, + "latency_s": 2.1791366660036147, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 87703552, + "prompt_kind": "long", + "t_relative_s": 9792.261810542084 + }, + { + "decode_latency_s": 0.06091050780378282, + "kv_live_bytes": 462274560, + "latency_s": 1.9499710001982749, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88424448, + "prompt_kind": "short", + "t_relative_s": 9797.271886250004 + }, + { + "decode_latency_s": 0.06786165104131214, + "kv_live_bytes": 462274560, + "latency_s": 2.1735169580206275, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88932352, + "prompt_kind": "long", + "t_relative_s": 9802.281943375245 + }, + { + "decode_latency_s": 0.06027551040460821, + "kv_live_bytes": 462274560, + "latency_s": 1.9297746657393873, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88834048, + "prompt_kind": "short", + "t_relative_s": 9807.291999208275 + }, + { + "decode_latency_s": 0.060558895842405036, + "kv_live_bytes": 462274560, + "latency_s": 1.9387914161197841, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88752128, + "prompt_kind": "long", + "t_relative_s": 9812.302071417216 + }, + { + "decode_latency_s": 0.060673250001855195, + "kv_live_bytes": 462274560, + "latency_s": 1.9424327500164509, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88883200, + "prompt_kind": "short", + "t_relative_s": 9817.31211454235 + }, + { + "decode_latency_s": 0.06836445444787387, + "kv_live_bytes": 462274560, + "latency_s": 2.1898033749312162, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 87441408, + "prompt_kind": "long", + "t_relative_s": 9822.322207458317 + }, + { + "decode_latency_s": 0.0667968216148438, + "kv_live_bytes": 462274560, + "latency_s": 2.1391367078758776, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88145920, + "prompt_kind": "short", + "t_relative_s": 9827.33229308296 + }, + { + "decode_latency_s": 0.06801744659605902, + "kv_live_bytes": 462274560, + "latency_s": 2.1784318750724196, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74940416, + "prompt_kind": "long", + "t_relative_s": 9832.34235400008 + }, + { + "decode_latency_s": 0.06035529034852516, + "kv_live_bytes": 462274560, + "latency_s": 1.9322581668384373, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75005952, + "prompt_kind": "short", + "t_relative_s": 9837.352423958015 + }, + { + "decode_latency_s": 0.0679013398475945, + "kv_live_bytes": 462274560, + "latency_s": 2.1742174169048667, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75137024, + "prompt_kind": "long", + "t_relative_s": 9842.362497624941 + }, + { + "decode_latency_s": 0.06351848956546746, + "kv_live_bytes": 462274560, + "latency_s": 2.0334972497075796, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75218944, + "prompt_kind": "short", + "t_relative_s": 9847.372545875143 + }, + { + "decode_latency_s": 0.06829838540579658, + "kv_live_bytes": 462274560, + "latency_s": 2.187780833337456, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75235328, + "prompt_kind": "long", + "t_relative_s": 9852.382628499996 + }, + { + "decode_latency_s": 0.06055714584363159, + "kv_live_bytes": 462274560, + "latency_s": 1.9386255419813097, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75759616, + "prompt_kind": "short", + "t_relative_s": 9857.392711667344 + }, + { + "decode_latency_s": 0.06790785936755128, + "kv_live_bytes": 462274560, + "latency_s": 2.175347290933132, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75841536, + "prompt_kind": "long", + "t_relative_s": 9862.40276366705 + }, + { + "decode_latency_s": 0.06023623957298696, + "kv_live_bytes": 462274560, + "latency_s": 1.928433082997799, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75038720, + "prompt_kind": "short", + "t_relative_s": 9867.412837458309 + }, + { + "decode_latency_s": 0.06035486586915795, + "kv_live_bytes": 462274560, + "latency_s": 1.9322514589875937, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75300864, + "prompt_kind": "long", + "t_relative_s": 9872.422916542273 + }, + { + "decode_latency_s": 0.060233552096178755, + "kv_live_bytes": 462274560, + "latency_s": 1.9283479996956885, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75300864, + "prompt_kind": "short", + "t_relative_s": 9877.43296950031 + }, + { + "decode_latency_s": 0.06838738544320222, + "kv_live_bytes": 462274560, + "latency_s": 2.190600708127022, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75415552, + "prompt_kind": "long", + "t_relative_s": 9882.4430455002 + }, + { + "decode_latency_s": 0.06600769399665296, + "kv_live_bytes": 462274560, + "latency_s": 2.1139379581436515, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74383360, + "prompt_kind": "short", + "t_relative_s": 9887.453161833342 + }, + { + "decode_latency_s": 0.06774815887911245, + "kv_live_bytes": 462274560, + "latency_s": 2.16881295805797, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69632000, + "prompt_kind": "long", + "t_relative_s": 9892.46323391702 + }, + { + "decode_latency_s": 0.06078150909161195, + "kv_live_bytes": 462274560, + "latency_s": 1.945783874951303, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69959680, + "prompt_kind": "short", + "t_relative_s": 9899.140753167216 + }, + { + "decode_latency_s": 0.06830529816215858, + "kv_live_bytes": 462274560, + "latency_s": 2.18761625001207, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68878336, + "prompt_kind": "long", + "t_relative_s": 9904.150836958084 + }, + { + "decode_latency_s": 0.06021749218052719, + "kv_live_bytes": 462274560, + "latency_s": 1.9278783751651645, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69910528, + "prompt_kind": "short", + "t_relative_s": 9909.160885208286 + }, + { + "decode_latency_s": 0.068127339851344, + "kv_live_bytes": 462274560, + "latency_s": 2.182158250361681, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70156288, + "prompt_kind": "long", + "t_relative_s": 9914.170958333183 + }, + { + "decode_latency_s": 0.06115946744102985, + "kv_live_bytes": 462274560, + "latency_s": 1.957890666089952, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70696960, + "prompt_kind": "short", + "t_relative_s": 9919.17715062527 + }, + { + "decode_latency_s": 0.06788977865653578, + "kv_live_bytes": 462274560, + "latency_s": 2.1751840417273343, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69468160, + "prompt_kind": "long", + "t_relative_s": 9924.187205625232 + }, + { + "decode_latency_s": 0.06730376822815742, + "kv_live_bytes": 462274560, + "latency_s": 2.1554558337666094, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69779456, + "prompt_kind": "short", + "t_relative_s": 9929.197296583094 + }, + { + "decode_latency_s": 0.06040716664574575, + "kv_live_bytes": 462274560, + "latency_s": 1.9338442077860236, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69844992, + "prompt_kind": "long", + "t_relative_s": 9934.20734391734 + }, + { + "decode_latency_s": 0.06036778257112019, + "kv_live_bytes": 462274560, + "latency_s": 1.932655541691929, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70483968, + "prompt_kind": "short", + "t_relative_s": 9939.217412875034 + }, + { + "decode_latency_s": 0.06825542316073552, + "kv_live_bytes": 462274560, + "latency_s": 2.1867667082697153, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70516736, + "prompt_kind": "long", + "t_relative_s": 9944.22745083319 + }, + { + "decode_latency_s": 0.06038929555506911, + "kv_live_bytes": 462274560, + "latency_s": 1.9332027090713382, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71057408, + "prompt_kind": "short", + "t_relative_s": 9949.237537083216 + }, + { + "decode_latency_s": 0.06656644790200517, + "kv_live_bytes": 462274560, + "latency_s": 2.1319045843556523, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71106560, + "prompt_kind": "long", + "t_relative_s": 9954.247621749993 + }, + { + "decode_latency_s": 0.06044232162821572, + "kv_live_bytes": 462274560, + "latency_s": 1.9350528749637306, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71139328, + "prompt_kind": "short", + "t_relative_s": 9959.257677874994 + }, + { + "decode_latency_s": 0.06871475653315429, + "kv_live_bytes": 462274560, + "latency_s": 2.2012724997475743, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71319552, + "prompt_kind": "long", + "t_relative_s": 9964.267762708012 + }, + { + "decode_latency_s": 0.060637063812464476, + "kv_live_bytes": 462274560, + "latency_s": 1.941211124882102, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71892992, + "prompt_kind": "short", + "t_relative_s": 9969.27786820801 + }, + { + "decode_latency_s": 0.06824430859705899, + "kv_live_bytes": 462274560, + "latency_s": 2.1864616670645773, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71483392, + "prompt_kind": "long", + "t_relative_s": 9974.287920708302 + }, + { + "decode_latency_s": 0.06048107812239323, + "kv_live_bytes": 462274560, + "latency_s": 1.936212666798383, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71417856, + "prompt_kind": "short", + "t_relative_s": 9979.297991583124 + }, + { + "decode_latency_s": 0.06796660284453537, + "kv_live_bytes": 462274560, + "latency_s": 2.176666708663106, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71860224, + "prompt_kind": "long", + "t_relative_s": 9984.308039499912 + }, + { + "decode_latency_s": 0.060238277350435965, + "kv_live_bytes": 462274560, + "latency_s": 1.928597167134285, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70516736, + "prompt_kind": "short", + "t_relative_s": 9989.318090583198 + }, + { + "decode_latency_s": 0.06828857812797651, + "kv_live_bytes": 462274560, + "latency_s": 2.187305625062436, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71614464, + "prompt_kind": "long", + "t_relative_s": 9994.328155792318 + }, + { + "decode_latency_s": 0.06043149478500709, + "kv_live_bytes": 462274560, + "latency_s": 1.9346124157309532, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72122368, + "prompt_kind": "short", + "t_relative_s": 9999.338221500162 + }, + { + "decode_latency_s": 0.06762654424528591, + "kv_live_bytes": 462274560, + "latency_s": 2.1663845414295793, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72204288, + "prompt_kind": "long", + "t_relative_s": 10004.348287832923 + }, + { + "decode_latency_s": 0.06053580209845677, + "kv_live_bytes": 462274560, + "latency_s": 1.9380000000819564, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72433664, + "prompt_kind": "short", + "t_relative_s": 10009.358372500166 + }, + { + "decode_latency_s": 0.06801692317822017, + "kv_live_bytes": 462274560, + "latency_s": 2.178311957977712, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71680000, + "prompt_kind": "long", + "t_relative_s": 10014.368432083167 + }, + { + "decode_latency_s": 0.06026401824783534, + "kv_live_bytes": 462274560, + "latency_s": 1.9293957501649857, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71286784, + "prompt_kind": "short", + "t_relative_s": 10019.378474917263 + }, + { + "decode_latency_s": 0.06831786588008981, + "kv_live_bytes": 462274560, + "latency_s": 2.187805207911879, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71860224, + "prompt_kind": "long", + "t_relative_s": 10024.38855104195 + }, + { + "decode_latency_s": 0.06054205728287343, + "kv_live_bytes": 462274560, + "latency_s": 1.938202042132616, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72466432, + "prompt_kind": "short", + "t_relative_s": 10029.398610292003 + }, + { + "decode_latency_s": 0.06805157422786579, + "kv_live_bytes": 462274560, + "latency_s": 2.1799073331058025, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72515584, + "prompt_kind": "long", + "t_relative_s": 10034.408682167064 + }, + { + "decode_latency_s": 0.060949887993047014, + "kv_live_bytes": 462274560, + "latency_s": 1.9512428753077984, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70156288, + "prompt_kind": "short", + "t_relative_s": 10039.418732041959 + }, + { + "decode_latency_s": 0.06792924999899697, + "kv_live_bytes": 462274560, + "latency_s": 2.175799749791622, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71892992, + "prompt_kind": "long", + "t_relative_s": 10044.428800042253 + }, + { + "decode_latency_s": 0.06052144792920444, + "kv_live_bytes": 462274560, + "latency_s": 1.9374854159541428, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72007680, + "prompt_kind": "short", + "t_relative_s": 10049.438875125255 + }, + { + "decode_latency_s": 0.06838379424880259, + "kv_live_bytes": 462274560, + "latency_s": 2.1909556249156594, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72089600, + "prompt_kind": "long", + "t_relative_s": 10054.448943417054 + }, + { + "decode_latency_s": 0.060544207037310116, + "kv_live_bytes": 462274560, + "latency_s": 1.9381692497991025, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72728576, + "prompt_kind": "short", + "t_relative_s": 10059.459048416931 + }, + { + "decode_latency_s": 0.06836973437748384, + "kv_live_bytes": 462274560, + "latency_s": 2.1900440002791584, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71778304, + "prompt_kind": "long", + "t_relative_s": 10064.463478042278 + }, + { + "decode_latency_s": 0.06666979037981946, + "kv_live_bytes": 462274560, + "latency_s": 2.135360292159021, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71925760, + "prompt_kind": "short", + "t_relative_s": 10069.473549458198 + }, + { + "decode_latency_s": 0.06774811455397867, + "kv_live_bytes": 462274560, + "latency_s": 2.1701329168863595, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72564736, + "prompt_kind": "long", + "t_relative_s": 10074.483598249964 + }, + { + "decode_latency_s": 0.060305253908154555, + "kv_live_bytes": 462274560, + "latency_s": 1.9306317502632737, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72744960, + "prompt_kind": "short", + "t_relative_s": 10079.49365662504 + }, + { + "decode_latency_s": 0.0685477942606667, + "kv_live_bytes": 462274560, + "latency_s": 2.1958199157379568, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72761344, + "prompt_kind": "long", + "t_relative_s": 10084.50373608293 + }, + { + "decode_latency_s": 0.06680486199911684, + "kv_live_bytes": 462274560, + "latency_s": 2.139448500238359, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73318400, + "prompt_kind": "short", + "t_relative_s": 10089.513795082923 + }, + { + "decode_latency_s": 0.06810383594711311, + "kv_live_bytes": 462274560, + "latency_s": 2.1817730832844973, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73383936, + "prompt_kind": "long", + "t_relative_s": 10094.523898000363 + }, + { + "decode_latency_s": 0.06044171484245453, + "kv_live_bytes": 462274560, + "latency_s": 1.934910542331636, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73400320, + "prompt_kind": "short", + "t_relative_s": 10099.5339540001 + }, + { + "decode_latency_s": 0.060514287775731646, + "kv_live_bytes": 462274560, + "latency_s": 1.9375425837934017, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "long", + "t_relative_s": 10104.544000875205 + }, + { + "decode_latency_s": 0.06033175651100464, + "kv_live_bytes": 462274560, + "latency_s": 1.931648749858141, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73400320, + "prompt_kind": "short", + "t_relative_s": 10109.554058917332 + }, + { + "decode_latency_s": 0.06840296094014775, + "kv_live_bytes": 462274560, + "latency_s": 2.191085583064705, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73498624, + "prompt_kind": "long", + "t_relative_s": 10114.564143833239 + }, + { + "decode_latency_s": 0.06066532162367366, + "kv_live_bytes": 462274560, + "latency_s": 1.942152834031731, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74153984, + "prompt_kind": "short", + "t_relative_s": 10119.574238958303 + }, + { + "decode_latency_s": 0.06778712758386973, + "kv_live_bytes": 462274560, + "latency_s": 2.172062083147466, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74301440, + "prompt_kind": "long", + "t_relative_s": 10124.584293541964 + }, + { + "decode_latency_s": 0.06652068489347585, + "kv_live_bytes": 462274560, + "latency_s": 2.130374124739319, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74350592, + "prompt_kind": "short", + "t_relative_s": 10129.594384250231 + }, + { + "decode_latency_s": 0.06805521092610434, + "kv_live_bytes": 462274560, + "latency_s": 2.1799665419384837, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74334208, + "prompt_kind": "long", + "t_relative_s": 10134.604438417125 + }, + { + "decode_latency_s": 0.06052732684474904, + "kv_live_bytes": 462274560, + "latency_s": 1.9377748752012849, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72335360, + "prompt_kind": "short", + "t_relative_s": 10139.614493333269 + }, + { + "decode_latency_s": 0.06788859114749357, + "kv_live_bytes": 462274560, + "latency_s": 2.1746405828744173, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73875456, + "prompt_kind": "long", + "t_relative_s": 10144.624553916976 + }, + { + "decode_latency_s": 0.06047223959467374, + "kv_live_bytes": 462274560, + "latency_s": 1.9360173749737442, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73908224, + "prompt_kind": "short", + "t_relative_s": 10149.634615958203 + }, + { + "decode_latency_s": 0.06780995181179605, + "kv_live_bytes": 462274560, + "latency_s": 2.1718086251057684, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72794112, + "prompt_kind": "long", + "t_relative_s": 10154.644703583326 + }, + { + "decode_latency_s": 0.06031677735154517, + "kv_live_bytes": 462274560, + "latency_s": 1.9311748328618705, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73498624, + "prompt_kind": "short", + "t_relative_s": 10159.654748833273 + }, + { + "decode_latency_s": 0.0681354778498644, + "kv_live_bytes": 462274560, + "latency_s": 2.182690708898008, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74104832, + "prompt_kind": "long", + "t_relative_s": 10164.664820167236 + }, + { + "decode_latency_s": 0.0606668059626827, + "kv_live_bytes": 462274560, + "latency_s": 1.9421276673674583, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74137600, + "prompt_kind": "short", + "t_relative_s": 10169.67490645824 + }, + { + "decode_latency_s": 0.06785089061304461, + "kv_live_bytes": 462274560, + "latency_s": 2.1734535829164088, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74235904, + "prompt_kind": "long", + "t_relative_s": 10174.684956167359 + }, + { + "decode_latency_s": 0.060379192684195004, + "kv_live_bytes": 462274560, + "latency_s": 1.9330131248570979, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74776576, + "prompt_kind": "short", + "t_relative_s": 10179.695048875175 + }, + { + "decode_latency_s": 0.06823705468559638, + "kv_live_bytes": 462274560, + "latency_s": 2.1848687501624227, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69746688, + "prompt_kind": "long", + "t_relative_s": 10184.70512333326 + }, + { + "decode_latency_s": 0.06083893097820692, + "kv_live_bytes": 462274560, + "latency_s": 1.947646166663617, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71499776, + "prompt_kind": "short", + "t_relative_s": 10189.715216042008 + }, + { + "decode_latency_s": 0.0682502421841491, + "kv_live_bytes": 462274560, + "latency_s": 2.1862532920204103, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73269248, + "prompt_kind": "long", + "t_relative_s": 10194.725270875264 + }, + { + "decode_latency_s": 0.06040840105561074, + "kv_live_bytes": 462274560, + "latency_s": 1.9339737920090556, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72777728, + "prompt_kind": "short", + "t_relative_s": 10199.735351417214 + }, + { + "decode_latency_s": 0.06835074609261937, + "kv_live_bytes": 462274560, + "latency_s": 2.189068084117025, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72630272, + "prompt_kind": "long", + "t_relative_s": 10204.745431458112 + }, + { + "decode_latency_s": 0.06045992187864613, + "kv_live_bytes": 462274560, + "latency_s": 1.9356533330865204, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72744960, + "prompt_kind": "short", + "t_relative_s": 10209.755469250027 + }, + { + "decode_latency_s": 0.06960319398785941, + "kv_live_bytes": 462274560, + "latency_s": 2.2291951249353588, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72220672, + "prompt_kind": "long", + "t_relative_s": 10214.76553758327 + }, + { + "decode_latency_s": 0.060314648450003006, + "kv_live_bytes": 462274560, + "latency_s": 1.930971666239202, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72859648, + "prompt_kind": "short", + "t_relative_s": 10219.775606708135 + }, + { + "decode_latency_s": 0.06850482942536473, + "kv_live_bytes": 462274560, + "latency_s": 2.1941657080315053, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73433088, + "prompt_kind": "long", + "t_relative_s": 10224.785692125093 + }, + { + "decode_latency_s": 0.06700727994029876, + "kv_live_bytes": 462274560, + "latency_s": 2.146055458113551, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73531392, + "prompt_kind": "short", + "t_relative_s": 10229.795750500169 + }, + { + "decode_latency_s": 0.06087461719289422, + "kv_live_bytes": 462274560, + "latency_s": 1.948968917131424, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73629696, + "prompt_kind": "long", + "t_relative_s": 10234.805828667246 + }, + { + "decode_latency_s": 0.06064003384381067, + "kv_live_bytes": 462274560, + "latency_s": 1.9412727090530097, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73711616, + "prompt_kind": "short", + "t_relative_s": 10239.815897500142 + }, + { + "decode_latency_s": 0.0681307695194846, + "kv_live_bytes": 462274560, + "latency_s": 2.182729249820113, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73777152, + "prompt_kind": "long", + "t_relative_s": 10244.818069499917 + }, + { + "decode_latency_s": 0.0603881237184396, + "kv_live_bytes": 462274560, + "latency_s": 1.9333896250464022, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73334784, + "prompt_kind": "short", + "t_relative_s": 10249.828153250273 + }, + { + "decode_latency_s": 0.06710228255542461, + "kv_live_bytes": 462274560, + "latency_s": 2.1489571668207645, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73859072, + "prompt_kind": "long", + "t_relative_s": 10254.838206707966 + }, + { + "decode_latency_s": 0.06056265103688929, + "kv_live_bytes": 462274560, + "latency_s": 1.9388775830157101, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73891840, + "prompt_kind": "short", + "t_relative_s": 10259.848252667114 + }, + { + "decode_latency_s": 0.0685386041586753, + "kv_live_bytes": 462274560, + "latency_s": 2.196037291083485, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74006528, + "prompt_kind": "long", + "t_relative_s": 10264.858354333322 + }, + { + "decode_latency_s": 0.060694529936881736, + "kv_live_bytes": 462274560, + "latency_s": 1.9429860417731106, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74006528, + "prompt_kind": "short", + "t_relative_s": 10269.868426207919 + }, + { + "decode_latency_s": 0.06826598959742114, + "kv_live_bytes": 462274560, + "latency_s": 2.1869723326526582, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71565312, + "prompt_kind": "long", + "t_relative_s": 10274.878474292345 + }, + { + "decode_latency_s": 0.06046051824523602, + "kv_live_bytes": 462274560, + "latency_s": 1.935670250095427, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73138176, + "prompt_kind": "short", + "t_relative_s": 10279.888547332957 + }, + { + "decode_latency_s": 0.06812335156428162, + "kv_live_bytes": 462274560, + "latency_s": 2.1814500829204917, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73646080, + "prompt_kind": "long", + "t_relative_s": 10284.898623333313 + }, + { + "decode_latency_s": 0.06681500261765905, + "kv_live_bytes": 462274560, + "latency_s": 2.1399519159458578, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73744384, + "prompt_kind": "short", + "t_relative_s": 10289.908687999938 + }, + { + "decode_latency_s": 0.06534415365604218, + "kv_live_bytes": 462274560, + "latency_s": 2.0924217076972127, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73809920, + "prompt_kind": "long", + "t_relative_s": 10294.91877620807 + }, + { + "decode_latency_s": 0.060560994781553745, + "kv_live_bytes": 462274560, + "latency_s": 1.9388237078674138, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74366976, + "prompt_kind": "short", + "t_relative_s": 10299.928852250334 + }, + { + "decode_latency_s": 0.06763475522166118, + "kv_live_bytes": 462274560, + "latency_s": 2.166310125030577, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73367552, + "prompt_kind": "long", + "t_relative_s": 10304.93889720831 + }, + { + "decode_latency_s": 0.06042908334347885, + "kv_live_bytes": 462274560, + "latency_s": 1.9346614172682166, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71647232, + "prompt_kind": "short", + "t_relative_s": 10309.948993749917 + }, + { + "decode_latency_s": 0.06802858856099192, + "kv_live_bytes": 462274560, + "latency_s": 2.1789345000870526, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74399744, + "prompt_kind": "long", + "t_relative_s": 10314.95907037519 + }, + { + "decode_latency_s": 0.06066759634995833, + "kv_live_bytes": 462274560, + "latency_s": 1.942228791769594, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74547200, + "prompt_kind": "short", + "t_relative_s": 10319.969121042173 + }, + { + "decode_latency_s": 0.06827258074190468, + "kv_live_bytes": 462274560, + "latency_s": 2.187305834144354, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74711040, + "prompt_kind": "long", + "t_relative_s": 10324.979208667297 + }, + { + "decode_latency_s": 0.06061697787663434, + "kv_live_bytes": 462274560, + "latency_s": 1.9405914582312107, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75268096, + "prompt_kind": "short", + "t_relative_s": 10329.989318250213 + }, + { + "decode_latency_s": 0.06797965755686164, + "kv_live_bytes": 462274560, + "latency_s": 2.177767042070627, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74514432, + "prompt_kind": "long", + "t_relative_s": 10334.99936245801 + }, + { + "decode_latency_s": 0.06023077343706973, + "kv_live_bytes": 462274560, + "latency_s": 1.9282993329688907, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74399744, + "prompt_kind": "short", + "t_relative_s": 10340.009428875055 + }, + { + "decode_latency_s": 0.06795536327990703, + "kv_live_bytes": 462274560, + "latency_s": 2.1763493339531124, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75038720, + "prompt_kind": "long", + "t_relative_s": 10345.019504250027 + }, + { + "decode_latency_s": 0.06052796743460931, + "kv_live_bytes": 462274560, + "latency_s": 1.937784249894321, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75137024, + "prompt_kind": "short", + "t_relative_s": 10350.029560707975 + }, + { + "decode_latency_s": 0.06803154687804636, + "kv_live_bytes": 462274560, + "latency_s": 2.1792453746311367, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75186176, + "prompt_kind": "long", + "t_relative_s": 10355.039657332934 + }, + { + "decode_latency_s": 0.060553192713996395, + "kv_live_bytes": 462274560, + "latency_s": 1.9385312502272427, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75218944, + "prompt_kind": "short", + "t_relative_s": 10360.049735083245 + }, + { + "decode_latency_s": 0.06789427994226571, + "kv_live_bytes": 462274560, + "latency_s": 2.1757045830599964, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75169792, + "prompt_kind": "long", + "t_relative_s": 10365.059779291973 + }, + { + "decode_latency_s": 0.06032896743272431, + "kv_live_bytes": 462274560, + "latency_s": 1.931438332889229, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74629120, + "prompt_kind": "short", + "t_relative_s": 10370.069842542056 + }, + { + "decode_latency_s": 0.06795682030497119, + "kv_live_bytes": 462274560, + "latency_s": 2.177188124973327, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74727424, + "prompt_kind": "long", + "t_relative_s": 10375.079877708107 + }, + { + "decode_latency_s": 0.060344065132085234, + "kv_live_bytes": 462274560, + "latency_s": 1.931783666368574, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74743808, + "prompt_kind": "short", + "t_relative_s": 10380.089967333246 + }, + { + "decode_latency_s": 0.060386368495528586, + "kv_live_bytes": 462274560, + "latency_s": 1.933236625045538, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74842112, + "prompt_kind": "long", + "t_relative_s": 10385.10004887497 + }, + { + "decode_latency_s": 0.060338184906868264, + "kv_live_bytes": 462274560, + "latency_s": 1.9317322922870517, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74727424, + "prompt_kind": "short", + "t_relative_s": 10390.110089791939 + }, + { + "decode_latency_s": 0.06791257031727582, + "kv_live_bytes": 462274560, + "latency_s": 2.1750637497752905, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74809344, + "prompt_kind": "long", + "t_relative_s": 10395.120166874956 + }, + { + "decode_latency_s": 0.06060425649047829, + "kv_live_bytes": 462274560, + "latency_s": 1.9401815831661224, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75415552, + "prompt_kind": "short", + "t_relative_s": 10400.130208583083 + }, + { + "decode_latency_s": 0.06033076823223382, + "kv_live_bytes": 462274560, + "latency_s": 1.931452624965459, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75448320, + "prompt_kind": "long", + "t_relative_s": 10405.140253708232 + }, + { + "decode_latency_s": 0.060391591119696386, + "kv_live_bytes": 462274560, + "latency_s": 1.933290584012866, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75513856, + "prompt_kind": "short", + "t_relative_s": 10410.150334957987 + }, + { + "decode_latency_s": 0.06810429815959651, + "kv_live_bytes": 462274560, + "latency_s": 2.1816272498108447, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75268096, + "prompt_kind": "long", + "t_relative_s": 10415.160416916944 + }, + { + "decode_latency_s": 0.06034217968408484, + "kv_live_bytes": 462274560, + "latency_s": 1.9318544575944543, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75350016, + "prompt_kind": "short", + "t_relative_s": 10420.170471041929 + }, + { + "decode_latency_s": 0.06810311197477859, + "kv_live_bytes": 462274560, + "latency_s": 2.18150816578418, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75251712, + "prompt_kind": "long", + "t_relative_s": 10425.180553167127 + }, + { + "decode_latency_s": 0.06059183203615248, + "kv_live_bytes": 462274560, + "latency_s": 1.939729958306998, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75415552, + "prompt_kind": "short", + "t_relative_s": 10430.19064937532 + }, + { + "decode_latency_s": 0.06041700261994265, + "kv_live_bytes": 462274560, + "latency_s": 1.9344914169050753, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75497472, + "prompt_kind": "long", + "t_relative_s": 10435.200705999974 + }, + { + "decode_latency_s": 0.060305476567009464, + "kv_live_bytes": 462274560, + "latency_s": 1.9306503748521209, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75481088, + "prompt_kind": "short", + "t_relative_s": 10440.210784832947 + }, + { + "decode_latency_s": 0.06815337367879692, + "kv_live_bytes": 462274560, + "latency_s": 2.182560999877751, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75612160, + "prompt_kind": "long", + "t_relative_s": 10445.220858583227 + }, + { + "decode_latency_s": 0.06056026824808214, + "kv_live_bytes": 462274560, + "latency_s": 1.939045624807477, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75644928, + "prompt_kind": "short", + "t_relative_s": 10450.230909750331 + }, + { + "decode_latency_s": 0.061370463532512076, + "kv_live_bytes": 462274560, + "latency_s": 1.9648185828700662, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74366976, + "prompt_kind": "long", + "t_relative_s": 10455.240975708235 + }, + { + "decode_latency_s": 0.060471843753475696, + "kv_live_bytes": 462274560, + "latency_s": 1.9360129591077566, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73252864, + "prompt_kind": "short", + "t_relative_s": 10460.251018125098 + }, + { + "decode_latency_s": 0.06780096614966169, + "kv_live_bytes": 462274560, + "latency_s": 2.171313290949911, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75464704, + "prompt_kind": "long", + "t_relative_s": 10465.261099667288 + }, + { + "decode_latency_s": 0.060626532562309876, + "kv_live_bytes": 462274560, + "latency_s": 1.940895417239517, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75513856, + "prompt_kind": "short", + "t_relative_s": 10470.271162583027 + }, + { + "decode_latency_s": 0.06073579429357778, + "kv_live_bytes": 462274560, + "latency_s": 1.9448239589110017, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75563008, + "prompt_kind": "long", + "t_relative_s": 10475.28120524995 + }, + { + "decode_latency_s": 0.06075315366615541, + "kv_live_bytes": 462274560, + "latency_s": 1.9448896669782698, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75579392, + "prompt_kind": "short", + "t_relative_s": 10480.291273500305 + }, + { + "decode_latency_s": 0.06506621222069953, + "kv_live_bytes": 462274560, + "latency_s": 2.0844864998944104, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72843264, + "prompt_kind": "long", + "t_relative_s": 10485.301314875018 + }, + { + "decode_latency_s": 0.06065034506900702, + "kv_live_bytes": 462274560, + "latency_s": 1.9416695414111018, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72859648, + "prompt_kind": "short", + "t_relative_s": 10490.311361625325 + }, + { + "decode_latency_s": 0.06782470182224642, + "kv_live_bytes": 462274560, + "latency_s": 2.172621625009924, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73269248, + "prompt_kind": "long", + "t_relative_s": 10495.321409041993 + }, + { + "decode_latency_s": 0.06069907812343445, + "kv_live_bytes": 462274560, + "latency_s": 1.9431141251698136, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74268672, + "prompt_kind": "short", + "t_relative_s": 10500.331497624982 + }, + { + "decode_latency_s": 0.06795527602662332, + "kv_live_bytes": 462274560, + "latency_s": 2.176874625030905, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73596928, + "prompt_kind": "long", + "t_relative_s": 10505.34154412523 + }, + { + "decode_latency_s": 0.06153184507275, + "kv_live_bytes": 462274560, + "latency_s": 1.96998087503016, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71811072, + "prompt_kind": "short", + "t_relative_s": 10510.351635125 + }, + { + "decode_latency_s": 0.060948842437937856, + "kv_live_bytes": 462274560, + "latency_s": 1.9513588747940958, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72400896, + "prompt_kind": "long", + "t_relative_s": 10515.361705916934 + }, + { + "decode_latency_s": 0.06088544272643048, + "kv_live_bytes": 462274560, + "latency_s": 1.9492390416562557, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73039872, + "prompt_kind": "short", + "t_relative_s": 10520.371755292173 + }, + { + "decode_latency_s": 0.06862470571650192, + "kv_live_bytes": 462274560, + "latency_s": 2.198279249947518, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74432512, + "prompt_kind": "long", + "t_relative_s": 10525.381835167296 + }, + { + "decode_latency_s": 0.06053874609642662, + "kv_live_bytes": 462274560, + "latency_s": 1.9380437079817057, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74579968, + "prompt_kind": "short", + "t_relative_s": 10530.391930125188 + }, + { + "decode_latency_s": 0.06805688413442113, + "kv_live_bytes": 462274560, + "latency_s": 2.180238999892026, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74579968, + "prompt_kind": "long", + "t_relative_s": 10535.399177458137 + }, + { + "decode_latency_s": 0.06029879297420848, + "kv_live_bytes": 462274560, + "latency_s": 1.9304949999786913, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74645504, + "prompt_kind": "short", + "t_relative_s": 10540.409259499982 + }, + { + "decode_latency_s": 0.06085755077947397, + "kv_live_bytes": 462274560, + "latency_s": 1.9484031246975064, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72679424, + "prompt_kind": "long", + "t_relative_s": 10545.419349292293 + }, + { + "decode_latency_s": 0.06038139325391967, + "kv_live_bytes": 462274560, + "latency_s": 1.9332667919807136, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72220672, + "prompt_kind": "short", + "t_relative_s": 10550.42939429218 + }, + { + "decode_latency_s": 0.06512330468103755, + "kv_live_bytes": 462274560, + "latency_s": 2.085046958178282, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73564160, + "prompt_kind": "long", + "t_relative_s": 10555.439484333154 + }, + { + "decode_latency_s": 0.06053167187201325, + "kv_live_bytes": 462274560, + "latency_s": 1.9378378330729902, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73760768, + "prompt_kind": "short", + "t_relative_s": 10560.449538375251 + }, + { + "decode_latency_s": 0.06099301823996939, + "kv_live_bytes": 462274560, + "latency_s": 1.952726375311613, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73302016, + "prompt_kind": "long", + "t_relative_s": 10565.459636125248 + }, + { + "decode_latency_s": 0.06069481901067775, + "kv_live_bytes": 462274560, + "latency_s": 1.9430565419606864, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73351168, + "prompt_kind": "short", + "t_relative_s": 10570.469733375125 + }, + { + "decode_latency_s": 0.06879942578962073, + "kv_live_bytes": 462274560, + "latency_s": 2.2038799161091447, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73318400, + "prompt_kind": "long", + "t_relative_s": 10575.479806667194 + }, + { + "decode_latency_s": 0.060634986992226914, + "kv_live_bytes": 462274560, + "latency_s": 1.941105583216995, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74088448, + "prompt_kind": "short", + "t_relative_s": 10580.489867125172 + }, + { + "decode_latency_s": 0.06758278905181214, + "kv_live_bytes": 462274560, + "latency_s": 2.1647134167142212, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74186752, + "prompt_kind": "long", + "t_relative_s": 10585.499918708112 + }, + { + "decode_latency_s": 0.06054729296010919, + "kv_live_bytes": 462274560, + "latency_s": 1.9383127917535603, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74301440, + "prompt_kind": "short", + "t_relative_s": 10590.509997417219 + }, + { + "decode_latency_s": 0.0683635507884901, + "kv_live_bytes": 462274560, + "latency_s": 2.189124791882932, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73547776, + "prompt_kind": "long", + "t_relative_s": 10595.520079292357 + }, + { + "decode_latency_s": 0.06031673827965278, + "kv_live_bytes": 462274560, + "latency_s": 1.9310847502201796, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73203712, + "prompt_kind": "short", + "t_relative_s": 10600.5301337922 + }, + { + "decode_latency_s": 0.06774228256836068, + "kv_live_bytes": 462274560, + "latency_s": 2.169726707972586, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73809920, + "prompt_kind": "long", + "t_relative_s": 10605.540194250178 + }, + { + "decode_latency_s": 0.06067479168996215, + "kv_live_bytes": 462274560, + "latency_s": 1.9424317916855216, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74416128, + "prompt_kind": "short", + "t_relative_s": 10610.55026937509 + }, + { + "decode_latency_s": 0.06773745053214952, + "kv_live_bytes": 462274560, + "latency_s": 2.1700994577258825, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74530816, + "prompt_kind": "long", + "t_relative_s": 10615.550335207954 + }, + { + "decode_latency_s": 0.06057756509108003, + "kv_live_bytes": 462274560, + "latency_s": 1.93925566598773, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74629120, + "prompt_kind": "short", + "t_relative_s": 10620.560418083332 + }, + { + "decode_latency_s": 0.0633227486978285, + "kv_live_bytes": 462274560, + "latency_s": 2.0274508339352906, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70238208, + "prompt_kind": "long", + "t_relative_s": 10625.568361292128 + }, + { + "decode_latency_s": 0.06025804555974901, + "kv_live_bytes": 462274560, + "latency_s": 1.9291251669637859, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69648384, + "prompt_kind": "short", + "t_relative_s": 10630.744953624904 + }, + { + "decode_latency_s": 0.06836911587743089, + "kv_live_bytes": 462274560, + "latency_s": 2.19016549969092, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68501504, + "prompt_kind": "long", + "t_relative_s": 10635.755046833307 + }, + { + "decode_latency_s": 0.06059992060181685, + "kv_live_bytes": 462274560, + "latency_s": 1.93999854195863, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70893568, + "prompt_kind": "short", + "t_relative_s": 10640.765125083271 + }, + { + "decode_latency_s": 0.06785580077848863, + "kv_live_bytes": 462274560, + "latency_s": 2.174058166332543, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71008256, + "prompt_kind": "long", + "t_relative_s": 10645.775179666933 + }, + { + "decode_latency_s": 0.060363384094671346, + "kv_live_bytes": 462274560, + "latency_s": 1.9324264591559768, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70991872, + "prompt_kind": "short", + "t_relative_s": 10650.785269750282 + }, + { + "decode_latency_s": 0.06845726825122256, + "kv_live_bytes": 462274560, + "latency_s": 2.1924427920021117, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69746688, + "prompt_kind": "long", + "t_relative_s": 10655.795338707976 + }, + { + "decode_latency_s": 0.060275714844465256, + "kv_live_bytes": 462274560, + "latency_s": 1.9297836660407484, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69992448, + "prompt_kind": "short", + "t_relative_s": 10660.805383542087 + }, + { + "decode_latency_s": 0.06839506250980776, + "kv_live_bytes": 462274560, + "latency_s": 2.1911537502892315, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70402048, + "prompt_kind": "long", + "t_relative_s": 10665.815473542083 + }, + { + "decode_latency_s": 0.06070330990769435, + "kv_live_bytes": 462274560, + "latency_s": 1.943388540763408, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71450624, + "prompt_kind": "short", + "t_relative_s": 10670.825557083357 + }, + { + "decode_latency_s": 0.06045887632353697, + "kv_live_bytes": 462274560, + "latency_s": 1.935793333221227, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71467008, + "prompt_kind": "long", + "t_relative_s": 10675.835598292295 + }, + { + "decode_latency_s": 0.06047240625775885, + "kv_live_bytes": 462274560, + "latency_s": 1.9358734576962888, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71532544, + "prompt_kind": "short", + "t_relative_s": 10680.84567595832 + }, + { + "decode_latency_s": 0.068130494793877, + "kv_live_bytes": 462274560, + "latency_s": 2.1817719168029726, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70926336, + "prompt_kind": "long", + "t_relative_s": 10685.855751583353 + }, + { + "decode_latency_s": 0.06038058853300754, + "kv_live_bytes": 462274560, + "latency_s": 1.9330730000510812, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70713344, + "prompt_kind": "short", + "t_relative_s": 10690.86334716715 + }, + { + "decode_latency_s": 0.06846554687945172, + "kv_live_bytes": 462274560, + "latency_s": 2.1928387503139675, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70926336, + "prompt_kind": "long", + "t_relative_s": 10695.873417958152 + }, + { + "decode_latency_s": 0.06575274090573657, + "kv_live_bytes": 462274560, + "latency_s": 2.1050750422291458, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66404352, + "prompt_kind": "short", + "t_relative_s": 10700.883460625075 + }, + { + "decode_latency_s": 0.06791072527994402, + "kv_live_bytes": 462274560, + "latency_s": 2.1750900419428945, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68943872, + "prompt_kind": "long", + "t_relative_s": 10705.89355041692 + }, + { + "decode_latency_s": 0.06076608202420175, + "kv_live_bytes": 462274560, + "latency_s": 1.945280082989484, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69599232, + "prompt_kind": "short", + "t_relative_s": 10710.903602542356 + }, + { + "decode_latency_s": 0.060788446629885584, + "kv_live_bytes": 462274560, + "latency_s": 1.9463212909176946, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69763072, + "prompt_kind": "long", + "t_relative_s": 10715.91365183331 + }, + { + "decode_latency_s": 0.06062146743352059, + "kv_live_bytes": 462274560, + "latency_s": 1.9407146251760423, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70303744, + "prompt_kind": "short", + "t_relative_s": 10720.923733042087 + }, + { + "decode_latency_s": 0.06828571484948043, + "kv_live_bytes": 462274560, + "latency_s": 2.187423000112176, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69533696, + "prompt_kind": "long", + "t_relative_s": 10725.93385308329 + }, + { + "decode_latency_s": 0.06081932420784142, + "kv_live_bytes": 462274560, + "latency_s": 1.9469832917675376, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69484544, + "prompt_kind": "short", + "t_relative_s": 10730.943976708222 + }, + { + "decode_latency_s": 0.06799333203525748, + "kv_live_bytes": 462274560, + "latency_s": 2.1780312499031425, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70008832, + "prompt_kind": "long", + "t_relative_s": 10735.954032917041 + }, + { + "decode_latency_s": 0.06039175781188533, + "kv_live_bytes": 462274560, + "latency_s": 1.9333679592236876, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70074368, + "prompt_kind": "short", + "t_relative_s": 10740.964155917056 + }, + { + "decode_latency_s": 0.06843472266336903, + "kv_live_bytes": 462274560, + "latency_s": 2.1913966247811913, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68976640, + "prompt_kind": "long", + "t_relative_s": 10745.974231041968 + }, + { + "decode_latency_s": 0.06715711067954544, + "kv_live_bytes": 462274560, + "latency_s": 2.1508987080305815, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69943296, + "prompt_kind": "short", + "t_relative_s": 10750.984329583123 + }, + { + "decode_latency_s": 0.06902291536971461, + "kv_live_bytes": 462274560, + "latency_s": 2.210654624737799, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69042176, + "prompt_kind": "long", + "t_relative_s": 10755.994407332968 + }, + { + "decode_latency_s": 0.06042542187788058, + "kv_live_bytes": 462274560, + "latency_s": 1.9344534161500633, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68698112, + "prompt_kind": "short", + "t_relative_s": 10761.004470292013 + }, + { + "decode_latency_s": 0.06818880340142641, + "kv_live_bytes": 462274560, + "latency_s": 2.183777166996151, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70434816, + "prompt_kind": "long", + "t_relative_s": 10766.014551333152 + }, + { + "decode_latency_s": 0.06049271352821961, + "kv_live_bytes": 462274560, + "latency_s": 1.93659762525931, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70500352, + "prompt_kind": "short", + "t_relative_s": 10771.015064458363 + }, + { + "decode_latency_s": 0.06819852734042797, + "kv_live_bytes": 462274560, + "latency_s": 2.1849105418659747, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70615040, + "prompt_kind": "long", + "t_relative_s": 10776.02514112508 + }, + { + "decode_latency_s": 0.06066282684332691, + "kv_live_bytes": 462274560, + "latency_s": 1.9419785412028432, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71139328, + "prompt_kind": "short", + "t_relative_s": 10781.035255458206 + }, + { + "decode_latency_s": 0.0697372981812805, + "kv_live_bytes": 462274560, + "latency_s": 2.2336712921969593, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 65552384, + "prompt_kind": "long", + "t_relative_s": 10786.045301917009 + }, + { + "decode_latency_s": 0.06656075263163075, + "kv_live_bytes": 462274560, + "latency_s": 2.1317875417880714, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68485120, + "prompt_kind": "short", + "t_relative_s": 10791.054444000125 + }, + { + "decode_latency_s": 0.06803742967895232, + "kv_live_bytes": 462274560, + "latency_s": 2.1795164169743657, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69943296, + "prompt_kind": "long", + "t_relative_s": 10796.06452295836 + }, + { + "decode_latency_s": 0.06104636850068346, + "kv_live_bytes": 462274560, + "latency_s": 1.954294040799141, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69517312, + "prompt_kind": "short", + "t_relative_s": 10801.074597250205 + }, + { + "decode_latency_s": 0.06938776302558836, + "kv_live_bytes": 462274560, + "latency_s": 2.2218194170854986, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70205440, + "prompt_kind": "long", + "t_relative_s": 10806.084641917143 + }, + { + "decode_latency_s": 0.06039639453229029, + "kv_live_bytes": 462274560, + "latency_s": 1.9336273749358952, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70729728, + "prompt_kind": "short", + "t_relative_s": 10811.094723958056 + }, + { + "decode_latency_s": 0.06802148959832266, + "kv_live_bytes": 462274560, + "latency_s": 2.178303875029087, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69795840, + "prompt_kind": "long", + "t_relative_s": 10816.104796583299 + }, + { + "decode_latency_s": 0.060445933588198386, + "kv_live_bytes": 462274560, + "latency_s": 1.9351733326911926, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70254592, + "prompt_kind": "short", + "t_relative_s": 10821.11484841723 + }, + { + "decode_latency_s": 0.06812234893732239, + "kv_live_bytes": 462274560, + "latency_s": 2.1818613749928772, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71221248, + "prompt_kind": "long", + "t_relative_s": 10826.124930292368 + }, + { + "decode_latency_s": 0.06308216533216182, + "kv_live_bytes": 462274560, + "latency_s": 2.019558749627322, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68009984, + "prompt_kind": "short", + "t_relative_s": 10831.126189458184 + }, + { + "decode_latency_s": 0.0681605703139212, + "kv_live_bytes": 462274560, + "latency_s": 2.1838855827227235, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68698112, + "prompt_kind": "long", + "t_relative_s": 10836.13624329213 + }, + { + "decode_latency_s": 0.060423722650739364, + "kv_live_bytes": 462274560, + "latency_s": 1.934425333980471, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69222400, + "prompt_kind": "short", + "t_relative_s": 10841.146299083252 + }, + { + "decode_latency_s": 0.0683020625001518, + "kv_live_bytes": 462274560, + "latency_s": 2.1872572917491198, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69255168, + "prompt_kind": "long", + "t_relative_s": 10846.155300583225 + }, + { + "decode_latency_s": 0.060858768250909634, + "kv_live_bytes": 462274560, + "latency_s": 1.9483586251735687, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69058560, + "prompt_kind": "short", + "t_relative_s": 10851.16534562502 + }, + { + "decode_latency_s": 0.06769912369782105, + "kv_live_bytes": 462274560, + "latency_s": 2.1680376660078764, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69337088, + "prompt_kind": "long", + "t_relative_s": 10856.175419458188 + }, + { + "decode_latency_s": 0.060871300782309845, + "kv_live_bytes": 462274560, + "latency_s": 1.9486858327873051, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69320704, + "prompt_kind": "short", + "t_relative_s": 10861.185479958076 + }, + { + "decode_latency_s": 0.06820842968591023, + "kv_live_bytes": 462274560, + "latency_s": 2.184758000075817, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69304320, + "prompt_kind": "long", + "t_relative_s": 10866.195532416925 + }, + { + "decode_latency_s": 0.06046143748972099, + "kv_live_bytes": 462274560, + "latency_s": 1.9356518746353686, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69402624, + "prompt_kind": "short", + "t_relative_s": 10871.20559104206 + }, + { + "decode_latency_s": 0.06795451694051735, + "kv_live_bytes": 462274560, + "latency_s": 2.1774445828050375, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68222976, + "prompt_kind": "long", + "t_relative_s": 10876.215649208054 + }, + { + "decode_latency_s": 0.060625494792475365, + "kv_live_bytes": 462274560, + "latency_s": 1.9407740831375122, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70402048, + "prompt_kind": "short", + "t_relative_s": 10881.225722458214 + }, + { + "decode_latency_s": 0.06811770831700414, + "kv_live_bytes": 462274560, + "latency_s": 2.1813135417178273, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69926912, + "prompt_kind": "long", + "t_relative_s": 10886.235800249968 + }, + { + "decode_latency_s": 0.06034467059362214, + "kv_live_bytes": 462274560, + "latency_s": 1.9319630828686059, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69992448, + "prompt_kind": "short", + "t_relative_s": 10891.245855417103 + }, + { + "decode_latency_s": 0.0684455676964717, + "kv_live_bytes": 462274560, + "latency_s": 2.192241624929011, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70107136, + "prompt_kind": "long", + "t_relative_s": 10896.255932417233 + }, + { + "decode_latency_s": 0.06070457422174513, + "kv_live_bytes": 462274560, + "latency_s": 1.9434061250649393, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70647808, + "prompt_kind": "short", + "t_relative_s": 10901.263328208122 + }, + { + "decode_latency_s": 0.06837624477338977, + "kv_live_bytes": 462274560, + "latency_s": 2.1908719167113304, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69943296, + "prompt_kind": "long", + "t_relative_s": 10906.273371417075 + }, + { + "decode_latency_s": 0.060597799470997415, + "kv_live_bytes": 462274560, + "latency_s": 1.9399238750338554, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70582272, + "prompt_kind": "short", + "t_relative_s": 10911.283434667159 + }, + { + "decode_latency_s": 0.06782299480983056, + "kv_live_bytes": 462274560, + "latency_s": 2.1726435418240726, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71122944, + "prompt_kind": "long", + "t_relative_s": 10916.293478208128 + }, + { + "decode_latency_s": 0.0603042421862483, + "kv_live_bytes": 462274560, + "latency_s": 1.9306375002488494, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70647808, + "prompt_kind": "short", + "t_relative_s": 10921.303507000208 + }, + { + "decode_latency_s": 0.06861032293818425, + "kv_live_bytes": 462274560, + "latency_s": 2.1976582501083612, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71286784, + "prompt_kind": "long", + "t_relative_s": 10926.313588958234 + }, + { + "decode_latency_s": 0.06078628252726048, + "kv_live_bytes": 462274560, + "latency_s": 1.9460223326459527, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71860224, + "prompt_kind": "short", + "t_relative_s": 10931.323646957986 + }, + { + "decode_latency_s": 0.06819014974462334, + "kv_live_bytes": 462274560, + "latency_s": 2.1846148329786956, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71286784, + "prompt_kind": "long", + "t_relative_s": 10936.333702375181 + }, + { + "decode_latency_s": 0.06323646356759127, + "kv_live_bytes": 462274560, + "latency_s": 2.024502709042281, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71008256, + "prompt_kind": "short", + "t_relative_s": 10941.34377537528 + }, + { + "decode_latency_s": 0.06770804297411814, + "kv_live_bytes": 462274560, + "latency_s": 2.169210707768798, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71942144, + "prompt_kind": "long", + "t_relative_s": 10946.353842624929 + }, + { + "decode_latency_s": 0.06054865365149453, + "kv_live_bytes": 462274560, + "latency_s": 1.938385207671672, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72040448, + "prompt_kind": "short", + "t_relative_s": 10951.363997499924 + }, + { + "decode_latency_s": 0.06781295961991418, + "kv_live_bytes": 462274560, + "latency_s": 2.172359958756715, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72122368, + "prompt_kind": "long", + "t_relative_s": 10956.37404050026 + }, + { + "decode_latency_s": 0.060651536463410594, + "kv_live_bytes": 462274560, + "latency_s": 1.9417556668631732, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71974912, + "prompt_kind": "short", + "t_relative_s": 10961.384115917142 + }, + { + "decode_latency_s": 0.06877186066412833, + "kv_live_bytes": 462274560, + "latency_s": 2.2023321660235524, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74743808, + "prompt_kind": "long", + "t_relative_s": 10966.394191583153 + }, + { + "decode_latency_s": 0.06061430991394445, + "kv_live_bytes": 462274560, + "latency_s": 1.940600958187133, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75448320, + "prompt_kind": "short", + "t_relative_s": 10971.404247042257 + }, + { + "decode_latency_s": 0.06803558465617243, + "kv_live_bytes": 462274560, + "latency_s": 2.178928917273879, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76546048, + "prompt_kind": "long", + "t_relative_s": 10976.41431783326 + }, + { + "decode_latency_s": 0.060734454440535046, + "kv_live_bytes": 462274560, + "latency_s": 1.9443988339044154, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76595200, + "prompt_kind": "short", + "t_relative_s": 10981.424406958278 + }, + { + "decode_latency_s": 0.06810049217892811, + "kv_live_bytes": 462274560, + "latency_s": 2.1815372919663787, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76513280, + "prompt_kind": "long", + "t_relative_s": 10986.434459500015 + }, + { + "decode_latency_s": 0.060435894527472556, + "kv_live_bytes": 462274560, + "latency_s": 1.9346932917833328, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77053952, + "prompt_kind": "short", + "t_relative_s": 10991.444530792069 + }, + { + "decode_latency_s": 0.06861910028965212, + "kv_live_bytes": 462274560, + "latency_s": 2.1978227919898927, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74432512, + "prompt_kind": "long", + "t_relative_s": 10996.454633791931 + }, + { + "decode_latency_s": 0.060763601562939584, + "kv_live_bytes": 462274560, + "latency_s": 1.9451976660639048, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76627968, + "prompt_kind": "short", + "t_relative_s": 11001.464723750018 + }, + { + "decode_latency_s": 0.06778998437221162, + "kv_live_bytes": 462274560, + "latency_s": 2.171857957728207, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76791808, + "prompt_kind": "long", + "t_relative_s": 11006.474778250325 + }, + { + "decode_latency_s": 0.06031496745708864, + "kv_live_bytes": 462274560, + "latency_s": 1.9309516670182347, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76611584, + "prompt_kind": "short", + "t_relative_s": 11011.484842792153 + }, + { + "decode_latency_s": 0.06830421225458849, + "kv_live_bytes": 462274560, + "latency_s": 2.18746625026688, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76677120, + "prompt_kind": "long", + "t_relative_s": 11016.494918208104 + }, + { + "decode_latency_s": 0.06714751954132225, + "kv_live_bytes": 462274560, + "latency_s": 2.150457375217229, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77316096, + "prompt_kind": "short", + "t_relative_s": 11021.504972583149 + }, + { + "decode_latency_s": 0.06795894012611825, + "kv_live_bytes": 462274560, + "latency_s": 2.177737249992788, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77217792, + "prompt_kind": "long", + "t_relative_s": 11026.515055292286 + }, + { + "decode_latency_s": 0.06719486849033274, + "kv_live_bytes": 462274560, + "latency_s": 2.151953165885061, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77381632, + "prompt_kind": "short", + "t_relative_s": 11031.525125416927 + }, + { + "decode_latency_s": 0.06764565233606845, + "kv_live_bytes": 462274560, + "latency_s": 2.1669547921046615, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77185024, + "prompt_kind": "long", + "t_relative_s": 11036.535193417221 + }, + { + "decode_latency_s": 0.06047145182674285, + "kv_live_bytes": 462274560, + "latency_s": 1.9359916248358786, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76840960, + "prompt_kind": "short", + "t_relative_s": 11041.545250541996 + }, + { + "decode_latency_s": 0.06831881508696824, + "kv_live_bytes": 462274560, + "latency_s": 2.187838749960065, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 76873728, + "prompt_kind": "long", + "t_relative_s": 11046.555331707932 + }, + { + "decode_latency_s": 0.060657734371488914, + "kv_live_bytes": 462274560, + "latency_s": 1.9419071250595152, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77414400, + "prompt_kind": "short", + "t_relative_s": 11051.565401874948 + }, + { + "decode_latency_s": 0.06794698047451675, + "kv_live_bytes": 462274560, + "latency_s": 2.1766339587047696, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77479936, + "prompt_kind": "long", + "t_relative_s": 11056.575471375138 + }, + { + "decode_latency_s": 0.06070974610338453, + "kv_live_bytes": 462274560, + "latency_s": 1.9435141254216433, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78069760, + "prompt_kind": "short", + "t_relative_s": 11061.585554417223 + }, + { + "decode_latency_s": 0.06802535025053658, + "kv_live_bytes": 462274560, + "latency_s": 2.178896375000477, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78086144, + "prompt_kind": "long", + "t_relative_s": 11066.595603791997 + }, + { + "decode_latency_s": 0.06047835024946835, + "kv_live_bytes": 462274560, + "latency_s": 1.936181957833469, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74694656, + "prompt_kind": "short", + "t_relative_s": 11071.605656458065 + }, + { + "decode_latency_s": 0.06803367055545095, + "kv_live_bytes": 462274560, + "latency_s": 2.178952833171934, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74711040, + "prompt_kind": "long", + "t_relative_s": 11076.615741458256 + }, + { + "decode_latency_s": 0.06623783333634492, + "kv_live_bytes": 462274560, + "latency_s": 2.1212582918815315, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74694656, + "prompt_kind": "short", + "t_relative_s": 11081.625806625001 + }, + { + "decode_latency_s": 0.06511213150224648, + "kv_live_bytes": 462274560, + "latency_s": 2.085017249919474, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67796992, + "prompt_kind": "long", + "t_relative_s": 11086.635870042257 + }, + { + "decode_latency_s": 0.06727492705977056, + "kv_live_bytes": 462274560, + "latency_s": 2.1548832920379937, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71614464, + "prompt_kind": "short", + "t_relative_s": 11091.645938416943 + }, + { + "decode_latency_s": 0.06806653515377548, + "kv_live_bytes": 462274560, + "latency_s": 2.1796882078051567, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70860800, + "prompt_kind": "long", + "t_relative_s": 11096.653444041964 + }, + { + "decode_latency_s": 0.06122893486462999, + "kv_live_bytes": 462274560, + "latency_s": 1.9601367502473295, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71680000, + "prompt_kind": "short", + "t_relative_s": 11101.663553917315 + }, + { + "decode_latency_s": 0.0679228645603871, + "kv_live_bytes": 462274560, + "latency_s": 2.1757516246289015, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72253440, + "prompt_kind": "long", + "t_relative_s": 11106.67362358328 + }, + { + "decode_latency_s": 0.060467307281214744, + "kv_live_bytes": 462274560, + "latency_s": 1.9358437079936266, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72777728, + "prompt_kind": "short", + "t_relative_s": 11111.683705416974 + }, + { + "decode_latency_s": 0.06803236978885252, + "kv_live_bytes": 462274560, + "latency_s": 2.17867024987936, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72990720, + "prompt_kind": "long", + "t_relative_s": 11116.693785833195 + }, + { + "decode_latency_s": 0.06711052864557132, + "kv_live_bytes": 462274560, + "latency_s": 2.1491990839131176, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73564160, + "prompt_kind": "short", + "t_relative_s": 11121.699826333206 + }, + { + "decode_latency_s": 0.06042030337266624, + "kv_live_bytes": 462274560, + "latency_s": 1.934425000101328, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73580544, + "prompt_kind": "long", + "t_relative_s": 11126.709908000194 + }, + { + "decode_latency_s": 0.06063891795929521, + "kv_live_bytes": 462274560, + "latency_s": 1.9412491251714528, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73629696, + "prompt_kind": "short", + "t_relative_s": 11131.719999332912 + }, + { + "decode_latency_s": 0.060477029939647764, + "kv_live_bytes": 462274560, + "latency_s": 1.9365155412815511, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73678848, + "prompt_kind": "long", + "t_relative_s": 11136.730051750317 + }, + { + "decode_latency_s": 0.06072484374453779, + "kv_live_bytes": 462274560, + "latency_s": 1.944124750327319, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73629696, + "prompt_kind": "short", + "t_relative_s": 11141.740130042192 + }, + { + "decode_latency_s": 0.06074837368214503, + "kv_live_bytes": 462274560, + "latency_s": 1.9455770421773195, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72990720, + "prompt_kind": "long", + "t_relative_s": 11146.750185417011 + }, + { + "decode_latency_s": 0.170569811220048, + "kv_live_bytes": 462274560, + "latency_s": 5.459243708755821, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 65159168, + "prompt_kind": "short", + "t_relative_s": 11151.760267667007 + }, + { + "decode_latency_s": 0.06750672655471135, + "kv_live_bytes": 462274560, + "latency_s": 2.1622861670330167, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68042752, + "prompt_kind": "long", + "t_relative_s": 11161.147078332957 + }, + { + "decode_latency_s": 0.06053491927741561, + "kv_live_bytes": 462274560, + "latency_s": 1.9379984168335795, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68763648, + "prompt_kind": "short", + "t_relative_s": 11166.157131500077 + }, + { + "decode_latency_s": 0.06784219140536152, + "kv_live_bytes": 462274560, + "latency_s": 2.1729509588330984, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69058560, + "prompt_kind": "long", + "t_relative_s": 11171.16721162526 + }, + { + "decode_latency_s": 0.06092326431826223, + "kv_live_bytes": 462274560, + "latency_s": 1.950398666318506, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69599232, + "prompt_kind": "short", + "t_relative_s": 11176.177275999915 + }, + { + "decode_latency_s": 0.06782219267915934, + "kv_live_bytes": 462274560, + "latency_s": 2.1724945418536663, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69779456, + "prompt_kind": "long", + "t_relative_s": 11181.187322708312 + }, + { + "decode_latency_s": 0.060617967435973696, + "kv_live_bytes": 462274560, + "latency_s": 1.9405817077495158, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70303744, + "prompt_kind": "short", + "t_relative_s": 11186.197400792036 + }, + { + "decode_latency_s": 0.06777217319176998, + "kv_live_bytes": 462274560, + "latency_s": 2.1706595830619335, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70369280, + "prompt_kind": "long", + "t_relative_s": 11191.207483208273 + }, + { + "decode_latency_s": 0.06053897656966001, + "kv_live_bytes": 462274560, + "latency_s": 1.938134958036244, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70434816, + "prompt_kind": "short", + "t_relative_s": 11196.21754045831 + }, + { + "decode_latency_s": 0.06832458724966273, + "kv_live_bytes": 462274560, + "latency_s": 2.1884665829129517, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70582272, + "prompt_kind": "long", + "t_relative_s": 11201.22762400005 + }, + { + "decode_latency_s": 0.06066759375971742, + "kv_live_bytes": 462274560, + "latency_s": 1.9421505411155522, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71188480, + "prompt_kind": "short", + "t_relative_s": 11206.237707042135 + }, + { + "decode_latency_s": 0.06789865625614766, + "kv_live_bytes": 462274560, + "latency_s": 2.1752792918123305, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71286784, + "prompt_kind": "long", + "t_relative_s": 11211.247758542188 + }, + { + "decode_latency_s": 0.06046705337939784, + "kv_live_bytes": 462274560, + "latency_s": 1.935704582836479, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71188480, + "prompt_kind": "short", + "t_relative_s": 11216.257821250241 + }, + { + "decode_latency_s": 0.060492889329907484, + "kv_live_bytes": 462274560, + "latency_s": 1.9366689161397517, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71286784, + "prompt_kind": "long", + "t_relative_s": 11221.267899292056 + }, + { + "decode_latency_s": 0.06043507684080396, + "kv_live_bytes": 462274560, + "latency_s": 1.9348056665621698, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70680576, + "prompt_kind": "short", + "t_relative_s": 11226.277951792348 + }, + { + "decode_latency_s": 0.06764444531290792, + "kv_live_bytes": 462274560, + "latency_s": 2.1672033751383424, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70844416, + "prompt_kind": "long", + "t_relative_s": 11231.288036042359 + }, + { + "decode_latency_s": 0.060751296870876104, + "kv_live_bytes": 462274560, + "latency_s": 1.9448669161647558, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71385088, + "prompt_kind": "short", + "t_relative_s": 11236.298135333229 + }, + { + "decode_latency_s": 0.06113083071250003, + "kv_live_bytes": 462274560, + "latency_s": 1.95757225016132, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71467008, + "prompt_kind": "long", + "t_relative_s": 11241.30819116719 + }, + { + "decode_latency_s": 0.06052170571638271, + "kv_live_bytes": 462274560, + "latency_s": 1.937572750262916, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71909376, + "prompt_kind": "short", + "t_relative_s": 11246.318269125186 + }, + { + "decode_latency_s": 0.06806059893278871, + "kv_live_bytes": 462274560, + "latency_s": 2.1799620841629803, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71876608, + "prompt_kind": "long", + "t_relative_s": 11251.328346417286 + }, + { + "decode_latency_s": 0.06060806510504335, + "kv_live_bytes": 462274560, + "latency_s": 1.940388583112508, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71942144, + "prompt_kind": "short", + "t_relative_s": 11256.338402042165 + }, + { + "decode_latency_s": 0.06088859765441157, + "kv_live_bytes": 462274560, + "latency_s": 1.9499179162085056, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71925760, + "prompt_kind": "long", + "t_relative_s": 11261.348473333288 + }, + { + "decode_latency_s": 0.060706278658472, + "kv_live_bytes": 462274560, + "latency_s": 1.9434062498621643, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71991296, + "prompt_kind": "short", + "t_relative_s": 11266.35854850011 + }, + { + "decode_latency_s": 0.06811907941300888, + "kv_live_bytes": 462274560, + "latency_s": 2.1822989997453988, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72040448, + "prompt_kind": "long", + "t_relative_s": 11271.368592625018 + }, + { + "decode_latency_s": 0.060466631533927284, + "kv_live_bytes": 462274560, + "latency_s": 1.9359042919240892, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72597504, + "prompt_kind": "short", + "t_relative_s": 11276.378676542081 + }, + { + "decode_latency_s": 0.06750307031325065, + "kv_live_bytes": 462274560, + "latency_s": 2.1616192501969635, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72024064, + "prompt_kind": "long", + "t_relative_s": 11281.388749583159 + }, + { + "decode_latency_s": 0.06708205991890281, + "kv_live_bytes": 462274560, + "latency_s": 2.148596915882081, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72138752, + "prompt_kind": "short", + "t_relative_s": 11286.398805249948 + }, + { + "decode_latency_s": 0.06817887758370489, + "kv_live_bytes": 462274560, + "latency_s": 2.184098625089973, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72253440, + "prompt_kind": "long", + "t_relative_s": 11291.40887466725 + }, + { + "decode_latency_s": 0.06074797917972319, + "kv_live_bytes": 462274560, + "latency_s": 1.9447245839983225, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72826880, + "prompt_kind": "short", + "t_relative_s": 11296.418960625306 + }, + { + "decode_latency_s": 0.0677998723986093, + "kv_live_bytes": 462274560, + "latency_s": 2.1717922911047935, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72908800, + "prompt_kind": "long", + "t_relative_s": 11301.429004082922 + }, + { + "decode_latency_s": 0.06036296093952842, + "kv_live_bytes": 462274560, + "latency_s": 1.9324996657669544, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73416704, + "prompt_kind": "short", + "t_relative_s": 11306.439082249999 + }, + { + "decode_latency_s": 0.06809398178302217, + "kv_live_bytes": 462274560, + "latency_s": 2.181131082586944, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73383936, + "prompt_kind": "long", + "t_relative_s": 11311.449180708267 + }, + { + "decode_latency_s": 0.060863350285217166, + "kv_live_bytes": 462274560, + "latency_s": 1.948502709157765, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73531392, + "prompt_kind": "short", + "t_relative_s": 11316.459249667358 + }, + { + "decode_latency_s": 0.06808765497407876, + "kv_live_bytes": 462274560, + "latency_s": 2.1812752918340266, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73646080, + "prompt_kind": "long", + "t_relative_s": 11321.469316666946 + }, + { + "decode_latency_s": 0.0667248789104633, + "kv_live_bytes": 462274560, + "latency_s": 2.1369757913053036, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74268672, + "prompt_kind": "short", + "t_relative_s": 11326.479246250354 + }, + { + "decode_latency_s": 0.068289389309939, + "kv_live_bytes": 462274560, + "latency_s": 2.187741417437792, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74317824, + "prompt_kind": "long", + "t_relative_s": 11331.489298083354 + }, + { + "decode_latency_s": 0.060257869787164964, + "kv_live_bytes": 462274560, + "latency_s": 1.929121000226587, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74088448, + "prompt_kind": "short", + "t_relative_s": 11336.499375124928 + }, + { + "decode_latency_s": 0.06796756903349888, + "kv_live_bytes": 462274560, + "latency_s": 2.1770233749412, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74121216, + "prompt_kind": "long", + "t_relative_s": 11341.509465833195 + }, + { + "decode_latency_s": 0.06073621223913506, + "kv_live_bytes": 462274560, + "latency_s": 1.944390666205436, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74219520, + "prompt_kind": "short", + "t_relative_s": 11346.519517833367 + }, + { + "decode_latency_s": 0.06803769399994053, + "kv_live_bytes": 462274560, + "latency_s": 2.17961816675961, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74268672, + "prompt_kind": "long", + "t_relative_s": 11351.529594458174 + }, + { + "decode_latency_s": 0.060829770838608965, + "kv_live_bytes": 462274560, + "latency_s": 1.9473370420746505, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73957376, + "prompt_kind": "short", + "t_relative_s": 11356.539678083267 + }, + { + "decode_latency_s": 0.06759347135084681, + "kv_live_bytes": 462274560, + "latency_s": 2.165106583852321, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73891840, + "prompt_kind": "long", + "t_relative_s": 11361.549727207981 + }, + { + "decode_latency_s": 0.06035843618155923, + "kv_live_bytes": 462274560, + "latency_s": 1.9324307087808847, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73924608, + "prompt_kind": "short", + "t_relative_s": 11366.559794625267 + }, + { + "decode_latency_s": 0.06827624086872675, + "kv_live_bytes": 462274560, + "latency_s": 2.1865018331445754, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74498048, + "prompt_kind": "long", + "t_relative_s": 11371.56986925006 + }, + { + "decode_latency_s": 0.06094162500812672, + "kv_live_bytes": 462274560, + "latency_s": 1.9509980827569962, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75890688, + "prompt_kind": "short", + "t_relative_s": 11376.579934583046 + }, + { + "decode_latency_s": 0.06807808073062915, + "kv_live_bytes": 462274560, + "latency_s": 2.1810030420310795, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75939840, + "prompt_kind": "long", + "t_relative_s": 11381.590024583042 + }, + { + "decode_latency_s": 0.06610247524804436, + "kv_live_bytes": 462274560, + "latency_s": 2.1161501253955066, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 49954816, + "prompt_kind": "short", + "t_relative_s": 11386.600103000179 + }, + { + "decode_latency_s": 0.06854924609069712, + "kv_live_bytes": 462274560, + "latency_s": 2.1955764167942107, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70991872, + "prompt_kind": "long", + "t_relative_s": 11391.610179125331 + }, + { + "decode_latency_s": 0.0660105937422486, + "kv_live_bytes": 462274560, + "latency_s": 2.114098208025098, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73318400, + "prompt_kind": "short", + "t_relative_s": 11396.620236250106 + }, + { + "decode_latency_s": 0.06770689190307166, + "kv_live_bytes": 462274560, + "latency_s": 2.168703665956855, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72925184, + "prompt_kind": "long", + "t_relative_s": 11401.630354083143 + }, + { + "decode_latency_s": 0.061066167982062325, + "kv_live_bytes": 462274560, + "latency_s": 1.9549725828692317, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73023488, + "prompt_kind": "short", + "t_relative_s": 11406.640474624932 + }, + { + "decode_latency_s": 0.06781505727849435, + "kv_live_bytes": 462274560, + "latency_s": 2.1725187497213483, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73187328, + "prompt_kind": "long", + "t_relative_s": 11411.65051229205 + }, + { + "decode_latency_s": 0.06274014321388677, + "kv_live_bytes": 462274560, + "latency_s": 2.0085462909191847, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73154560, + "prompt_kind": "short", + "t_relative_s": 11416.660569333006 + }, + { + "decode_latency_s": 0.06788562110159546, + "kv_live_bytes": 462274560, + "latency_s": 2.1748331249691546, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73744384, + "prompt_kind": "long", + "t_relative_s": 11421.670671250205 + }, + { + "decode_latency_s": 0.06083989713806659, + "kv_live_bytes": 462274560, + "latency_s": 1.9477647920139134, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74366976, + "prompt_kind": "short", + "t_relative_s": 11426.67469183309 + }, + { + "decode_latency_s": 0.06767979946744163, + "kv_live_bytes": 462274560, + "latency_s": 2.1679427078925073, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74498048, + "prompt_kind": "long", + "t_relative_s": 11431.684752332978 + }, + { + "decode_latency_s": 0.06060454164980911, + "kv_live_bytes": 462274560, + "latency_s": 1.9402754590846598, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74612736, + "prompt_kind": "short", + "t_relative_s": 11436.694841207936 + }, + { + "decode_latency_s": 0.06086032549501397, + "kv_live_bytes": 462274560, + "latency_s": 1.9488054169341922, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71073792, + "prompt_kind": "long", + "t_relative_s": 11441.704931292217 + }, + { + "decode_latency_s": 0.06059171483502723, + "kv_live_bytes": 462274560, + "latency_s": 1.9397932076826692, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73515008, + "prompt_kind": "short", + "t_relative_s": 11446.71501383325 + }, + { + "decode_latency_s": 0.06700419008848257, + "kv_live_bytes": 462274560, + "latency_s": 2.1465864591300488, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74153984, + "prompt_kind": "long", + "t_relative_s": 11451.725059458055 + }, + { + "decode_latency_s": 0.06107965233968571, + "kv_live_bytes": 462274560, + "latency_s": 1.9553923751227558, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74711040, + "prompt_kind": "short", + "t_relative_s": 11456.735157333314 + }, + { + "decode_latency_s": 0.0608053671894595, + "kv_live_bytes": 462274560, + "latency_s": 1.9467642079107463, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71958528, + "prompt_kind": "long", + "t_relative_s": 11461.745208083186 + }, + { + "decode_latency_s": 0.06065760419005528, + "kv_live_bytes": 462274560, + "latency_s": 1.9417944164015353, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72040448, + "prompt_kind": "short", + "t_relative_s": 11466.755292583257 + }, + { + "decode_latency_s": 0.06848187498690095, + "kv_live_bytes": 462274560, + "latency_s": 2.1931579592637718, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73940992, + "prompt_kind": "long", + "t_relative_s": 11471.765328875277 + }, + { + "decode_latency_s": 0.06065470834437292, + "kv_live_bytes": 462274560, + "latency_s": 1.9418955422006547, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73302016, + "prompt_kind": "short", + "t_relative_s": 11476.775386041962 + }, + { + "decode_latency_s": 0.06848780209838878, + "kv_live_bytes": 462274560, + "latency_s": 2.196363208349794, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74055680, + "prompt_kind": "long", + "t_relative_s": 11481.785434625112 + }, + { + "decode_latency_s": 0.060400825503165834, + "kv_live_bytes": 462274560, + "latency_s": 1.9336962075904012, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74711040, + "prompt_kind": "short", + "t_relative_s": 11486.795533375349 + }, + { + "decode_latency_s": 0.06847948305949103, + "kv_live_bytes": 462274560, + "latency_s": 2.192854166030884, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68190208, + "prompt_kind": "long", + "t_relative_s": 11491.805594542064 + }, + { + "decode_latency_s": 0.06093864844297059, + "kv_live_bytes": 462274560, + "latency_s": 1.9509719996713102, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72843264, + "prompt_kind": "short", + "t_relative_s": 11496.815671250224 + }, + { + "decode_latency_s": 0.06859646743396297, + "kv_live_bytes": 462274560, + "latency_s": 2.1972457910887897, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74022912, + "prompt_kind": "long", + "t_relative_s": 11501.82571220817 + }, + { + "decode_latency_s": 0.0606056536489632, + "kv_live_bytes": 462274560, + "latency_s": 1.940283834002912, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73269248, + "prompt_kind": "short", + "t_relative_s": 11506.835790750105 + }, + { + "decode_latency_s": 0.06837882031686604, + "kv_live_bytes": 462274560, + "latency_s": 2.1890505412593484, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88473600, + "prompt_kind": "long", + "t_relative_s": 11511.845865542069 + }, + { + "decode_latency_s": 0.061301713532884605, + "kv_live_bytes": 462274560, + "latency_s": 1.9625964588485658, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 87048192, + "prompt_kind": "short", + "t_relative_s": 11516.855916792061 + }, + { + "decode_latency_s": 0.06780525262001902, + "kv_live_bytes": 462274560, + "latency_s": 2.171940708067268, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90554368, + "prompt_kind": "long", + "t_relative_s": 11521.866012917366 + }, + { + "decode_latency_s": 0.06063878643908538, + "kv_live_bytes": 462274560, + "latency_s": 1.9412225410342216, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90652672, + "prompt_kind": "short", + "t_relative_s": 11526.876074292231 + }, + { + "decode_latency_s": 0.06814776040846482, + "kv_live_bytes": 462274560, + "latency_s": 2.1830286672338843, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90652672, + "prompt_kind": "long", + "t_relative_s": 11531.88613395812 + }, + { + "decode_latency_s": 0.0629502838710323, + "kv_live_bytes": 462274560, + "latency_s": 2.0153394169174135, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90275840, + "prompt_kind": "short", + "t_relative_s": 11536.896231375169 + }, + { + "decode_latency_s": 0.0680539895693073, + "kv_live_bytes": 462274560, + "latency_s": 2.1801775419153273, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90652672, + "prompt_kind": "long", + "t_relative_s": 11541.90632366715 + }, + { + "decode_latency_s": 0.06096551561495289, + "kv_live_bytes": 462274560, + "latency_s": 1.951694083865732, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91258880, + "prompt_kind": "short", + "t_relative_s": 11546.906752957962 + }, + { + "decode_latency_s": 0.060384703130694106, + "kv_live_bytes": 462274560, + "latency_s": 1.9334808751009405, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74219520, + "prompt_kind": "long", + "t_relative_s": 11551.916797833052 + }, + { + "decode_latency_s": 0.060539669284480624, + "kv_live_bytes": 462274560, + "latency_s": 1.9382145423442125, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74268672, + "prompt_kind": "short", + "t_relative_s": 11556.926869875286 + }, + { + "decode_latency_s": 0.06067421221814584, + "kv_live_bytes": 462274560, + "latency_s": 1.9424814581871033, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70172672, + "prompt_kind": "long", + "t_relative_s": 11561.93695516698 + }, + { + "decode_latency_s": 0.061067835937137716, + "kv_live_bytes": 462274560, + "latency_s": 1.9550177496857941, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71041024, + "prompt_kind": "short", + "t_relative_s": 11566.947044833098 + }, + { + "decode_latency_s": 0.06044439584366046, + "kv_live_bytes": 462274560, + "latency_s": 1.9351522498764098, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71385088, + "prompt_kind": "long", + "t_relative_s": 11571.957122582942 + }, + { + "decode_latency_s": 0.060355486974003725, + "kv_live_bytes": 462274560, + "latency_s": 1.9322434170171618, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71319552, + "prompt_kind": "short", + "t_relative_s": 11576.967193375342 + }, + { + "decode_latency_s": 0.06760601824498735, + "kv_live_bytes": 462274560, + "latency_s": 2.165399417281151, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73646080, + "prompt_kind": "long", + "t_relative_s": 11581.977246042341 + }, + { + "decode_latency_s": 0.060430930971051566, + "kv_live_bytes": 462274560, + "latency_s": 1.934576999861747, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73826304, + "prompt_kind": "short", + "t_relative_s": 11586.987298625056 + }, + { + "decode_latency_s": 0.06798665755195543, + "kv_live_bytes": 462274560, + "latency_s": 2.177375125233084, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73596928, + "prompt_kind": "long", + "t_relative_s": 11591.997385083232 + }, + { + "decode_latency_s": 0.06054967187810689, + "kv_live_bytes": 462274560, + "latency_s": 1.9384567090310156, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73236480, + "prompt_kind": "short", + "t_relative_s": 11597.007437082939 + }, + { + "decode_latency_s": 0.06801474999519996, + "kv_live_bytes": 462274560, + "latency_s": 2.1783913327381015, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73826304, + "prompt_kind": "long", + "t_relative_s": 11602.017526499927 + }, + { + "decode_latency_s": 0.06057382293511182, + "kv_live_bytes": 462274560, + "latency_s": 1.9391420828178525, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74350592, + "prompt_kind": "short", + "t_relative_s": 11607.027637583204 + }, + { + "decode_latency_s": 0.06720506772398949, + "kv_live_bytes": 462274560, + "latency_s": 2.1530422498472035, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74547200, + "prompt_kind": "long", + "t_relative_s": 11612.037696917076 + }, + { + "decode_latency_s": 0.06054486590437591, + "kv_live_bytes": 462274560, + "latency_s": 1.9382404158823192, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74645504, + "prompt_kind": "short", + "t_relative_s": 11617.047787375282 + }, + { + "decode_latency_s": 0.061004337258054875, + "kv_live_bytes": 462274560, + "latency_s": 1.9530354170128703, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69681152, + "prompt_kind": "long", + "t_relative_s": 11622.057854167186 + }, + { + "decode_latency_s": 0.06306271225912496, + "kv_live_bytes": 462274560, + "latency_s": 2.0192087083123624, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68419584, + "prompt_kind": "short", + "t_relative_s": 11627.067943083122 + }, + { + "decode_latency_s": 0.06819746484688949, + "kv_live_bytes": 462274560, + "latency_s": 2.1849260423332453, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73089024, + "prompt_kind": "long", + "t_relative_s": 11632.078015583102 + }, + { + "decode_latency_s": 0.06059880732209422, + "kv_live_bytes": 462274560, + "latency_s": 1.9399665840901434, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73777152, + "prompt_kind": "short", + "t_relative_s": 11637.088089250028 + }, + { + "decode_latency_s": 0.06714502735121641, + "kv_live_bytes": 462274560, + "latency_s": 2.1514947921968997, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73744384, + "prompt_kind": "long", + "t_relative_s": 11642.09813775029 + }, + { + "decode_latency_s": 0.06800459764781408, + "kv_live_bytes": 462274560, + "latency_s": 2.1771147921681404, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91193344, + "prompt_kind": "short", + "t_relative_s": 11647.108226375189 + }, + { + "decode_latency_s": 0.06860037631122395, + "kv_live_bytes": 462274560, + "latency_s": 2.1973653747700155, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90324992, + "prompt_kind": "long", + "t_relative_s": 11652.118279417045 + }, + { + "decode_latency_s": 0.06090137371211313, + "kv_live_bytes": 462274560, + "latency_s": 1.9499712083488703, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88719360, + "prompt_kind": "short", + "t_relative_s": 11657.128355250228 + }, + { + "decode_latency_s": 0.06854924350045621, + "kv_live_bytes": 462274560, + "latency_s": 2.195789041928947, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90931200, + "prompt_kind": "long", + "t_relative_s": 11662.138422042131 + }, + { + "decode_latency_s": 0.06034507160075009, + "kv_live_bytes": 462274560, + "latency_s": 1.9319213340058923, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91570176, + "prompt_kind": "short", + "t_relative_s": 11667.148471792229 + }, + { + "decode_latency_s": 0.06827405469084624, + "kv_live_bytes": 462274560, + "latency_s": 2.186956999823451, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91766784, + "prompt_kind": "long", + "t_relative_s": 11672.158578208182 + }, + { + "decode_latency_s": 0.0669156627409393, + "kv_live_bytes": 462274560, + "latency_s": 2.1431044172495604, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91930624, + "prompt_kind": "short", + "t_relative_s": 11677.168652958237 + }, + { + "decode_latency_s": 0.06831838413199876, + "kv_live_bytes": 462274560, + "latency_s": 2.1880018752999604, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73646080, + "prompt_kind": "long", + "t_relative_s": 11682.178704208229 + }, + { + "decode_latency_s": 0.06680842577770818, + "kv_live_bytes": 462274560, + "latency_s": 2.1390127083286643, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 65667072, + "prompt_kind": "short", + "t_relative_s": 11687.188751833048 + }, + { + "decode_latency_s": 0.0686461562581826, + "kv_live_bytes": 462274560, + "latency_s": 2.1984247909858823, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73531392, + "prompt_kind": "long", + "t_relative_s": 11692.198810042348 + }, + { + "decode_latency_s": 0.06057084244093858, + "kv_live_bytes": 462274560, + "latency_s": 1.9392404998652637, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70500352, + "prompt_kind": "short", + "t_relative_s": 11697.208864083048 + }, + { + "decode_latency_s": 0.06643172005715314, + "kv_live_bytes": 462274560, + "latency_s": 2.1281534167937934, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73711616, + "prompt_kind": "long", + "t_relative_s": 11702.218918667175 + }, + { + "decode_latency_s": 0.06030408461811021, + "kv_live_bytes": 462274560, + "latency_s": 1.9305972498841584, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73252864, + "prompt_kind": "short", + "t_relative_s": 11707.228999667335 + }, + { + "decode_latency_s": 0.06830015884770546, + "kv_live_bytes": 462274560, + "latency_s": 2.187455916777253, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73744384, + "prompt_kind": "long", + "t_relative_s": 11712.239085208159 + }, + { + "decode_latency_s": 0.060824096377473325, + "kv_live_bytes": 462274560, + "latency_s": 1.947316374629736, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68059136, + "prompt_kind": "short", + "t_relative_s": 11717.249125875067 + }, + { + "decode_latency_s": 0.06844604556681588, + "kv_live_bytes": 462274560, + "latency_s": 2.1927137090824544, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73318400, + "prompt_kind": "long", + "t_relative_s": 11722.259204083122 + }, + { + "decode_latency_s": 0.06052576821821276, + "kv_live_bytes": 462274560, + "latency_s": 1.9376422921195626, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74039296, + "prompt_kind": "short", + "t_relative_s": 11727.269270292018 + }, + { + "decode_latency_s": 0.0681912304717116, + "kv_live_bytes": 462274560, + "latency_s": 2.184387667104602, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74498048, + "prompt_kind": "long", + "t_relative_s": 11732.279319417197 + }, + { + "decode_latency_s": 0.060535440090461634, + "kv_live_bytes": 462274560, + "latency_s": 1.9379112920723855, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74530816, + "prompt_kind": "short", + "t_relative_s": 11737.289404875133 + }, + { + "decode_latency_s": 0.06913800913025625, + "kv_live_bytes": 462274560, + "latency_s": 2.2143787923268974, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72744960, + "prompt_kind": "long", + "t_relative_s": 11742.299460417125 + }, + { + "decode_latency_s": 0.061814497399609536, + "kv_live_bytes": 462274560, + "latency_s": 1.9789448748342693, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72105984, + "prompt_kind": "short", + "t_relative_s": 11747.309533042368 + }, + { + "decode_latency_s": 0.17430418096773792, + "kv_live_bytes": 462274560, + "latency_s": 5.5786101669073105, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67862528, + "prompt_kind": "long", + "t_relative_s": 11752.31957920827 + }, + { + "decode_latency_s": 0.06031286981306039, + "kv_live_bytes": 462274560, + "latency_s": 1.9309049169532955, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68550656, + "prompt_kind": "short", + "t_relative_s": 11760.740496166982 + }, + { + "decode_latency_s": 0.06870187890308443, + "kv_live_bytes": 462274560, + "latency_s": 2.200213958043605, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68747264, + "prompt_kind": "long", + "t_relative_s": 11765.75057420833 + }, + { + "decode_latency_s": 0.060516833342262544, + "kv_live_bytes": 462274560, + "latency_s": 1.937346207909286, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69435392, + "prompt_kind": "short", + "t_relative_s": 11770.760648875032 + }, + { + "decode_latency_s": 0.06901311327237636, + "kv_live_bytes": 462274560, + "latency_s": 2.210480500012636, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 65961984, + "prompt_kind": "long", + "t_relative_s": 11775.770704458002 + }, + { + "decode_latency_s": 0.060323601559503004, + "kv_live_bytes": 462274560, + "latency_s": 1.931344416923821, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68141056, + "prompt_kind": "short", + "t_relative_s": 11780.78076654207 + }, + { + "decode_latency_s": 0.07049756772175897, + "kv_live_bytes": 462274560, + "latency_s": 2.2570884586311877, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 89915392, + "prompt_kind": "long", + "t_relative_s": 11785.790827624965 + }, + { + "decode_latency_s": 0.061546966127934866, + "kv_live_bytes": 462274560, + "latency_s": 1.970306457951665, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90488832, + "prompt_kind": "short", + "t_relative_s": 11790.800893417094 + }, + { + "decode_latency_s": 0.06757203774759546, + "kv_live_bytes": 462274560, + "latency_s": 2.1646915832534432, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90554368, + "prompt_kind": "long", + "t_relative_s": 11795.810947000049 + }, + { + "decode_latency_s": 0.0604895807482535, + "kv_live_bytes": 462274560, + "latency_s": 1.936432915739715, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90636288, + "prompt_kind": "short", + "t_relative_s": 11800.820948458277 + }, + { + "decode_latency_s": 0.06848775650723837, + "kv_live_bytes": 462274560, + "latency_s": 2.193626417312771, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 89948160, + "prompt_kind": "long", + "t_relative_s": 11805.831008207984 + }, + { + "decode_latency_s": 0.060751554687158205, + "kv_live_bytes": 462274560, + "latency_s": 1.944864375051111, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90521600, + "prompt_kind": "short", + "t_relative_s": 11810.841079958249 + }, + { + "decode_latency_s": 0.06066574477881659, + "kv_live_bytes": 462274560, + "latency_s": 1.942583791911602, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90701824, + "prompt_kind": "long", + "t_relative_s": 11815.851126166992 + }, + { + "decode_latency_s": 0.06028447656717617, + "kv_live_bytes": 462274560, + "latency_s": 1.929960499983281, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90619904, + "prompt_kind": "short", + "t_relative_s": 11820.861185375135 + }, + { + "decode_latency_s": 0.06881077216530684, + "kv_live_bytes": 462274560, + "latency_s": 2.2037384156137705, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91226112, + "prompt_kind": "long", + "t_relative_s": 11825.871261416934 + }, + { + "decode_latency_s": 0.060457494779257104, + "kv_live_bytes": 462274560, + "latency_s": 1.9354906249791384, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91357184, + "prompt_kind": "short", + "t_relative_s": 11830.881308125332 + }, + { + "decode_latency_s": 0.06830701041326392, + "kv_live_bytes": 462274560, + "latency_s": 2.188435500022024, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90587136, + "prompt_kind": "long", + "t_relative_s": 11835.891407625284 + }, + { + "decode_latency_s": 0.06677598696842324, + "kv_live_bytes": 462274560, + "latency_s": 2.138766041956842, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91242496, + "prompt_kind": "short", + "t_relative_s": 11840.901477625128 + }, + { + "decode_latency_s": 0.06791719920875039, + "kv_live_bytes": 462274560, + "latency_s": 2.1756909997202456, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91242496, + "prompt_kind": "long", + "t_relative_s": 11845.91152512515 + }, + { + "decode_latency_s": 0.06026782939443365, + "kv_live_bytes": 462274560, + "latency_s": 1.9294876670464873, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91193344, + "prompt_kind": "short", + "t_relative_s": 11850.921623167116 + }, + { + "decode_latency_s": 0.06042775390960742, + "kv_live_bytes": 462274560, + "latency_s": 1.9345789169892669, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91193344, + "prompt_kind": "long", + "t_relative_s": 11855.93169441726 + }, + { + "decode_latency_s": 0.06677452866279054, + "kv_live_bytes": 462274560, + "latency_s": 2.1385985407978296, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91258880, + "prompt_kind": "short", + "t_relative_s": 11860.941750708036 + }, + { + "decode_latency_s": 0.06821777606091928, + "kv_live_bytes": 462274560, + "latency_s": 2.185297125019133, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91275264, + "prompt_kind": "long", + "t_relative_s": 11865.951858083252 + }, + { + "decode_latency_s": 0.060584731778362766, + "kv_live_bytes": 462274560, + "latency_s": 1.9394928752444685, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91832320, + "prompt_kind": "short", + "t_relative_s": 11870.961926292162 + }, + { + "decode_latency_s": 0.060566218744497746, + "kv_live_bytes": 462274560, + "latency_s": 1.9390031662769616, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91832320, + "prompt_kind": "long", + "t_relative_s": 11875.971989833284 + }, + { + "decode_latency_s": 0.060323330719256774, + "kv_live_bytes": 462274560, + "latency_s": 1.9312187083996832, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 92241920, + "prompt_kind": "short", + "t_relative_s": 11880.982051375322 + }, + { + "decode_latency_s": 0.06858194920641836, + "kv_live_bytes": 462274560, + "latency_s": 2.1962877502664924, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91914240, + "prompt_kind": "long", + "t_relative_s": 11885.992136875167 + }, + { + "decode_latency_s": 0.06051701694377698, + "kv_live_bytes": 462274560, + "latency_s": 1.9374147499911487, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 91963392, + "prompt_kind": "short", + "t_relative_s": 11891.002205375116 + }, + { + "decode_latency_s": 0.06060880079166964, + "kv_live_bytes": 462274560, + "latency_s": 1.940643667243421, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71188480, + "prompt_kind": "long", + "t_relative_s": 11896.01228391705 + }, + { + "decode_latency_s": 0.06078389716276433, + "kv_live_bytes": 462274560, + "latency_s": 1.945899416692555, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71843840, + "prompt_kind": "short", + "t_relative_s": 11901.022362667136 + }, + { + "decode_latency_s": 0.06818730078521185, + "kv_live_bytes": 462274560, + "latency_s": 2.1842637499794364, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71892992, + "prompt_kind": "long", + "t_relative_s": 11906.027683957946 + }, + { + "decode_latency_s": 0.060378395806765184, + "kv_live_bytes": 462274560, + "latency_s": 1.9330090419389307, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72302592, + "prompt_kind": "short", + "t_relative_s": 11911.037771875039 + }, + { + "decode_latency_s": 0.06807230990671087, + "kv_live_bytes": 462274560, + "latency_s": 2.1803262503817677, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71860224, + "prompt_kind": "long", + "t_relative_s": 11916.04785737535 + }, + { + "decode_latency_s": 0.06484793880372308, + "kv_live_bytes": 462274560, + "latency_s": 2.07598870806396, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 89505792, + "prompt_kind": "short", + "t_relative_s": 11921.057898292318 + }, + { + "decode_latency_s": 0.06799349481298123, + "kv_live_bytes": 462274560, + "latency_s": 2.1782364998944104, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 89423872, + "prompt_kind": "long", + "t_relative_s": 11926.06798116723 + }, + { + "decode_latency_s": 0.06141072134778369, + "kv_live_bytes": 462274560, + "latency_s": 1.965949125122279, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90062848, + "prompt_kind": "short", + "t_relative_s": 11931.07806575019 + }, + { + "decode_latency_s": 0.06802014193090145, + "kv_live_bytes": 462274560, + "latency_s": 2.178968334104866, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88358912, + "prompt_kind": "long", + "t_relative_s": 11936.08827862516 + }, + { + "decode_latency_s": 0.06041471353091765, + "kv_live_bytes": 462274560, + "latency_s": 1.9340493329800665, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90439680, + "prompt_kind": "short", + "t_relative_s": 11941.09834758332 + }, + { + "decode_latency_s": 0.06828670832328498, + "kv_live_bytes": 462274560, + "latency_s": 2.1875349576584995, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90456064, + "prompt_kind": "long", + "t_relative_s": 11946.108400125057 + }, + { + "decode_latency_s": 0.060594087219215, + "kv_live_bytes": 462274560, + "latency_s": 1.9398123752325773, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90406912, + "prompt_kind": "short", + "t_relative_s": 11951.118492207956 + }, + { + "decode_latency_s": 0.0684599935048027, + "kv_live_bytes": 462274560, + "latency_s": 2.1929817497730255, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 89604096, + "prompt_kind": "long", + "t_relative_s": 11956.12853912497 + }, + { + "decode_latency_s": 0.06030586326960474, + "kv_live_bytes": 462274560, + "latency_s": 1.9307335829362273, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90095616, + "prompt_kind": "short", + "t_relative_s": 11961.138607167173 + }, + { + "decode_latency_s": 0.06776464974973351, + "kv_live_bytes": 462274560, + "latency_s": 2.1705324999056756, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90210304, + "prompt_kind": "long", + "t_relative_s": 11966.1486800001 + }, + { + "decode_latency_s": 0.06059016668586992, + "kv_live_bytes": 462274560, + "latency_s": 1.9397534998133779, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90800128, + "prompt_kind": "short", + "t_relative_s": 11971.158727000002 + }, + { + "decode_latency_s": 0.06821944272087421, + "kv_live_bytes": 462274560, + "latency_s": 2.185310082975775, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90882048, + "prompt_kind": "long", + "t_relative_s": 11976.168795292266 + }, + { + "decode_latency_s": 0.06054946615768131, + "kv_live_bytes": 462274560, + "latency_s": 1.9383710422553122, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 90963968, + "prompt_kind": "short", + "t_relative_s": 11981.178866750095 + }, + { + "decode_latency_s": 0.06854413803375792, + "kv_live_bytes": 462274560, + "latency_s": 2.1953324167989194, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68747264, + "prompt_kind": "long", + "t_relative_s": 11986.188920750283 + }, + { + "decode_latency_s": 0.06594852084526792, + "kv_live_bytes": 462274560, + "latency_s": 2.112128082662821, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71483392, + "prompt_kind": "short", + "t_relative_s": 11991.198982333299 + }, + { + "decode_latency_s": 0.06804072916565929, + "kv_live_bytes": 462274560, + "latency_s": 2.179753000382334, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72712192, + "prompt_kind": "long", + "t_relative_s": 11996.209062833339 + }, + { + "decode_latency_s": 0.06686188280582428, + "kv_live_bytes": 462274560, + "latency_s": 2.141231916844845, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73351168, + "prompt_kind": "short", + "t_relative_s": 12001.219128458295 + }, + { + "decode_latency_s": 0.06834567578334827, + "kv_live_bytes": 462274560, + "latency_s": 2.1897309999912977, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73433088, + "prompt_kind": "long", + "t_relative_s": 12006.229202917311 + }, + { + "decode_latency_s": 0.060311343739158474, + "kv_live_bytes": 462274560, + "latency_s": 1.9308476247824728, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73007104, + "prompt_kind": "short", + "t_relative_s": 12011.23927487526 + }, + { + "decode_latency_s": 0.068412322900258, + "kv_live_bytes": 462274560, + "latency_s": 2.1907887905836105, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72597504, + "prompt_kind": "long", + "t_relative_s": 12016.249347958248 + }, + { + "decode_latency_s": 0.06697625390370376, + "kv_live_bytes": 462274560, + "latency_s": 2.144789499696344, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73515008, + "prompt_kind": "short", + "t_relative_s": 12021.259401042014 + }, + { + "decode_latency_s": 0.06803421874064952, + "kv_live_bytes": 462274560, + "latency_s": 2.1794537086971104, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73744384, + "prompt_kind": "long", + "t_relative_s": 12026.2695035832 + }, + { + "decode_latency_s": 0.06675049867772032, + "kv_live_bytes": 462274560, + "latency_s": 2.1379614579491317, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74350592, + "prompt_kind": "short", + "t_relative_s": 12031.279605207965 + }, + { + "decode_latency_s": 0.06848659114621114, + "kv_live_bytes": 462274560, + "latency_s": 2.193657541181892, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74399744, + "prompt_kind": "long", + "t_relative_s": 12036.289661792107 + }, + { + "decode_latency_s": 0.06040168881008867, + "kv_live_bytes": 462274560, + "latency_s": 1.9337522499263287, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73826304, + "prompt_kind": "short", + "t_relative_s": 12041.29974012496 + }, + { + "decode_latency_s": 0.06867190366028808, + "kv_live_bytes": 462274560, + "latency_s": 2.19975558295846, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70991872, + "prompt_kind": "long", + "t_relative_s": 12046.309816000052 + }, + { + "decode_latency_s": 0.06063204556994606, + "kv_live_bytes": 462274560, + "latency_s": 1.9411635003052652, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73613312, + "prompt_kind": "short", + "t_relative_s": 12051.319876417052 + }, + { + "decode_latency_s": 0.0677522825426422, + "kv_live_bytes": 462274560, + "latency_s": 2.1706743338145316, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78430208, + "prompt_kind": "long", + "t_relative_s": 12056.329947292339 + }, + { + "decode_latency_s": 0.06640959765354637, + "kv_live_bytes": 462274560, + "latency_s": 2.1267543751746416, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78741504, + "prompt_kind": "short", + "t_relative_s": 12061.340021792334 + }, + { + "decode_latency_s": 0.06783591795829125, + "kv_live_bytes": 462274560, + "latency_s": 2.173259750008583, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79822848, + "prompt_kind": "long", + "t_relative_s": 12066.350050583016 + }, + { + "decode_latency_s": 0.06374675262486562, + "kv_live_bytes": 462274560, + "latency_s": 2.0409959158860147, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78053376, + "prompt_kind": "short", + "t_relative_s": 12071.360115708318 + }, + { + "decode_latency_s": 0.06774293880152982, + "kv_live_bytes": 462274560, + "latency_s": 2.170000209007412, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79134720, + "prompt_kind": "long", + "t_relative_s": 12076.37017370807 + }, + { + "decode_latency_s": 0.06052855467714835, + "kv_live_bytes": 462274560, + "latency_s": 1.9377029589377344, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79200256, + "prompt_kind": "short", + "t_relative_s": 12081.380240083206 + }, + { + "decode_latency_s": 0.06814198959909845, + "kv_live_bytes": 462274560, + "latency_s": 2.1827631248161197, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79331328, + "prompt_kind": "long", + "t_relative_s": 12086.390292041935 + }, + { + "decode_latency_s": 0.06048942315101158, + "kv_live_bytes": 462274560, + "latency_s": 1.9364282498136163, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79790080, + "prompt_kind": "short", + "t_relative_s": 12091.400369917043 + }, + { + "decode_latency_s": 0.06800268232473172, + "kv_live_bytes": 462274560, + "latency_s": 2.1777124581858516, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78692352, + "prompt_kind": "long", + "t_relative_s": 12096.410419000313 + }, + { + "decode_latency_s": 0.060352079395670444, + "kv_live_bytes": 462274560, + "latency_s": 1.9322599591687322, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68730880, + "prompt_kind": "short", + "t_relative_s": 12101.42045537522 + }, + { + "decode_latency_s": 0.06841043359600008, + "kv_live_bytes": 462274560, + "latency_s": 2.19097250001505, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74022912, + "prompt_kind": "long", + "t_relative_s": 12106.430513292085 + }, + { + "decode_latency_s": 0.06108236718864646, + "kv_live_bytes": 462274560, + "latency_s": 1.9556569578126073, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71630848, + "prompt_kind": "short", + "t_relative_s": 12111.44056012528 + }, + { + "decode_latency_s": 0.0675408749957569, + "kv_live_bytes": 462274560, + "latency_s": 2.1636573341675103, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72368128, + "prompt_kind": "long", + "t_relative_s": 12116.450617583003 + }, + { + "decode_latency_s": 0.06053736197645776, + "kv_live_bytes": 462274560, + "latency_s": 1.9380205422639847, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73023488, + "prompt_kind": "short", + "t_relative_s": 12121.460709000006 + }, + { + "decode_latency_s": 0.06826668097346555, + "kv_live_bytes": 462274560, + "latency_s": 2.1862141247838736, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73039872, + "prompt_kind": "long", + "t_relative_s": 12126.470794500317 + }, + { + "decode_latency_s": 0.06079582289385144, + "kv_live_bytes": 462274560, + "latency_s": 1.9464260833337903, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71991296, + "prompt_kind": "short", + "t_relative_s": 12131.480844791979 + }, + { + "decode_latency_s": 0.06828922005661298, + "kv_live_bytes": 462274560, + "latency_s": 2.187331000342965, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72400896, + "prompt_kind": "long", + "t_relative_s": 12136.490931042004 + }, + { + "decode_latency_s": 0.06059075391385704, + "kv_live_bytes": 462274560, + "latency_s": 1.939709875266999, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72613888, + "prompt_kind": "short", + "t_relative_s": 12141.500974291936 + }, + { + "decode_latency_s": 0.0681617395603098, + "kv_live_bytes": 462274560, + "latency_s": 2.1833870839327574, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "long", + "t_relative_s": 12146.511050375178 + }, + { + "decode_latency_s": 0.06710160028887913, + "kv_live_bytes": 462274560, + "latency_s": 2.148867500014603, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73121792, + "prompt_kind": "short", + "t_relative_s": 12151.521126166917 + }, + { + "decode_latency_s": 0.060678911468130536, + "kv_live_bytes": 462274560, + "latency_s": 1.9431546670384705, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73039872, + "prompt_kind": "long", + "t_relative_s": 12156.53116104193 + }, + { + "decode_latency_s": 0.06015461587230675, + "kv_live_bytes": 462274560, + "latency_s": 1.9258967908099294, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71827456, + "prompt_kind": "short", + "t_relative_s": 12161.541224417277 + }, + { + "decode_latency_s": 0.06076595053309575, + "kv_live_bytes": 462274560, + "latency_s": 1.945755708962679, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70795264, + "prompt_kind": "long", + "t_relative_s": 12166.551308458205 + }, + { + "decode_latency_s": 0.0665575924795121, + "kv_live_bytes": 462274560, + "latency_s": 2.131628416944295, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71565312, + "prompt_kind": "short", + "t_relative_s": 12171.561357667204 + }, + { + "decode_latency_s": 0.06804114974511322, + "kv_live_bytes": 462274560, + "latency_s": 2.179458625148982, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72237056, + "prompt_kind": "long", + "t_relative_s": 12176.571476125158 + }, + { + "decode_latency_s": 0.06055552343605086, + "kv_live_bytes": 462274560, + "latency_s": 1.9386054170317948, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72794112, + "prompt_kind": "short", + "t_relative_s": 12181.581581417006 + }, + { + "decode_latency_s": 0.06802313671505544, + "kv_live_bytes": 462274560, + "latency_s": 2.179129040800035, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72794112, + "prompt_kind": "long", + "t_relative_s": 12186.591637917329 + }, + { + "decode_latency_s": 0.06203959896811284, + "kv_live_bytes": 462274560, + "latency_s": 1.9861859171651304, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 82821120, + "prompt_kind": "short", + "t_relative_s": 12191.601719500031 + }, + { + "decode_latency_s": 0.06817565493111033, + "kv_live_bytes": 462274560, + "latency_s": 2.183706416748464, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 82788352, + "prompt_kind": "long", + "t_relative_s": 12196.611761583015 + }, + { + "decode_latency_s": 0.06056633462139871, + "kv_live_bytes": 462274560, + "latency_s": 1.9390190420672297, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 83361792, + "prompt_kind": "short", + "t_relative_s": 12201.621844167355 + }, + { + "decode_latency_s": 0.06792688541463576, + "kv_live_bytes": 462274560, + "latency_s": 2.175219625234604, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 82624512, + "prompt_kind": "long", + "t_relative_s": 12206.631925249938 + }, + { + "decode_latency_s": 0.06029224609665107, + "kv_live_bytes": 462274560, + "latency_s": 1.9302652077749372, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 83099648, + "prompt_kind": "short", + "t_relative_s": 12211.641968333162 + }, + { + "decode_latency_s": 0.06833311196533032, + "kv_live_bytes": 462274560, + "latency_s": 2.188750416971743, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 83853312, + "prompt_kind": "long", + "t_relative_s": 12216.652045125142 + }, + { + "decode_latency_s": 0.060515218749060296, + "kv_live_bytes": 462274560, + "latency_s": 1.9372894582338631, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 84017152, + "prompt_kind": "short", + "t_relative_s": 12221.662111333106 + }, + { + "decode_latency_s": 0.0614861289068358, + "kv_live_bytes": 462274560, + "latency_s": 1.9685254590585828, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79036416, + "prompt_kind": "long", + "t_relative_s": 12226.67218691716 + }, + { + "decode_latency_s": 0.060385403645341285, + "kv_live_bytes": 462274560, + "latency_s": 1.9332340830005705, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68698112, + "prompt_kind": "short", + "t_relative_s": 12231.68226583302 + }, + { + "decode_latency_s": 0.060266608066740446, + "kv_live_bytes": 462274560, + "latency_s": 1.9294495838694274, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69795840, + "prompt_kind": "long", + "t_relative_s": 12236.692406458315 + }, + { + "decode_latency_s": 0.06073337109410204, + "kv_live_bytes": 462274560, + "latency_s": 1.9444221672601998, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70418432, + "prompt_kind": "short", + "t_relative_s": 12241.702451958321 + }, + { + "decode_latency_s": 0.06805600911320653, + "kv_live_bytes": 462274560, + "latency_s": 2.180113833397627, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73023488, + "prompt_kind": "long", + "t_relative_s": 12246.71254483331 + }, + { + "decode_latency_s": 0.06645940493035596, + "kv_live_bytes": 462274560, + "latency_s": 2.1277047088369727, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 40058880, + "prompt_kind": "short", + "t_relative_s": 12251.722641333006 + }, + { + "decode_latency_s": 0.0683059778821189, + "kv_live_bytes": 462274560, + "latency_s": 2.1879805829375982, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71270400, + "prompt_kind": "long", + "t_relative_s": 12256.732736125123 + }, + { + "decode_latency_s": 0.06726237760449294, + "kv_live_bytes": 462274560, + "latency_s": 2.154225541744381, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72531968, + "prompt_kind": "short", + "t_relative_s": 12261.742789958138 + }, + { + "decode_latency_s": 0.06781838934693951, + "kv_live_bytes": 462274560, + "latency_s": 2.1725820833817124, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "long", + "t_relative_s": 12266.752908250317 + }, + { + "decode_latency_s": 0.06063890234509017, + "kv_live_bytes": 462274560, + "latency_s": 1.9412767910398543, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73351168, + "prompt_kind": "short", + "t_relative_s": 12271.76297695795 + }, + { + "decode_latency_s": 0.06802449087263085, + "kv_live_bytes": 462274560, + "latency_s": 2.1792213330045342, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73318400, + "prompt_kind": "long", + "t_relative_s": 12276.77302316716 + }, + { + "decode_latency_s": 0.06301358202472329, + "kv_live_bytes": 462274560, + "latency_s": 2.017794458195567, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72204288, + "prompt_kind": "short", + "t_relative_s": 12281.78308650013 + }, + { + "decode_latency_s": 0.0665631458396092, + "kv_live_bytes": 462274560, + "latency_s": 2.1320356247015297, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72761344, + "prompt_kind": "long", + "t_relative_s": 12286.793161833193 + }, + { + "decode_latency_s": 0.06699499087699223, + "kv_live_bytes": 462274560, + "latency_s": 2.145975916646421, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72859648, + "prompt_kind": "short", + "t_relative_s": 12291.803226875141 + }, + { + "decode_latency_s": 0.06803354296425823, + "kv_live_bytes": 462274560, + "latency_s": 2.178874040953815, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73482240, + "prompt_kind": "long", + "t_relative_s": 12296.806971250102 + }, + { + "decode_latency_s": 0.060237921876250766, + "kv_live_bytes": 462274560, + "latency_s": 1.9285459169186652, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72925184, + "prompt_kind": "short", + "t_relative_s": 12301.817049417179 + }, + { + "decode_latency_s": 0.06823134116712026, + "kv_live_bytes": 462274560, + "latency_s": 2.185677208006382, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73416704, + "prompt_kind": "long", + "t_relative_s": 12306.827105167322 + }, + { + "decode_latency_s": 0.06053378515935037, + "kv_live_bytes": 462274560, + "latency_s": 1.9378919582813978, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73482240, + "prompt_kind": "short", + "t_relative_s": 12311.83718600031 + }, + { + "decode_latency_s": 0.06848526561225299, + "kv_live_bytes": 462274560, + "latency_s": 2.193434542044997, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73236480, + "prompt_kind": "long", + "t_relative_s": 12316.847264000215 + }, + { + "decode_latency_s": 0.06037917059438769, + "kv_live_bytes": 462274560, + "latency_s": 1.933083708398044, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73302016, + "prompt_kind": "short", + "t_relative_s": 12321.857304542325 + }, + { + "decode_latency_s": 0.06134595703042578, + "kv_live_bytes": 462274560, + "latency_s": 1.9643642911687493, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79986688, + "prompt_kind": "long", + "t_relative_s": 12326.86737699993 + }, + { + "decode_latency_s": 0.06064971744490322, + "kv_live_bytes": 462274560, + "latency_s": 1.9416121668182313, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 80625664, + "prompt_kind": "short", + "t_relative_s": 12331.877434582915 + }, + { + "decode_latency_s": 0.06804838932293933, + "kv_live_bytes": 462274560, + "latency_s": 2.1796928751282394, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 80642048, + "prompt_kind": "long", + "t_relative_s": 12336.887492833193 + }, + { + "decode_latency_s": 0.06314783202833496, + "kv_live_bytes": 462274560, + "latency_s": 2.0216832919977605, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 77414400, + "prompt_kind": "short", + "t_relative_s": 12341.89757854212 + }, + { + "decode_latency_s": 0.0683806119923247, + "kv_live_bytes": 462274560, + "latency_s": 2.190392999909818, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 78036992, + "prompt_kind": "long", + "t_relative_s": 12346.90764695825 + }, + { + "decode_latency_s": 0.06587107031373307, + "kv_live_bytes": 462274560, + "latency_s": 2.1092764157801867, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 79429632, + "prompt_kind": "short", + "t_relative_s": 12351.917745000217 + }, + { + "decode_latency_s": 0.06332341537927277, + "kv_live_bytes": 462274560, + "latency_s": 2.0273245410062373, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73416704, + "prompt_kind": "long", + "t_relative_s": 12356.927866375074 + }, + { + "decode_latency_s": 0.0657087265572045, + "kv_live_bytes": 462274560, + "latency_s": 2.103538792114705, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66682880, + "prompt_kind": "short", + "t_relative_s": 12361.937920500059 + }, + { + "decode_latency_s": 0.06160232680849731, + "kv_live_bytes": 462274560, + "latency_s": 1.9723236253485084, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69435392, + "prompt_kind": "long", + "t_relative_s": 12366.948025042191 + }, + { + "decode_latency_s": 0.06055927996931132, + "kv_live_bytes": 462274560, + "latency_s": 1.9388695838861167, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68648960, + "prompt_kind": "short", + "t_relative_s": 12371.958112500142 + }, + { + "decode_latency_s": 0.06804214842850342, + "kv_live_bytes": 462274560, + "latency_s": 2.179209374822676, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71188480, + "prompt_kind": "long", + "t_relative_s": 12376.968189542182 + }, + { + "decode_latency_s": 0.06705392186995596, + "kv_live_bytes": 462274560, + "latency_s": 2.1474569169804454, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71303168, + "prompt_kind": "short", + "t_relative_s": 12381.978245417122 + }, + { + "decode_latency_s": 0.06847968359943479, + "kv_live_bytes": 462274560, + "latency_s": 2.194163749925792, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71368704, + "prompt_kind": "long", + "t_relative_s": 12386.988349042367 + }, + { + "decode_latency_s": 0.06120491924230009, + "kv_live_bytes": 462274560, + "latency_s": 1.959353500045836, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71843840, + "prompt_kind": "short", + "t_relative_s": 12391.998425000347 + }, + { + "decode_latency_s": 0.07000991146196611, + "kv_live_bytes": 462274560, + "latency_s": 2.2419300000183284, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70828032, + "prompt_kind": "long", + "t_relative_s": 12397.008472917136 + }, + { + "decode_latency_s": 0.060683378906105645, + "kv_live_bytes": 462274560, + "latency_s": 1.9427917920984328, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68009984, + "prompt_kind": "short", + "t_relative_s": 12402.018532750197 + }, + { + "decode_latency_s": 0.0675717968842946, + "kv_live_bytes": 462274560, + "latency_s": 2.1653196248225868, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69959680, + "prompt_kind": "long", + "t_relative_s": 12407.02859195834 + }, + { + "decode_latency_s": 0.060525971348397434, + "kv_live_bytes": 462274560, + "latency_s": 1.9378149593248963, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70025216, + "prompt_kind": "short", + "t_relative_s": 12412.038496000227 + }, + { + "decode_latency_s": 0.060492770848213695, + "kv_live_bytes": 462274560, + "latency_s": 1.9367706668563187, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71106560, + "prompt_kind": "long", + "t_relative_s": 12417.048552083317 + }, + { + "decode_latency_s": 0.060592832043766975, + "kv_live_bytes": 462274560, + "latency_s": 1.9397217920050025, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71761920, + "prompt_kind": "short", + "t_relative_s": 12422.058679167181 + }, + { + "decode_latency_s": 0.06818072784517426, + "kv_live_bytes": 462274560, + "latency_s": 2.1834432086907327, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71499776, + "prompt_kind": "long", + "t_relative_s": 12427.068765250035 + }, + { + "decode_latency_s": 0.06041372397157829, + "kv_live_bytes": 462274560, + "latency_s": 1.9341088752262294, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70975488, + "prompt_kind": "short", + "t_relative_s": 12432.078827542253 + }, + { + "decode_latency_s": 0.0678368072694866, + "kv_live_bytes": 462274560, + "latency_s": 2.1732739582657814, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71090176, + "prompt_kind": "long", + "t_relative_s": 12437.083540417254 + }, + { + "decode_latency_s": 0.060394764310331084, + "kv_live_bytes": 462274560, + "latency_s": 1.9334860830567777, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71106560, + "prompt_kind": "short", + "t_relative_s": 12442.09362262534 + }, + { + "decode_latency_s": 0.06801666534738615, + "kv_live_bytes": 462274560, + "latency_s": 2.1782588749192655, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71270400, + "prompt_kind": "long", + "t_relative_s": 12447.103744583204 + }, + { + "decode_latency_s": 0.06058164453133941, + "kv_live_bytes": 462274560, + "latency_s": 1.9395700409077108, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71860224, + "prompt_kind": "short", + "t_relative_s": 12452.113787250128 + }, + { + "decode_latency_s": 0.06791738409083337, + "kv_live_bytes": 462274560, + "latency_s": 2.175918749999255, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71909376, + "prompt_kind": "long", + "t_relative_s": 12457.123858999927 + }, + { + "decode_latency_s": 0.060788664064602926, + "kv_live_bytes": 462274560, + "latency_s": 1.9460438750684261, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74186752, + "prompt_kind": "short", + "t_relative_s": 12462.133949249983 + }, + { + "decode_latency_s": 0.06066256119811442, + "kv_live_bytes": 462274560, + "latency_s": 1.9425432910211384, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71761920, + "prompt_kind": "long", + "t_relative_s": 12467.14401533315 + }, + { + "decode_latency_s": 0.06042421094025485, + "kv_live_bytes": 462274560, + "latency_s": 1.9344650409184396, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71761920, + "prompt_kind": "short", + "t_relative_s": 12472.154091042 + }, + { + "decode_latency_s": 0.06802586068806704, + "kv_live_bytes": 462274560, + "latency_s": 2.1786280423402786, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72810496, + "prompt_kind": "long", + "t_relative_s": 12477.164187000133 + }, + { + "decode_latency_s": 0.060673162748571485, + "kv_live_bytes": 462274560, + "latency_s": 1.942486499901861, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73498624, + "prompt_kind": "short", + "t_relative_s": 12482.174243125133 + }, + { + "decode_latency_s": 0.06032848305767402, + "kv_live_bytes": 462274560, + "latency_s": 1.9315682090818882, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73465856, + "prompt_kind": "long", + "t_relative_s": 12487.18432179233 + }, + { + "decode_latency_s": 0.0664334804750979, + "kv_live_bytes": 462274560, + "latency_s": 2.127881250344217, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73580544, + "prompt_kind": "short", + "t_relative_s": 12492.1944007501 + }, + { + "decode_latency_s": 0.09556336195964832, + "kv_live_bytes": 462274560, + "latency_s": 3.0589225003495812, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69599232, + "prompt_kind": "long", + "t_relative_s": 12497.204446833115 + }, + { + "decode_latency_s": 0.06151049218897242, + "kv_live_bytes": 462274560, + "latency_s": 1.9691338748671114, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70352896, + "prompt_kind": "short", + "t_relative_s": 12504.418915000279 + }, + { + "decode_latency_s": 0.06050996744306758, + "kv_live_bytes": 462274560, + "latency_s": 1.9376341253519058, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70893568, + "prompt_kind": "long", + "t_relative_s": 12509.42896229215 + }, + { + "decode_latency_s": 0.06038380468089599, + "kv_live_bytes": 462274560, + "latency_s": 1.9331583338789642, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69222400, + "prompt_kind": "short", + "t_relative_s": 12514.439010708127 + }, + { + "decode_latency_s": 0.06855121352418792, + "kv_live_bytes": 462274560, + "latency_s": 2.195669916924089, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69304320, + "prompt_kind": "long", + "t_relative_s": 12519.449082750361 + }, + { + "decode_latency_s": 0.06662582680291962, + "kv_live_bytes": 462274560, + "latency_s": 2.133633750025183, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69402624, + "prompt_kind": "short", + "t_relative_s": 12524.459149500355 + }, + { + "decode_latency_s": 0.06058233724616002, + "kv_live_bytes": 462274560, + "latency_s": 1.9398237499408424, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69369856, + "prompt_kind": "long", + "t_relative_s": 12529.469205208123 + }, + { + "decode_latency_s": 0.06072704818507191, + "kv_live_bytes": 462274560, + "latency_s": 1.9440426668152213, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70074368, + "prompt_kind": "short", + "t_relative_s": 12534.479293291923 + }, + { + "decode_latency_s": 0.06798729037109297, + "kv_live_bytes": 462274560, + "latency_s": 2.1775752496905625, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69353472, + "prompt_kind": "long", + "t_relative_s": 12539.489339125343 + }, + { + "decode_latency_s": 0.06672069399792235, + "kv_live_bytes": 462274560, + "latency_s": 2.136847708374262, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69337088, + "prompt_kind": "short", + "t_relative_s": 12544.499414708 + }, + { + "decode_latency_s": 0.0678510937577812, + "kv_live_bytes": 462274560, + "latency_s": 2.173056958243251, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70025216, + "prompt_kind": "long", + "t_relative_s": 12549.50946279196 + }, + { + "decode_latency_s": 0.06034032293246128, + "kv_live_bytes": 462274560, + "latency_s": 1.9317574589513242, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70008832, + "prompt_kind": "short", + "t_relative_s": 12554.519584625028 + }, + { + "decode_latency_s": 0.06822050259506796, + "kv_live_bytes": 462274560, + "latency_s": 2.1849387916736305, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69877760, + "prompt_kind": "long", + "t_relative_s": 12559.52965804236 + }, + { + "decode_latency_s": 0.06678687890234869, + "kv_live_bytes": 462274560, + "latency_s": 2.138633292168379, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70565888, + "prompt_kind": "short", + "t_relative_s": 12564.539702500217 + }, + { + "decode_latency_s": 0.06767161327297799, + "kv_live_bytes": 462274560, + "latency_s": 2.167852624785155, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70598656, + "prompt_kind": "long", + "t_relative_s": 12569.549780750182 + }, + { + "decode_latency_s": 0.0668764817819465, + "kv_live_bytes": 462274560, + "latency_s": 2.141805708874017, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70664192, + "prompt_kind": "short", + "t_relative_s": 12574.559883708134 + }, + { + "decode_latency_s": 0.06756966015382204, + "kv_live_bytes": 462274560, + "latency_s": 2.1645514997653663, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70615040, + "prompt_kind": "long", + "t_relative_s": 12579.569934708066 + }, + { + "decode_latency_s": 0.06030339976132382, + "kv_live_bytes": 462274560, + "latency_s": 1.9306007497943938, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70729728, + "prompt_kind": "short", + "t_relative_s": 12584.575063792057 + }, + { + "decode_latency_s": 0.06637801302713342, + "kv_live_bytes": 462274560, + "latency_s": 2.125746500212699, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69910528, + "prompt_kind": "long", + "t_relative_s": 12589.585207541939 + }, + { + "decode_latency_s": 0.0610824492177926, + "kv_live_bytes": 462274560, + "latency_s": 1.9559603752568364, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69435392, + "prompt_kind": "short", + "t_relative_s": 12594.595241250005 + }, + { + "decode_latency_s": 0.06353088282048702, + "kv_live_bytes": 462274560, + "latency_s": 2.0346877919510007, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71729152, + "prompt_kind": "long", + "t_relative_s": 12599.605297124945 + }, + { + "decode_latency_s": 0.06686354297562502, + "kv_live_bytes": 462274560, + "latency_s": 2.141300041228533, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72351744, + "prompt_kind": "short", + "t_relative_s": 12604.615396792069 + }, + { + "decode_latency_s": 0.0677087434742134, + "kv_live_bytes": 462274560, + "latency_s": 2.1692739170975983, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72482816, + "prompt_kind": "long", + "t_relative_s": 12609.625439583324 + }, + { + "decode_latency_s": 0.06057444789621513, + "kv_live_bytes": 462274560, + "latency_s": 1.9392180829308927, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71385088, + "prompt_kind": "short", + "t_relative_s": 12614.635529292282 + }, + { + "decode_latency_s": 0.060836315125925466, + "kv_live_bytes": 462274560, + "latency_s": 1.948036041110754, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71843840, + "prompt_kind": "long", + "t_relative_s": 12619.64562162524 + }, + { + "decode_latency_s": 0.06043762889748905, + "kv_live_bytes": 462274560, + "latency_s": 1.9349272497929633, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72433664, + "prompt_kind": "short", + "t_relative_s": 12624.655661917292 + }, + { + "decode_latency_s": 0.06846043231780641, + "kv_live_bytes": 462274560, + "latency_s": 2.1926879161037505, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72499200, + "prompt_kind": "long", + "t_relative_s": 12629.665754917078 + }, + { + "decode_latency_s": 0.06686473306035623, + "kv_live_bytes": 462274560, + "latency_s": 2.141379290726036, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73039872, + "prompt_kind": "short", + "t_relative_s": 12634.673539374955 + }, + { + "decode_latency_s": 0.06786630468559451, + "kv_live_bytes": 462274560, + "latency_s": 2.1741765001788735, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72941568, + "prompt_kind": "long", + "t_relative_s": 12639.68359050015 + }, + { + "decode_latency_s": 0.06061113539908547, + "kv_live_bytes": 462274560, + "latency_s": 1.9403442088514566, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72663040, + "prompt_kind": "short", + "t_relative_s": 12644.693679625168 + }, + { + "decode_latency_s": 0.06751523046114016, + "kv_live_bytes": 462274560, + "latency_s": 2.1624902500770986, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72663040, + "prompt_kind": "long", + "t_relative_s": 12649.703735332936 + }, + { + "decode_latency_s": 0.060592563822865486, + "kv_live_bytes": 462274560, + "latency_s": 1.9398444998078048, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "short", + "t_relative_s": 12654.713799792342 + }, + { + "decode_latency_s": 0.06811096222372726, + "kv_live_bytes": 462274560, + "latency_s": 2.1820968752726912, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73072640, + "prompt_kind": "long", + "t_relative_s": 12659.723831708077 + }, + { + "decode_latency_s": 0.06052062238450162, + "kv_live_bytes": 462274560, + "latency_s": 1.9376163752749562, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71860224, + "prompt_kind": "short", + "t_relative_s": 12664.733951500151 + }, + { + "decode_latency_s": 0.060582822901778854, + "kv_live_bytes": 462274560, + "latency_s": 1.9400875843130052, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70467584, + "prompt_kind": "long", + "t_relative_s": 12669.744026625063 + }, + { + "decode_latency_s": 0.06090637498709839, + "kv_live_bytes": 462274560, + "latency_s": 1.9498642077669501, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70189056, + "prompt_kind": "short", + "t_relative_s": 12674.754124208353 + }, + { + "decode_latency_s": 0.06775191925407853, + "kv_live_bytes": 462274560, + "latency_s": 2.170211249962449, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71090176, + "prompt_kind": "long", + "t_relative_s": 12679.764173833188 + }, + { + "decode_latency_s": 0.06037860414653551, + "kv_live_bytes": 462274560, + "latency_s": 1.93302454194054, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71073792, + "prompt_kind": "short", + "t_relative_s": 12684.774250833318 + }, + { + "decode_latency_s": 0.06767369659792166, + "kv_live_bytes": 462274560, + "latency_s": 2.1673330417834222, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71761920, + "prompt_kind": "long", + "t_relative_s": 12689.784335208125 + }, + { + "decode_latency_s": 0.06666170702374075, + "kv_live_bytes": 462274560, + "latency_s": 2.1347525003366172, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72335360, + "prompt_kind": "short", + "t_relative_s": 12694.79438512493 + }, + { + "decode_latency_s": 0.06802615756168962, + "kv_live_bytes": 462274560, + "latency_s": 2.17944779060781, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72400896, + "prompt_kind": "long", + "t_relative_s": 12699.80446608318 + }, + { + "decode_latency_s": 0.060697029941366054, + "kv_live_bytes": 462274560, + "latency_s": 1.9431094168685377, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71860224, + "prompt_kind": "short", + "t_relative_s": 12704.814550833311 + }, + { + "decode_latency_s": 0.06093204428907484, + "kv_live_bytes": 462274560, + "latency_s": 1.950738999992609, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70828032, + "prompt_kind": "long", + "t_relative_s": 12709.824602667242 + }, + { + "decode_latency_s": 0.06091399482102133, + "kv_live_bytes": 462274560, + "latency_s": 1.9500075839459896, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71368704, + "prompt_kind": "short", + "t_relative_s": 12714.83465562528 + }, + { + "decode_latency_s": 0.060591736968490295, + "kv_live_bytes": 462274560, + "latency_s": 1.9401217079721391, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71483392, + "prompt_kind": "long", + "t_relative_s": 12719.844726958312 + }, + { + "decode_latency_s": 0.06051379686687142, + "kv_live_bytes": 462274560, + "latency_s": 1.9372537923045456, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72007680, + "prompt_kind": "short", + "t_relative_s": 12724.85480708303 + }, + { + "decode_latency_s": 0.06778858724283054, + "kv_live_bytes": 462274560, + "latency_s": 2.171304459217936, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71974912, + "prompt_kind": "long", + "t_relative_s": 12729.864857167006 + }, + { + "decode_latency_s": 0.06421019011759199, + "kv_live_bytes": 462274560, + "latency_s": 2.055574500001967, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 63848448, + "prompt_kind": "short", + "t_relative_s": 12734.874945749994 + }, + { + "decode_latency_s": 0.06737270051962696, + "kv_live_bytes": 462274560, + "latency_s": 2.1585252499207854, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 86016000, + "prompt_kind": "long", + "t_relative_s": 12739.88499129191 + }, + { + "decode_latency_s": 0.0605725755012827, + "kv_live_bytes": 462274560, + "latency_s": 1.9391442500054836, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 86654976, + "prompt_kind": "short", + "t_relative_s": 12744.895076707937 + }, + { + "decode_latency_s": 0.06901502342952881, + "kv_live_bytes": 462274560, + "latency_s": 2.2103301668539643, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 85835776, + "prompt_kind": "long", + "t_relative_s": 12749.905130667146 + }, + { + "decode_latency_s": 0.06048633070895448, + "kv_live_bytes": 462274560, + "latency_s": 1.9364865408279002, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 86605824, + "prompt_kind": "short", + "t_relative_s": 12754.915211291984 + }, + { + "decode_latency_s": 0.06847165364888497, + "kv_live_bytes": 462274560, + "latency_s": 2.193290624767542, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 87048192, + "prompt_kind": "long", + "t_relative_s": 12759.925298375078 + }, + { + "decode_latency_s": 0.06702564972511027, + "kv_live_bytes": 462274560, + "latency_s": 2.1464442922733724, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 87097344, + "prompt_kind": "short", + "t_relative_s": 12764.935369832907 + }, + { + "decode_latency_s": 0.06781489193963353, + "kv_live_bytes": 462274560, + "latency_s": 2.172777791041881, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 87097344, + "prompt_kind": "long", + "t_relative_s": 12769.945435666945 + }, + { + "decode_latency_s": 0.0606259531195974, + "kv_live_bytes": 462274560, + "latency_s": 1.9408390000462532, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 87785472, + "prompt_kind": "short", + "t_relative_s": 12774.954695458058 + }, + { + "decode_latency_s": 0.06799674087960739, + "kv_live_bytes": 462274560, + "latency_s": 2.1779312500730157, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 87228416, + "prompt_kind": "long", + "t_relative_s": 12779.96474645799 + }, + { + "decode_latency_s": 0.060591152345295995, + "kv_live_bytes": 462274560, + "latency_s": 1.9396640420891345, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 87212032, + "prompt_kind": "short", + "t_relative_s": 12784.974843667354 + }, + { + "decode_latency_s": 0.06748383202648256, + "kv_live_bytes": 462274560, + "latency_s": 2.1619081660173833, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 87769088, + "prompt_kind": "long", + "t_relative_s": 12789.984901708085 + }, + { + "decode_latency_s": 0.0603771432506619, + "kv_live_bytes": 462274560, + "latency_s": 1.9329254999756813, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 87670784, + "prompt_kind": "short", + "t_relative_s": 12794.994977457915 + }, + { + "decode_latency_s": 0.06795380597759504, + "kv_live_bytes": 462274560, + "latency_s": 2.1761570000089705, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 87687168, + "prompt_kind": "long", + "t_relative_s": 12800.005065958016 + }, + { + "decode_latency_s": 0.060665027340292, + "kv_live_bytes": 462274560, + "latency_s": 1.9422607501037419, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88309760, + "prompt_kind": "short", + "t_relative_s": 12805.015132083092 + }, + { + "decode_latency_s": 0.0687276614626171, + "kv_live_bytes": 462274560, + "latency_s": 2.2021284578368068, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88375296, + "prompt_kind": "long", + "t_relative_s": 12810.025213791989 + }, + { + "decode_latency_s": 0.060559067729627714, + "kv_live_bytes": 462274560, + "latency_s": 1.9387327921576798, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88424448, + "prompt_kind": "short", + "t_relative_s": 12815.03530554194 + }, + { + "decode_latency_s": 0.0678450130217243, + "kv_live_bytes": 462274560, + "latency_s": 2.1730440841056406, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 88391680, + "prompt_kind": "long", + "t_relative_s": 12820.045355082955 + }, + { + "decode_latency_s": 0.06045430731319357, + "kv_live_bytes": 462274560, + "latency_s": 1.9354394171386957, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 87113728, + "prompt_kind": "short", + "t_relative_s": 12825.055429417174 + }, + { + "decode_latency_s": 0.06787135025660973, + "kv_live_bytes": 462274560, + "latency_s": 2.173744291998446, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72712192, + "prompt_kind": "long", + "t_relative_s": 12830.06550737517 + }, + { + "decode_latency_s": 0.06049020444334019, + "kv_live_bytes": 462274560, + "latency_s": 1.9365677502937615, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72892416, + "prompt_kind": "short", + "t_relative_s": 12835.075549917296 + }, + { + "decode_latency_s": 0.06077203124004882, + "kv_live_bytes": 462274560, + "latency_s": 1.9458558750338852, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72925184, + "prompt_kind": "long", + "t_relative_s": 12840.08562262496 + }, + { + "decode_latency_s": 0.060522227868204936, + "kv_live_bytes": 462274560, + "latency_s": 1.9374513337388635, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "short", + "t_relative_s": 12845.09570762515 + }, + { + "decode_latency_s": 0.06716876302380115, + "kv_live_bytes": 462274560, + "latency_s": 2.1517697921954095, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73564160, + "prompt_kind": "long", + "t_relative_s": 12850.10576274991 + }, + { + "decode_latency_s": 0.060330545558827, + "kv_live_bytes": 462274560, + "latency_s": 1.931458625011146, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73056256, + "prompt_kind": "short", + "t_relative_s": 12855.115825625136 + }, + { + "decode_latency_s": 0.06773342836822849, + "kv_live_bytes": 462274560, + "latency_s": 2.169765332713723, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71090176, + "prompt_kind": "long", + "t_relative_s": 12860.125913457945 + }, + { + "decode_latency_s": 0.06033552606822923, + "kv_live_bytes": 462274560, + "latency_s": 1.9316654172725976, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72859648, + "prompt_kind": "short", + "t_relative_s": 12865.13596837502 + }, + { + "decode_latency_s": 0.07807435546419583, + "kv_live_bytes": 462274560, + "latency_s": 2.4998329584486783, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 161841152, + "prompt_kind": "long", + "t_relative_s": 12870.14604908321 + }, + { + "decode_latency_s": 0.06055687631305773, + "kv_live_bytes": 462274560, + "latency_s": 1.938626834191382, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 162496512, + "prompt_kind": "short", + "t_relative_s": 12875.156119625084 + }, + { + "decode_latency_s": 0.06792177734314464, + "kv_live_bytes": 462274560, + "latency_s": 2.175840583164245, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 161988608, + "prompt_kind": "long", + "t_relative_s": 12880.166165833361 + }, + { + "decode_latency_s": 0.06034234375692904, + "kv_live_bytes": 462274560, + "latency_s": 1.931848666165024, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 135905280, + "prompt_kind": "short", + "t_relative_s": 12885.176240125205 + }, + { + "decode_latency_s": 0.06427817710209638, + "kv_live_bytes": 462274560, + "latency_s": 2.0589282917790115, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72073216, + "prompt_kind": "long", + "t_relative_s": 12890.186340500135 + }, + { + "decode_latency_s": 0.06060832811635919, + "kv_live_bytes": 462274560, + "latency_s": 1.9402456250973046, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72826880, + "prompt_kind": "short", + "t_relative_s": 12895.196409583092 + }, + { + "decode_latency_s": 0.06774494658748154, + "kv_live_bytes": 462274560, + "latency_s": 2.1706103747710586, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72826880, + "prompt_kind": "long", + "t_relative_s": 12900.206475958228 + }, + { + "decode_latency_s": 0.060474122408777475, + "kv_live_bytes": 462274560, + "latency_s": 1.9360447502695024, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72859648, + "prompt_kind": "short", + "t_relative_s": 12905.216559207998 + }, + { + "decode_latency_s": 0.06763907552522141, + "kv_live_bytes": 462274560, + "latency_s": 2.166478833183646, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73367552, + "prompt_kind": "long", + "t_relative_s": 12910.226625167299 + }, + { + "decode_latency_s": 0.06063489321968518, + "kv_live_bytes": 462274560, + "latency_s": 1.9412282924167812, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72712192, + "prompt_kind": "short", + "t_relative_s": 12915.236675917171 + }, + { + "decode_latency_s": 0.06043409115227405, + "kv_live_bytes": 462274560, + "latency_s": 1.9348486671224236, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "long", + "t_relative_s": 12920.246725500096 + }, + { + "decode_latency_s": 0.060302852842141874, + "kv_live_bytes": 462274560, + "latency_s": 1.9305790001526475, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73547776, + "prompt_kind": "short", + "t_relative_s": 12925.256813958287 + }, + { + "decode_latency_s": 0.06827020439959597, + "kv_live_bytes": 462274560, + "latency_s": 2.1865135827101767, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72908800, + "prompt_kind": "long", + "t_relative_s": 12930.266893499997 + }, + { + "decode_latency_s": 0.060473492194432765, + "kv_live_bytes": 462274560, + "latency_s": 1.9360456657595932, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73564160, + "prompt_kind": "short", + "t_relative_s": 12935.276966625359 + }, + { + "decode_latency_s": 0.06788657550350763, + "kv_live_bytes": 462274560, + "latency_s": 2.174618124961853, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "long", + "t_relative_s": 12940.28703270806 + }, + { + "decode_latency_s": 0.06708126043668017, + "kv_live_bytes": 462274560, + "latency_s": 2.1485397503711283, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72105984, + "prompt_kind": "short", + "t_relative_s": 12945.288252166938 + }, + { + "decode_latency_s": 0.06062224609195255, + "kv_live_bytes": 462274560, + "latency_s": 1.9408700419589877, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73203712, + "prompt_kind": "long", + "t_relative_s": 12950.29830520833 + }, + { + "decode_latency_s": 0.06273550259356853, + "kv_live_bytes": 462274560, + "latency_s": 2.0090939588844776, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72695808, + "prompt_kind": "short", + "t_relative_s": 12955.305431208108 + }, + { + "decode_latency_s": 0.06843810025020503, + "kv_live_bytes": 462274560, + "latency_s": 2.1916692908853292, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72531968, + "prompt_kind": "long", + "t_relative_s": 12960.315479082987 + }, + { + "decode_latency_s": 0.060388678379240446, + "kv_live_bytes": 462274560, + "latency_s": 1.93334695789963, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73154560, + "prompt_kind": "short", + "t_relative_s": 12965.325531250332 + }, + { + "decode_latency_s": 0.06719903384509962, + "kv_live_bytes": 462274560, + "latency_s": 2.152428791858256, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73744384, + "prompt_kind": "long", + "t_relative_s": 12970.335598333273 + }, + { + "decode_latency_s": 0.060804385429946706, + "kv_live_bytes": 462274560, + "latency_s": 1.9465377503074706, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73842688, + "prompt_kind": "short", + "t_relative_s": 12975.345636500046 + }, + { + "decode_latency_s": 0.06879642838612199, + "kv_live_bytes": 462274560, + "latency_s": 2.203975958749652, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72712192, + "prompt_kind": "long", + "t_relative_s": 12980.355833792128 + }, + { + "decode_latency_s": 0.06070422787161078, + "kv_live_bytes": 462274560, + "latency_s": 1.9433449157513678, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72761344, + "prompt_kind": "short", + "t_relative_s": 12985.365904417355 + }, + { + "decode_latency_s": 0.06829987891251221, + "kv_live_bytes": 462274560, + "latency_s": 2.1876917500048876, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72761344, + "prompt_kind": "long", + "t_relative_s": 12990.373242083006 + }, + { + "decode_latency_s": 0.06068709504324943, + "kv_live_bytes": 462274560, + "latency_s": 1.9429439157247543, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72876032, + "prompt_kind": "short", + "t_relative_s": 12995.383325292263 + }, + { + "decode_latency_s": 0.06744751821679529, + "kv_live_bytes": 462274560, + "latency_s": 2.1603085827082396, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72335360, + "prompt_kind": "long", + "t_relative_s": 13000.39338662522 + }, + { + "decode_latency_s": 0.07752461846393999, + "kv_live_bytes": 462274560, + "latency_s": 2.4816520418971777, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 369082368, + "prompt_kind": "short", + "t_relative_s": 13005.403439000249 + }, + { + "decode_latency_s": 0.06837092056230176, + "kv_live_bytes": 462274560, + "latency_s": 2.1899695000611246, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 369754112, + "prompt_kind": "long", + "t_relative_s": 13010.413508875296 + }, + { + "decode_latency_s": 0.06700592706329189, + "kv_live_bytes": 462274560, + "latency_s": 2.145571666304022, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 370638848, + "prompt_kind": "short", + "t_relative_s": 13015.423576083034 + }, + { + "decode_latency_s": 0.06794668622023892, + "kv_live_bytes": 462274560, + "latency_s": 2.1764519168064, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 370671616, + "prompt_kind": "long", + "t_relative_s": 13020.43364199996 + }, + { + "decode_latency_s": 0.060884291684487835, + "kv_live_bytes": 462274560, + "latency_s": 1.9491459587588906, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 371245056, + "prompt_kind": "short", + "t_relative_s": 13025.443705458194 + }, + { + "decode_latency_s": 0.06771327213209588, + "kv_live_bytes": 462274560, + "latency_s": 2.169195707887411, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 370393088, + "prompt_kind": "long", + "t_relative_s": 13030.453761416953 + }, + { + "decode_latency_s": 0.06112135809962638, + "kv_live_bytes": 462274560, + "latency_s": 1.9569397079758346, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 133791744, + "prompt_kind": "short", + "t_relative_s": 13035.457929375116 + }, + { + "decode_latency_s": 0.06770039063121658, + "kv_live_bytes": 462274560, + "latency_s": 2.168245708104223, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 137527296, + "prompt_kind": "long", + "t_relative_s": 13040.468009083066 + }, + { + "decode_latency_s": 0.060468506504548714, + "kv_live_bytes": 462274560, + "latency_s": 1.9358442081138492, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 137592832, + "prompt_kind": "short", + "t_relative_s": 13045.478080042172 + }, + { + "decode_latency_s": 0.06781785808561835, + "kv_live_bytes": 462274560, + "latency_s": 2.1727734999731183, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 137625600, + "prompt_kind": "long", + "t_relative_s": 13050.48816516716 + }, + { + "decode_latency_s": 0.0607548385305563, + "kv_live_bytes": 462274560, + "latency_s": 1.9449351248331368, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 138199040, + "prompt_kind": "short", + "t_relative_s": 13055.498260917142 + }, + { + "decode_latency_s": 0.06845624087145552, + "kv_live_bytes": 462274560, + "latency_s": 2.1930207079276443, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75546624, + "prompt_kind": "long", + "t_relative_s": 13060.508312625345 + }, + { + "decode_latency_s": 0.06032545049674809, + "kv_live_bytes": 462274560, + "latency_s": 1.9313397924415767, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73564160, + "prompt_kind": "short", + "t_relative_s": 13065.518375874963 + }, + { + "decode_latency_s": 0.07024892450135667, + "kv_live_bytes": 462274560, + "latency_s": 2.2503557498566806, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71303168, + "prompt_kind": "long", + "t_relative_s": 13070.528467332944 + }, + { + "decode_latency_s": 0.06459480337798595, + "kv_live_bytes": 462274560, + "latency_s": 2.0686736246570945, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67502080, + "prompt_kind": "short", + "t_relative_s": 13075.538553458173 + }, + { + "decode_latency_s": 0.0686629895499209, + "kv_live_bytes": 462274560, + "latency_s": 2.1995620410889387, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71630848, + "prompt_kind": "long", + "t_relative_s": 13080.548636124935 + }, + { + "decode_latency_s": 0.06106135545996949, + "kv_live_bytes": 462274560, + "latency_s": 1.9548945422284305, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72499200, + "prompt_kind": "short", + "t_relative_s": 13085.55871587526 + }, + { + "decode_latency_s": 0.0678782747418154, + "kv_live_bytes": 462274560, + "latency_s": 2.17377462470904, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73711616, + "prompt_kind": "long", + "t_relative_s": 13090.568826333154 + }, + { + "decode_latency_s": 0.060610473970882595, + "kv_live_bytes": 462274560, + "latency_s": 1.9403874170966446, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72515584, + "prompt_kind": "short", + "t_relative_s": 13095.578881542198 + }, + { + "decode_latency_s": 0.06039452212280594, + "kv_live_bytes": 462274560, + "latency_s": 1.9336641668342054, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73220096, + "prompt_kind": "long", + "t_relative_s": 13100.588940167334 + }, + { + "decode_latency_s": 0.06027887237723917, + "kv_live_bytes": 462274560, + "latency_s": 1.9298136252909899, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71696384, + "prompt_kind": "short", + "t_relative_s": 13105.599014250096 + }, + { + "decode_latency_s": 0.06817038281587884, + "kv_live_bytes": 462274560, + "latency_s": 2.1831988748162985, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73416704, + "prompt_kind": "long", + "t_relative_s": 13110.609135458246 + }, + { + "decode_latency_s": 0.060709695317200385, + "kv_live_bytes": 462274560, + "latency_s": 1.9436309589073062, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74055680, + "prompt_kind": "short", + "t_relative_s": 13115.619192375336 + }, + { + "decode_latency_s": 0.06834313541185111, + "kv_live_bytes": 462274560, + "latency_s": 2.1894141668453813, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74088448, + "prompt_kind": "long", + "t_relative_s": 13120.629282125272 + }, + { + "decode_latency_s": 0.060960315095144324, + "kv_live_bytes": 462274560, + "latency_s": 1.951992165762931, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69353472, + "prompt_kind": "short", + "t_relative_s": 13125.639371875208 + }, + { + "decode_latency_s": 0.06816426433215383, + "kv_live_bytes": 462274560, + "latency_s": 2.1839180001989007, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72531968, + "prompt_kind": "long", + "t_relative_s": 13130.649413333274 + }, + { + "decode_latency_s": 0.06060037630959414, + "kv_live_bytes": 462274560, + "latency_s": 1.9404917089268565, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73170944, + "prompt_kind": "short", + "t_relative_s": 13135.650595291983 + }, + { + "decode_latency_s": 0.06838927212811541, + "kv_live_bytes": 462274560, + "latency_s": 2.189988249912858, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72204288, + "prompt_kind": "long", + "t_relative_s": 13140.660665583331 + }, + { + "decode_latency_s": 0.06062749218835961, + "kv_live_bytes": 462274560, + "latency_s": 1.9410836668685079, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72876032, + "prompt_kind": "short", + "t_relative_s": 13145.670746208169 + }, + { + "decode_latency_s": 0.06508869530807715, + "kv_live_bytes": 462274560, + "latency_s": 2.083789417054504, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70205440, + "prompt_kind": "long", + "t_relative_s": 13150.680844082963 + }, + { + "decode_latency_s": 0.06313315361330751, + "kv_live_bytes": 462274560, + "latency_s": 2.021301709115505, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67993600, + "prompt_kind": "short", + "t_relative_s": 13155.690906083211 + }, + { + "decode_latency_s": 0.0808047630416695, + "kv_live_bytes": 462274560, + "latency_s": 2.5873335828073323, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68304896, + "prompt_kind": "long", + "t_relative_s": 13160.70099579217 + }, + { + "decode_latency_s": 0.060509528644615784, + "kv_live_bytes": 462274560, + "latency_s": 1.9370786249637604, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68452352, + "prompt_kind": "short", + "t_relative_s": 13166.976198417135 + }, + { + "decode_latency_s": 0.06839648699678946, + "kv_live_bytes": 462274560, + "latency_s": 2.191176749765873, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69812224, + "prompt_kind": "long", + "t_relative_s": 13171.986247667111 + }, + { + "decode_latency_s": 0.06047572265379131, + "kv_live_bytes": 462274560, + "latency_s": 1.9361036247573793, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70205440, + "prompt_kind": "short", + "t_relative_s": 13176.996327083092 + }, + { + "decode_latency_s": 0.06893590102845337, + "kv_live_bytes": 462274560, + "latency_s": 2.2080909996293485, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 64700416, + "prompt_kind": "long", + "t_relative_s": 13182.006427125074 + }, + { + "decode_latency_s": 0.061041484368615784, + "kv_live_bytes": 462274560, + "latency_s": 1.9542019157670438, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66158592, + "prompt_kind": "short", + "t_relative_s": 13187.016527249943 + }, + { + "decode_latency_s": 0.06834437632642221, + "kv_live_bytes": 462274560, + "latency_s": 2.188585708849132, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68616192, + "prompt_kind": "long", + "t_relative_s": 13192.026584208012 + }, + { + "decode_latency_s": 0.06034713018743787, + "kv_live_bytes": 462274560, + "latency_s": 1.932048958260566, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69779456, + "prompt_kind": "short", + "t_relative_s": 13197.03665400017 + }, + { + "decode_latency_s": 0.06367287760076579, + "kv_live_bytes": 462274560, + "latency_s": 2.0392832499928772, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70631424, + "prompt_kind": "long", + "t_relative_s": 13202.046711042058 + }, + { + "decode_latency_s": 0.0608373854047386, + "kv_live_bytes": 462274560, + "latency_s": 1.947643875144422, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70713344, + "prompt_kind": "short", + "t_relative_s": 13207.0567871253 + }, + { + "decode_latency_s": 0.06868080858839676, + "kv_live_bytes": 462274560, + "latency_s": 2.1998633327893913, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70369280, + "prompt_kind": "long", + "t_relative_s": 13212.066871250048 + }, + { + "decode_latency_s": 0.06727582159510348, + "kv_live_bytes": 462274560, + "latency_s": 2.154875624924898, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69533696, + "prompt_kind": "short", + "t_relative_s": 13217.076924417168 + }, + { + "decode_latency_s": 0.06830955078476109, + "kv_live_bytes": 462274560, + "latency_s": 2.1881591249257326, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70746112, + "prompt_kind": "long", + "t_relative_s": 13222.086976458319 + }, + { + "decode_latency_s": 0.061402053412166424, + "kv_live_bytes": 462274560, + "latency_s": 1.9657944589853287, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70139904, + "prompt_kind": "short", + "t_relative_s": 13227.097065958194 + }, + { + "decode_latency_s": 0.06876401694898959, + "kv_live_bytes": 462274560, + "latency_s": 2.202897499781102, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 60964864, + "prompt_kind": "long", + "t_relative_s": 13232.10717533296 + }, + { + "decode_latency_s": 0.06150485416583251, + "kv_live_bytes": 462274560, + "latency_s": 1.9690705831162632, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67305472, + "prompt_kind": "short", + "t_relative_s": 13237.117252292112 + }, + { + "decode_latency_s": 0.06340702863235492, + "kv_live_bytes": 462274560, + "latency_s": 2.0302725001238286, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67420160, + "prompt_kind": "long", + "t_relative_s": 13242.121431500185 + }, + { + "decode_latency_s": 0.061551479157060385, + "kv_live_bytes": 462274560, + "latency_s": 1.9706897079013288, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 65667072, + "prompt_kind": "short", + "t_relative_s": 13247.131489207968 + }, + { + "decode_latency_s": 0.15072389971464872, + "kv_live_bytes": 462274560, + "latency_s": 4.933303042314947, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66453504, + "prompt_kind": "long", + "t_relative_s": 13252.141570042353 + }, + { + "decode_latency_s": 0.060514816403156146, + "kv_live_bytes": 462274560, + "latency_s": 1.9372880421578884, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67485696, + "prompt_kind": "short", + "t_relative_s": 13262.399750542361 + }, + { + "decode_latency_s": 0.0680208580888575, + "kv_live_bytes": 462274560, + "latency_s": 2.1795350410975516, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68616192, + "prompt_kind": "long", + "t_relative_s": 13267.409806250129 + }, + { + "decode_latency_s": 0.06043172006320674, + "kv_live_bytes": 462274560, + "latency_s": 1.934680209029466, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69206016, + "prompt_kind": "short", + "t_relative_s": 13272.419869417325 + }, + { + "decode_latency_s": 0.06840775390446652, + "kv_live_bytes": 462274560, + "latency_s": 2.1911447909660637, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69320704, + "prompt_kind": "long", + "t_relative_s": 13277.429934666958 + }, + { + "decode_latency_s": 0.06091985806415323, + "kv_live_bytes": 462274560, + "latency_s": 1.9502418329939246, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69435392, + "prompt_kind": "short", + "t_relative_s": 13282.439996832982 + }, + { + "decode_latency_s": 0.06831155989493709, + "kv_live_bytes": 462274560, + "latency_s": 2.188370666000992, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 68911104, + "prompt_kind": "long", + "t_relative_s": 13287.450106667355 + }, + { + "decode_latency_s": 0.060635338537395, + "kv_live_bytes": 462274560, + "latency_s": 1.9411270003765821, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69468160, + "prompt_kind": "short", + "t_relative_s": 13292.46020400012 + }, + { + "decode_latency_s": 0.06051694921916351, + "kv_live_bytes": 462274560, + "latency_s": 1.9376977919600904, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69566464, + "prompt_kind": "long", + "t_relative_s": 13297.470258875284 + }, + { + "decode_latency_s": 0.06070329947397113, + "kv_live_bytes": 462274560, + "latency_s": 1.9437972498126328, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70189056, + "prompt_kind": "short", + "t_relative_s": 13302.475327958353 + }, + { + "decode_latency_s": 0.06787632685154676, + "kv_live_bytes": 462274560, + "latency_s": 2.1745457500219345, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70320128, + "prompt_kind": "long", + "t_relative_s": 13307.485409750137 + }, + { + "decode_latency_s": 0.06694727734429762, + "kv_live_bytes": 462274560, + "latency_s": 2.1448355419561267, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70549504, + "prompt_kind": "short", + "t_relative_s": 13312.495461833198 + }, + { + "decode_latency_s": 0.06739726039813831, + "kv_live_bytes": 462274560, + "latency_s": 2.1594184590503573, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70615040, + "prompt_kind": "long", + "t_relative_s": 13317.505561542232 + }, + { + "decode_latency_s": 0.060846638021757826, + "kv_live_bytes": 462274560, + "latency_s": 1.9486625422723591, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71188480, + "prompt_kind": "short", + "t_relative_s": 13322.515645000152 + }, + { + "decode_latency_s": 0.06807456380920485, + "kv_live_bytes": 462274560, + "latency_s": 2.181027582846582, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70631424, + "prompt_kind": "long", + "t_relative_s": 13327.525696583092 + }, + { + "decode_latency_s": 0.06031286455981899, + "kv_live_bytes": 462274560, + "latency_s": 1.9309257920831442, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71188480, + "prompt_kind": "short", + "t_relative_s": 13332.53575362498 + }, + { + "decode_latency_s": 0.06831378905917518, + "kv_live_bytes": 462274560, + "latency_s": 2.1883934582583606, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71303168, + "prompt_kind": "long", + "t_relative_s": 13337.545845542103 + }, + { + "decode_latency_s": 0.06072850522468798, + "kv_live_bytes": 462274560, + "latency_s": 1.944171333219856, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71335936, + "prompt_kind": "short", + "t_relative_s": 13342.555904916953 + }, + { + "decode_latency_s": 0.06855861586518586, + "kv_live_bytes": 462274560, + "latency_s": 2.1963787502609193, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71385088, + "prompt_kind": "long", + "t_relative_s": 13347.565960583277 + }, + { + "decode_latency_s": 0.060570610701688565, + "kv_live_bytes": 462274560, + "latency_s": 1.939016207586974, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71925760, + "prompt_kind": "short", + "t_relative_s": 13352.576037333347 + }, + { + "decode_latency_s": 0.06884485547197983, + "kv_live_bytes": 462274560, + "latency_s": 2.2051807092502713, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69894144, + "prompt_kind": "long", + "t_relative_s": 13357.586091875099 + }, + { + "decode_latency_s": 0.0605033775937045, + "kv_live_bytes": 462274560, + "latency_s": 1.9370726249180734, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70483968, + "prompt_kind": "short", + "t_relative_s": 13362.596165542025 + }, + { + "decode_latency_s": 0.06834649867960252, + "kv_live_bytes": 462274560, + "latency_s": 2.1888320832513273, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71286784, + "prompt_kind": "long", + "t_relative_s": 13367.606236916967 + }, + { + "decode_latency_s": 0.06061874871375039, + "kv_live_bytes": 462274560, + "latency_s": 1.9405922079458833, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71319552, + "prompt_kind": "short", + "t_relative_s": 13372.612152083311 + }, + { + "decode_latency_s": 0.06788903252163436, + "kv_live_bytes": 462274560, + "latency_s": 2.174688957631588, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71467008, + "prompt_kind": "long", + "t_relative_s": 13377.622206000146 + }, + { + "decode_latency_s": 0.06059355598699767, + "kv_live_bytes": 462274560, + "latency_s": 1.9398132497444749, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72024064, + "prompt_kind": "short", + "t_relative_s": 13382.63228479214 + }, + { + "decode_latency_s": 0.06821729037619662, + "kv_live_bytes": 462274560, + "latency_s": 2.1849212497472763, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71942144, + "prompt_kind": "long", + "t_relative_s": 13387.642344791908 + }, + { + "decode_latency_s": 0.060535033873748034, + "kv_live_bytes": 462274560, + "latency_s": 1.9380284999497235, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71237632, + "prompt_kind": "short", + "t_relative_s": 13392.652410792187 + }, + { + "decode_latency_s": 0.06818375518196262, + "kv_live_bytes": 462274560, + "latency_s": 2.1835214579477906, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71909376, + "prompt_kind": "long", + "t_relative_s": 13397.662498916965 + }, + { + "decode_latency_s": 0.060515139310155064, + "kv_live_bytes": 462274560, + "latency_s": 1.9373204591684043, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71974912, + "prompt_kind": "short", + "t_relative_s": 13402.672567958012 + }, + { + "decode_latency_s": 0.06812706771597732, + "kv_live_bytes": 462274560, + "latency_s": 2.1824351670220494, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72122368, + "prompt_kind": "long", + "t_relative_s": 13407.682661625091 + }, + { + "decode_latency_s": 0.060455350248957984, + "kv_live_bytes": 462274560, + "latency_s": 1.9353781249374151, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72630272, + "prompt_kind": "short", + "t_relative_s": 13412.6927424171 + }, + { + "decode_latency_s": 0.06807487628248055, + "kv_live_bytes": 462274560, + "latency_s": 2.1802332089282572, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72744960, + "prompt_kind": "long", + "t_relative_s": 13417.702794750221 + }, + { + "decode_latency_s": 0.06072395050432533, + "kv_live_bytes": 462274560, + "latency_s": 1.944072250276804, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72220672, + "prompt_kind": "short", + "t_relative_s": 13422.712847582996 + }, + { + "decode_latency_s": 0.06885069921554532, + "kv_live_bytes": 462274560, + "latency_s": 2.206698624882847, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72810496, + "prompt_kind": "long", + "t_relative_s": 13427.722929083277 + }, + { + "decode_latency_s": 0.06666032294742763, + "kv_live_bytes": 462274560, + "latency_s": 2.135297750122845, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72237056, + "prompt_kind": "short", + "t_relative_s": 13432.732974792365 + }, + { + "decode_latency_s": 0.06809702735336032, + "kv_live_bytes": 462274560, + "latency_s": 2.1817728341557086, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72269824, + "prompt_kind": "long", + "t_relative_s": 13437.743032750208 + }, + { + "decode_latency_s": 0.06418879037664738, + "kv_live_bytes": 462274560, + "latency_s": 2.055903166066855, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72712192, + "prompt_kind": "short", + "t_relative_s": 13442.753135167062 + }, + { + "decode_latency_s": 0.0679862304677954, + "kv_live_bytes": 462274560, + "latency_s": 2.1776399998925626, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72761344, + "prompt_kind": "long", + "t_relative_s": 13447.763169958256 + }, + { + "decode_latency_s": 0.060286947918939404, + "kv_live_bytes": 462274560, + "latency_s": 1.9301109169609845, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73089024, + "prompt_kind": "short", + "t_relative_s": 13452.773211667314 + }, + { + "decode_latency_s": 0.06767002865672112, + "kv_live_bytes": 462274560, + "latency_s": 2.1679591671563685, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73154560, + "prompt_kind": "long", + "t_relative_s": 13457.783291208092 + }, + { + "decode_latency_s": 0.06078774741035886, + "kv_live_bytes": 462274560, + "latency_s": 1.946037500165403, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73203712, + "prompt_kind": "short", + "t_relative_s": 13462.793361958116 + }, + { + "decode_latency_s": 0.06777730341127608, + "kv_live_bytes": 462274560, + "latency_s": 2.171828791964799, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73121792, + "prompt_kind": "long", + "t_relative_s": 13467.80340812495 + }, + { + "decode_latency_s": 0.06064391665859148, + "kv_live_bytes": 462274560, + "latency_s": 1.9414180838502944, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73760768, + "prompt_kind": "short", + "t_relative_s": 13472.813473875169 + }, + { + "decode_latency_s": 0.06774003646569327, + "kv_live_bytes": 462274560, + "latency_s": 2.169948250055313, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73252864, + "prompt_kind": "long", + "t_relative_s": 13477.823513166979 + }, + { + "decode_latency_s": 0.06444701952568721, + "kv_live_bytes": 462274560, + "latency_s": 2.063535540830344, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 62013440, + "prompt_kind": "short", + "t_relative_s": 13482.833564500324 + }, + { + "decode_latency_s": 0.06048873958934564, + "kv_live_bytes": 462274560, + "latency_s": 1.9369863751344383, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69730304, + "prompt_kind": "long", + "t_relative_s": 13487.843679457903 + }, + { + "decode_latency_s": 0.06092832422291394, + "kv_live_bytes": 462274560, + "latency_s": 1.950518083292991, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71892992, + "prompt_kind": "short", + "t_relative_s": 13492.853759250138 + }, + { + "decode_latency_s": 0.07563639713043813, + "kv_live_bytes": 462274560, + "latency_s": 2.421140917111188, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69779456, + "prompt_kind": "long", + "t_relative_s": 13497.863797375001 + }, + { + "decode_latency_s": 0.06025573566148523, + "kv_live_bytes": 462274560, + "latency_s": 1.929055250249803, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70385664, + "prompt_kind": "short", + "t_relative_s": 13504.988724207971 + }, + { + "decode_latency_s": 0.06743561718030833, + "kv_live_bytes": 462274560, + "latency_s": 2.160733124706894, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70926336, + "prompt_kind": "long", + "t_relative_s": 13509.998802792281 + }, + { + "decode_latency_s": 0.06740822138090152, + "kv_live_bytes": 462274560, + "latency_s": 2.1586557500995696, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71499776, + "prompt_kind": "short", + "t_relative_s": 13515.008874417283 + }, + { + "decode_latency_s": 0.06821707294147927, + "kv_live_bytes": 462274560, + "latency_s": 2.185502375010401, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71565312, + "prompt_kind": "long", + "t_relative_s": 13520.018932458013 + }, + { + "decode_latency_s": 0.0608258163993014, + "kv_live_bytes": 462274560, + "latency_s": 1.9473236673511565, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71139328, + "prompt_kind": "short", + "t_relative_s": 13525.02898195805 + }, + { + "decode_latency_s": 0.06808383984025568, + "kv_live_bytes": 462274560, + "latency_s": 2.1805908330716193, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71106560, + "prompt_kind": "long", + "t_relative_s": 13530.039062667172 + }, + { + "decode_latency_s": 0.060720321591361426, + "kv_live_bytes": 462274560, + "latency_s": 1.9439695831388235, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71745536, + "prompt_kind": "short", + "t_relative_s": 13535.049113166984 + }, + { + "decode_latency_s": 0.06758938799612224, + "kv_live_bytes": 462274560, + "latency_s": 2.165010791271925, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71778304, + "prompt_kind": "long", + "t_relative_s": 13540.05919912504 + }, + { + "decode_latency_s": 0.06058278256386984, + "kv_live_bytes": 462274560, + "latency_s": 1.939411458093673, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72286208, + "prompt_kind": "short", + "t_relative_s": 13545.069297125097 + }, + { + "decode_latency_s": 0.06764643489441369, + "kv_live_bytes": 462274560, + "latency_s": 2.1669144169427454, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72204288, + "prompt_kind": "long", + "t_relative_s": 13550.079347542021 + }, + { + "decode_latency_s": 0.06073447788367048, + "kv_live_bytes": 462274560, + "latency_s": 1.9443652504123747, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72286208, + "prompt_kind": "short", + "t_relative_s": 13555.089431458153 + }, + { + "decode_latency_s": 0.06769679946592078, + "kv_live_bytes": 462274560, + "latency_s": 2.1683890828862786, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71778304, + "prompt_kind": "long", + "t_relative_s": 13560.099519958254 + }, + { + "decode_latency_s": 0.0604198906221427, + "kv_live_bytes": 462274560, + "latency_s": 1.9344026669859886, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72433664, + "prompt_kind": "short", + "t_relative_s": 13565.10961775016 + }, + { + "decode_latency_s": 0.06770146875351202, + "kv_live_bytes": 462274560, + "latency_s": 2.168960167095065, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72450048, + "prompt_kind": "long", + "t_relative_s": 13570.119701832999 + }, + { + "decode_latency_s": 0.060524898435687646, + "kv_live_bytes": 462274560, + "latency_s": 1.9375513335689902, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72974336, + "prompt_kind": "short", + "t_relative_s": 13575.125117749907 + }, + { + "decode_latency_s": 0.06756424740888178, + "kv_live_bytes": 462274560, + "latency_s": 2.164104249794036, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72892416, + "prompt_kind": "long", + "t_relative_s": 13580.135163708124 + }, + { + "decode_latency_s": 0.06035254820017144, + "kv_live_bytes": 462274560, + "latency_s": 1.932146499864757, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72974336, + "prompt_kind": "short", + "t_relative_s": 13585.145233792253 + }, + { + "decode_latency_s": 0.06757920053496491, + "kv_live_bytes": 462274560, + "latency_s": 2.1643073330633342, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72187904, + "prompt_kind": "long", + "t_relative_s": 13590.155304000247 + }, + { + "decode_latency_s": 0.060335776026477106, + "kv_live_bytes": 462274560, + "latency_s": 1.9316463340073824, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72859648, + "prompt_kind": "short", + "t_relative_s": 13595.165355958045 + }, + { + "decode_latency_s": 0.0679352356819436, + "kv_live_bytes": 462274560, + "latency_s": 2.176263333763927, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72892416, + "prompt_kind": "long", + "t_relative_s": 13600.175445500296 + }, + { + "decode_latency_s": 0.06059872396872379, + "kv_live_bytes": 462274560, + "latency_s": 1.939953625202179, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73482240, + "prompt_kind": "short", + "t_relative_s": 13605.18554437533 + }, + { + "decode_latency_s": 0.06820099998731166, + "kv_live_bytes": 462274560, + "latency_s": 2.1848720000125468, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73400320, + "prompt_kind": "long", + "t_relative_s": 13610.195588333067 + }, + { + "decode_latency_s": 0.06031697134312708, + "kv_live_bytes": 462274560, + "latency_s": 1.9310281672514975, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73515008, + "prompt_kind": "short", + "t_relative_s": 13615.205658792052 + }, + { + "decode_latency_s": 0.06798730859009083, + "kv_live_bytes": 462274560, + "latency_s": 2.1774609577842057, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73465856, + "prompt_kind": "long", + "t_relative_s": 13620.215740292333 + }, + { + "decode_latency_s": 0.0603341080568498, + "kv_live_bytes": 462274560, + "latency_s": 1.9316010419279337, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73564160, + "prompt_kind": "short", + "t_relative_s": 13625.225795292296 + }, + { + "decode_latency_s": 0.06744127735146321, + "kv_live_bytes": 462274560, + "latency_s": 2.160275917034596, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73629696, + "prompt_kind": "long", + "t_relative_s": 13630.235873917118 + }, + { + "decode_latency_s": 0.060624568999628536, + "kv_live_bytes": 462274560, + "latency_s": 1.940745749976486, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73629696, + "prompt_kind": "short", + "t_relative_s": 13635.24596487498 + }, + { + "decode_latency_s": 0.06839512371516321, + "kv_live_bytes": 462274560, + "latency_s": 2.1905703339725733, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73531392, + "prompt_kind": "long", + "t_relative_s": 13640.25600916706 + }, + { + "decode_latency_s": 0.060452906240243465, + "kv_live_bytes": 462274560, + "latency_s": 1.9353815829381347, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73744384, + "prompt_kind": "short", + "t_relative_s": 13645.266088000033 + }, + { + "decode_latency_s": 0.06810787109134253, + "kv_live_bytes": 462274560, + "latency_s": 2.1812980421818793, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73777152, + "prompt_kind": "long", + "t_relative_s": 13650.276156499982 + }, + { + "decode_latency_s": 0.060625914062256925, + "kv_live_bytes": 462274560, + "latency_s": 1.9408958749845624, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73809920, + "prompt_kind": "short", + "t_relative_s": 13655.286202125251 + }, + { + "decode_latency_s": 0.06800227606436238, + "kv_live_bytes": 462274560, + "latency_s": 2.178674125112593, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73875456, + "prompt_kind": "long", + "t_relative_s": 13660.296285250224 + }, + { + "decode_latency_s": 0.06064756769046653, + "kv_live_bytes": 462274560, + "latency_s": 1.9415174159221351, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73875456, + "prompt_kind": "short", + "t_relative_s": 13665.306351708248 + }, + { + "decode_latency_s": 0.06789521615428384, + "kv_live_bytes": 462274560, + "latency_s": 2.174719417002052, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73891840, + "prompt_kind": "long", + "t_relative_s": 13670.316400958225 + }, + { + "decode_latency_s": 0.060443217429565266, + "kv_live_bytes": 462274560, + "latency_s": 1.935079500079155, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73777152, + "prompt_kind": "short", + "t_relative_s": 13675.326465375256 + }, + { + "decode_latency_s": 0.06840068490419071, + "kv_live_bytes": 462274560, + "latency_s": 2.190851209219545, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73875456, + "prompt_kind": "long", + "t_relative_s": 13680.33653987525 + }, + { + "decode_latency_s": 0.06061215103545692, + "kv_live_bytes": 462274560, + "latency_s": 1.9404091672040522, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74383360, + "prompt_kind": "short", + "t_relative_s": 13685.346611124929 + }, + { + "decode_latency_s": 0.06750149479194079, + "kv_live_bytes": 462274560, + "latency_s": 2.1624837503768504, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74514432, + "prompt_kind": "long", + "t_relative_s": 13690.356717917137 + }, + { + "decode_latency_s": 0.06060152084683068, + "kv_live_bytes": 462274560, + "latency_s": 1.9400392496027052, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74514432, + "prompt_kind": "short", + "t_relative_s": 13695.366797875147 + }, + { + "decode_latency_s": 0.06798668490955606, + "kv_live_bytes": 462274560, + "latency_s": 2.177874334156513, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74448896, + "prompt_kind": "long", + "t_relative_s": 13700.37685341714 + }, + { + "decode_latency_s": 0.06033330337959342, + "kv_live_bytes": 462274560, + "latency_s": 1.931554083712399, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74465280, + "prompt_kind": "short", + "t_relative_s": 13705.386900499929 + }, + { + "decode_latency_s": 0.06746480597939808, + "kv_live_bytes": 462274560, + "latency_s": 2.1605225834064186, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74514432, + "prompt_kind": "long", + "t_relative_s": 13710.396983667277 + }, + { + "decode_latency_s": 0.060710145815392025, + "kv_live_bytes": 462274560, + "latency_s": 1.9435334159061313, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74596352, + "prompt_kind": "short", + "t_relative_s": 13715.407015624922 + }, + { + "decode_latency_s": 0.06752490362850949, + "kv_live_bytes": 462274560, + "latency_s": 2.163277416024357, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74645504, + "prompt_kind": "long", + "t_relative_s": 13720.417076291982 + }, + { + "decode_latency_s": 0.06080925000424031, + "kv_live_bytes": 462274560, + "latency_s": 1.946700416970998, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74645504, + "prompt_kind": "short", + "t_relative_s": 13725.427143041976 + }, + { + "decode_latency_s": 0.06813137499557342, + "kv_live_bytes": 462274560, + "latency_s": 2.1820510840043426, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74481664, + "prompt_kind": "long", + "t_relative_s": 13730.437194125261 + }, + { + "decode_latency_s": 0.060474677055026405, + "kv_live_bytes": 462274560, + "latency_s": 1.9360984582453966, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74498048, + "prompt_kind": "short", + "t_relative_s": 13735.447253542021 + }, + { + "decode_latency_s": 0.06794856381020509, + "kv_live_bytes": 462274560, + "latency_s": 2.176333792041987, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74547200, + "prompt_kind": "long", + "t_relative_s": 13740.45734583307 + }, + { + "decode_latency_s": 0.06084742055099923, + "kv_live_bytes": 462274560, + "latency_s": 1.9479643749073148, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75071488, + "prompt_kind": "short", + "t_relative_s": 13745.467411458027 + }, + { + "decode_latency_s": 0.06782408725121059, + "kv_live_bytes": 462274560, + "latency_s": 2.172793624922633, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75186176, + "prompt_kind": "long", + "t_relative_s": 13750.477445542347 + }, + { + "decode_latency_s": 0.06076531381404493, + "kv_live_bytes": 462274560, + "latency_s": 1.9453078326769173, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75087872, + "prompt_kind": "short", + "t_relative_s": 13755.487532875035 + }, + { + "decode_latency_s": 0.0683043463359354, + "kv_live_bytes": 462274560, + "latency_s": 2.1882740827277303, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74301440, + "prompt_kind": "long", + "t_relative_s": 13760.49758854229 + }, + { + "decode_latency_s": 0.06035394403443206, + "kv_live_bytes": 462274560, + "latency_s": 1.9322002921253443, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73596928, + "prompt_kind": "short", + "t_relative_s": 13765.507656875066 + }, + { + "decode_latency_s": 0.06762645444541704, + "kv_live_bytes": 462274560, + "latency_s": 2.1655843751505017, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74448896, + "prompt_kind": "long", + "t_relative_s": 13770.517740167212 + }, + { + "decode_latency_s": 0.0608459427166963, + "kv_live_bytes": 462274560, + "latency_s": 1.9479233329184353, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74481664, + "prompt_kind": "short", + "t_relative_s": 13775.527777832933 + }, + { + "decode_latency_s": 0.06727701562340371, + "kv_live_bytes": 462274560, + "latency_s": 2.155203207861632, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74514432, + "prompt_kind": "long", + "t_relative_s": 13780.537833500188 + }, + { + "decode_latency_s": 0.06325515753997024, + "kv_live_bytes": 462274560, + "latency_s": 2.0251493747346103, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72630272, + "prompt_kind": "short", + "t_relative_s": 13785.547886332963 + }, + { + "decode_latency_s": 0.06758977471326943, + "kv_live_bytes": 462274560, + "latency_s": 2.164970957674086, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73449472, + "prompt_kind": "long", + "t_relative_s": 13790.557942500338 + }, + { + "decode_latency_s": 0.06037636849214323, + "kv_live_bytes": 462274560, + "latency_s": 1.932940624654293, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73170944, + "prompt_kind": "short", + "t_relative_s": 13795.568019500002 + }, + { + "decode_latency_s": 0.06773624350898899, + "kv_live_bytes": 462274560, + "latency_s": 2.1692174579948187, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73318400, + "prompt_kind": "long", + "t_relative_s": 13800.578091125004 + }, + { + "decode_latency_s": 0.06073764062603004, + "kv_live_bytes": 462274560, + "latency_s": 1.9444977920502424, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73908224, + "prompt_kind": "short", + "t_relative_s": 13805.58812637534 + }, + { + "decode_latency_s": 0.0676922747225035, + "kv_live_bytes": 462274560, + "latency_s": 2.1688627498224378, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73973760, + "prompt_kind": "long", + "t_relative_s": 13810.598214332946 + }, + { + "decode_latency_s": 0.060915759138879366, + "kv_live_bytes": 462274560, + "latency_s": 1.9501343332231045, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74530816, + "prompt_kind": "short", + "t_relative_s": 13815.608322083019 + }, + { + "decode_latency_s": 0.06054839324497152, + "kv_live_bytes": 462274560, + "latency_s": 1.9384458749555051, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74039296, + "prompt_kind": "long", + "t_relative_s": 13820.618364125025 + }, + { + "decode_latency_s": 0.060534274714882486, + "kv_live_bytes": 462274560, + "latency_s": 1.9379958328790963, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74039296, + "prompt_kind": "short", + "t_relative_s": 13825.62843129225 + }, + { + "decode_latency_s": 0.06803252472309396, + "kv_live_bytes": 462274560, + "latency_s": 2.178990458138287, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74055680, + "prompt_kind": "long", + "t_relative_s": 13830.638512125239 + }, + { + "decode_latency_s": 0.06051016537821852, + "kv_live_bytes": 462274560, + "latency_s": 1.9373002499341965, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74121216, + "prompt_kind": "short", + "t_relative_s": 13835.648552624974 + }, + { + "decode_latency_s": 0.06831459766544867, + "kv_live_bytes": 462274560, + "latency_s": 2.188310916069895, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74285056, + "prompt_kind": "long", + "t_relative_s": 13840.658633917104 + }, + { + "decode_latency_s": 0.061100677092326805, + "kv_live_bytes": 462274560, + "latency_s": 1.956015749834478, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74776576, + "prompt_kind": "short", + "t_relative_s": 13845.668726083357 + }, + { + "decode_latency_s": 0.0668927656224696, + "kv_live_bytes": 462274560, + "latency_s": 2.143161749932915, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74907648, + "prompt_kind": "long", + "t_relative_s": 13850.678779750131 + }, + { + "decode_latency_s": 0.060593295565922745, + "kv_live_bytes": 462274560, + "latency_s": 1.9398996662348509, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74891264, + "prompt_kind": "short", + "t_relative_s": 13855.68138604192 + }, + { + "decode_latency_s": 0.06039118880289607, + "kv_live_bytes": 462274560, + "latency_s": 1.9334867079742253, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73875456, + "prompt_kind": "long", + "t_relative_s": 13860.691468417179 + }, + { + "decode_latency_s": 0.060546546868863516, + "kv_live_bytes": 462274560, + "latency_s": 1.9385100421495736, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74104832, + "prompt_kind": "short", + "t_relative_s": 13865.701517499983 + }, + { + "decode_latency_s": 0.06746539581217803, + "kv_live_bytes": 462274560, + "latency_s": 2.161473708227277, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74514432, + "prompt_kind": "long", + "t_relative_s": 13870.711602708325 + }, + { + "decode_latency_s": 0.060716644540661946, + "kv_live_bytes": 462274560, + "latency_s": 1.9437311249785125, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74907648, + "prompt_kind": "short", + "t_relative_s": 13875.721682291944 + }, + { + "decode_latency_s": 0.06043939452501945, + "kv_live_bytes": 462274560, + "latency_s": 1.9353271671570837, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75038720, + "prompt_kind": "long", + "t_relative_s": 13880.731736292131 + }, + { + "decode_latency_s": 0.06068432684696745, + "kv_live_bytes": 462274560, + "latency_s": 1.942689208779484, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74514432, + "prompt_kind": "short", + "t_relative_s": 13885.741817499977 + }, + { + "decode_latency_s": 0.06783286199788563, + "kv_live_bytes": 462274560, + "latency_s": 2.1726511251181364, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73433088, + "prompt_kind": "long", + "t_relative_s": 13890.751900041942 + }, + { + "decode_latency_s": 0.0606395976501517, + "kv_live_bytes": 462274560, + "latency_s": 1.9413813748396933, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73580544, + "prompt_kind": "short", + "t_relative_s": 13895.761956417002 + }, + { + "decode_latency_s": 0.06802989062271081, + "kv_live_bytes": 462274560, + "latency_s": 2.179140332620591, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74153984, + "prompt_kind": "long", + "t_relative_s": 13900.772040083073 + }, + { + "decode_latency_s": 0.06127884116722271, + "kv_live_bytes": 462274560, + "latency_s": 1.96175791695714, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74137600, + "prompt_kind": "short", + "t_relative_s": 13905.782103333157 + }, + { + "decode_latency_s": 0.06055851040582638, + "kv_live_bytes": 462274560, + "latency_s": 1.9387291660532355, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74842112, + "prompt_kind": "long", + "t_relative_s": 13910.792181750294 + }, + { + "decode_latency_s": 0.060611464839894325, + "kv_live_bytes": 462274560, + "latency_s": 1.9403987089172006, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74285056, + "prompt_kind": "short", + "t_relative_s": 13915.802248999942 + }, + { + "decode_latency_s": 0.06786541144538205, + "kv_live_bytes": 462274560, + "latency_s": 2.174290666822344, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74285056, + "prompt_kind": "long", + "t_relative_s": 13920.812296333257 + }, + { + "decode_latency_s": 0.060468643219792284, + "kv_live_bytes": 462274560, + "latency_s": 1.9358862908557057, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73695232, + "prompt_kind": "short", + "t_relative_s": 13925.822358750273 + }, + { + "decode_latency_s": 0.06819551302760374, + "kv_live_bytes": 462274560, + "latency_s": 2.1837873328477144, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74366976, + "prompt_kind": "long", + "t_relative_s": 13930.832434375305 + }, + { + "decode_latency_s": 0.06070769269717857, + "kv_live_bytes": 462274560, + "latency_s": 1.943481083959341, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74842112, + "prompt_kind": "short", + "t_relative_s": 13935.842487333342 + }, + { + "decode_latency_s": 0.06070476431341376, + "kv_live_bytes": 462274560, + "latency_s": 1.9435748751275241, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74956800, + "prompt_kind": "long", + "t_relative_s": 13940.852571375202 + }, + { + "decode_latency_s": 0.060562945320270956, + "kv_live_bytes": 462274560, + "latency_s": 1.938785333186388, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74858496, + "prompt_kind": "short", + "t_relative_s": 13945.862707500346 + }, + { + "decode_latency_s": 0.06823991014971398, + "kv_live_bytes": 462274560, + "latency_s": 2.1860742093995214, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74874880, + "prompt_kind": "long", + "t_relative_s": 13950.87275737524 + }, + { + "decode_latency_s": 0.060576084622880444, + "kv_live_bytes": 462274560, + "latency_s": 1.9393225843086839, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73875456, + "prompt_kind": "short", + "t_relative_s": 13955.882849541958 + }, + { + "decode_latency_s": 0.06746470181678887, + "kv_live_bytes": 462274560, + "latency_s": 2.161239583976567, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74514432, + "prompt_kind": "long", + "t_relative_s": 13960.892907375004 + }, + { + "decode_latency_s": 0.060330178384901956, + "kv_live_bytes": 462274560, + "latency_s": 1.9314064583741128, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75055104, + "prompt_kind": "short", + "t_relative_s": 13965.90299412515 + }, + { + "decode_latency_s": 0.06810980990121607, + "kv_live_bytes": 462274560, + "latency_s": 2.1811715411022305, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74481664, + "prompt_kind": "long", + "t_relative_s": 13970.913067458197 + }, + { + "decode_latency_s": 0.06023952865507454, + "kv_live_bytes": 462274560, + "latency_s": 1.928567708004266, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74645504, + "prompt_kind": "short", + "t_relative_s": 13975.923116333317 + }, + { + "decode_latency_s": 0.06812970053579193, + "kv_live_bytes": 462274560, + "latency_s": 2.182219082955271, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74678272, + "prompt_kind": "long", + "t_relative_s": 13980.933193000033 + }, + { + "decode_latency_s": 0.06084578904847149, + "kv_live_bytes": 462274560, + "latency_s": 1.947895917110145, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74711040, + "prompt_kind": "short", + "t_relative_s": 13985.943304332905 + }, + { + "decode_latency_s": 0.06053563543537166, + "kv_live_bytes": 462274560, + "latency_s": 1.9384592501446605, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74743808, + "prompt_kind": "long", + "t_relative_s": 13990.953357083257 + }, + { + "decode_latency_s": 0.06044978775025811, + "kv_live_bytes": 462274560, + "latency_s": 1.9352167500182986, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 75284480, + "prompt_kind": "short", + "t_relative_s": 13995.963452208322 + }, + { + "decode_latency_s": 0.06816899219120387, + "kv_live_bytes": 462274560, + "latency_s": 2.183446791023016, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74399744, + "prompt_kind": "long", + "t_relative_s": 14000.973533208016 + }, + { + "decode_latency_s": 0.06032330858579371, + "kv_live_bytes": 462274560, + "latency_s": 1.9312290837988257, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74678272, + "prompt_kind": "short", + "t_relative_s": 14005.98357616691 + }, + { + "decode_latency_s": 0.06780986981175374, + "kv_live_bytes": 462274560, + "latency_s": 2.1722359163686633, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74694656, + "prompt_kind": "long", + "t_relative_s": 14010.993663208093 + }, + { + "decode_latency_s": 0.06086561587289907, + "kv_live_bytes": 462274560, + "latency_s": 1.948492584284395, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74711040, + "prompt_kind": "short", + "t_relative_s": 14016.003771833144 + }, + { + "decode_latency_s": 0.06744768749922514, + "kv_live_bytes": 462274560, + "latency_s": 2.160916999913752, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74825728, + "prompt_kind": "long", + "t_relative_s": 14021.013830000069 + }, + { + "decode_latency_s": 0.06035879427508917, + "kv_live_bytes": 462274560, + "latency_s": 1.9323807079344988, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74596352, + "prompt_kind": "short", + "t_relative_s": 14026.023908625357 + }, + { + "decode_latency_s": 0.06944196484982967, + "kv_live_bytes": 462274560, + "latency_s": 2.223787957802415, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72630272, + "prompt_kind": "long", + "t_relative_s": 14031.03397837514 + }, + { + "decode_latency_s": 0.060251898437854834, + "kv_live_bytes": 462274560, + "latency_s": 1.9289319589734077, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74498048, + "prompt_kind": "short", + "t_relative_s": 14036.044042250142 + }, + { + "decode_latency_s": 0.07119676303409506, + "kv_live_bytes": 462274560, + "latency_s": 2.2805724581703544, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73482240, + "prompt_kind": "long", + "t_relative_s": 14041.054113958031 + }, + { + "decode_latency_s": 0.060626497404882684, + "kv_live_bytes": 462274560, + "latency_s": 1.9408632921986282, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73236480, + "prompt_kind": "short", + "t_relative_s": 14046.064189542085 + }, + { + "decode_latency_s": 0.06811642840330023, + "kv_live_bytes": 462274560, + "latency_s": 2.1818767921067774, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73728000, + "prompt_kind": "long", + "t_relative_s": 14051.074266416952 + }, + { + "decode_latency_s": 0.06041889716289006, + "kv_live_bytes": 462274560, + "latency_s": 1.9342926666140556, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 69435392, + "prompt_kind": "short", + "t_relative_s": 14056.08431962505 + }, + { + "decode_latency_s": 0.06817281249095686, + "kv_live_bytes": 462274560, + "latency_s": 2.1838462497107685, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73891840, + "prompt_kind": "long", + "t_relative_s": 14061.094358708244 + }, + { + "decode_latency_s": 0.06026770702737849, + "kv_live_bytes": 462274560, + "latency_s": 1.9294798751361668, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73973760, + "prompt_kind": "short", + "t_relative_s": 14066.10443341732 + }, + { + "decode_latency_s": 0.0677700755186379, + "kv_live_bytes": 462274560, + "latency_s": 2.170136125292629, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74203136, + "prompt_kind": "long", + "t_relative_s": 14071.114556458313 + }, + { + "decode_latency_s": 0.06086703775508795, + "kv_live_bytes": 462274560, + "latency_s": 1.948732334189117, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74285056, + "prompt_kind": "short", + "t_relative_s": 14076.12461400032 + }, + { + "decode_latency_s": 0.06828672137635294, + "kv_live_bytes": 462274560, + "latency_s": 2.1875518327578902, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74383360, + "prompt_kind": "long", + "t_relative_s": 14081.134701167233 + }, + { + "decode_latency_s": 0.060965717435465194, + "kv_live_bytes": 462274560, + "latency_s": 1.9517026250250638, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74268672, + "prompt_kind": "short", + "t_relative_s": 14086.144822957925 + }, + { + "decode_latency_s": 0.06830729034845717, + "kv_live_bytes": 462274560, + "latency_s": 2.188245415687561, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74874880, + "prompt_kind": "long", + "t_relative_s": 14091.15487591736 + }, + { + "decode_latency_s": 0.060381115865311585, + "kv_live_bytes": 462274560, + "latency_s": 1.9331602090969682, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74956800, + "prompt_kind": "short", + "t_relative_s": 14096.16496241698 + }, + { + "decode_latency_s": 0.0680011666554492, + "kv_live_bytes": 462274560, + "latency_s": 2.1777805001474917, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74219520, + "prompt_kind": "long", + "t_relative_s": 14101.175030750223 + }, + { + "decode_latency_s": 0.06035796743526589, + "kv_live_bytes": 462274560, + "latency_s": 1.9323762501589954, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74481664, + "prompt_kind": "short", + "t_relative_s": 14106.185069125146 + }, + { + "decode_latency_s": 0.06054552344721742, + "kv_live_bytes": 462274560, + "latency_s": 1.9384057503193617, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74645504, + "prompt_kind": "long", + "t_relative_s": 14111.19515166711 + }, + { + "decode_latency_s": 0.06343681509315502, + "kv_live_bytes": 462274560, + "latency_s": 2.0310006248764694, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66764800, + "prompt_kind": "short", + "t_relative_s": 14116.205223750323 + }, + { + "decode_latency_s": 0.0674204726528842, + "kv_live_bytes": 462274560, + "latency_s": 2.1597415832802653, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73613312, + "prompt_kind": "long", + "t_relative_s": 14121.215301292017 + }, + { + "decode_latency_s": 0.06049582554260269, + "kv_live_bytes": 462274560, + "latency_s": 1.9367585419677198, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73678848, + "prompt_kind": "short", + "t_relative_s": 14126.225344750099 + }, + { + "decode_latency_s": 0.06857640753150918, + "kv_live_bytes": 462274560, + "latency_s": 2.196833458263427, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74088448, + "prompt_kind": "long", + "t_relative_s": 14131.235415792093 + }, + { + "decode_latency_s": 0.06308974609419238, + "kv_live_bytes": 462274560, + "latency_s": 2.019810250028968, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72646656, + "prompt_kind": "short", + "t_relative_s": 14136.245464042295 + }, + { + "decode_latency_s": 0.06980350390949752, + "kv_live_bytes": 462274560, + "latency_s": 2.2360777501016855, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70680576, + "prompt_kind": "long", + "t_relative_s": 14141.25554145826 + }, + { + "decode_latency_s": 0.06088421356980689, + "kv_live_bytes": 462274560, + "latency_s": 1.9491889169439673, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72761344, + "prompt_kind": "short", + "t_relative_s": 14146.265609750059 + }, + { + "decode_latency_s": 0.06799307030451018, + "kv_live_bytes": 462274560, + "latency_s": 2.1775334998965263, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72777728, + "prompt_kind": "long", + "t_relative_s": 14151.275685042143 + }, + { + "decode_latency_s": 0.060616083355853334, + "kv_live_bytes": 462274560, + "latency_s": 1.940571874845773, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72957952, + "prompt_kind": "short", + "t_relative_s": 14156.28576291725 + }, + { + "decode_latency_s": 0.06753648960147984, + "kv_live_bytes": 462274560, + "latency_s": 2.1631582910194993, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73990144, + "prompt_kind": "long", + "t_relative_s": 14161.295822250191 + }, + { + "decode_latency_s": 0.060718488282873295, + "kv_live_bytes": 462274560, + "latency_s": 1.943889459129423, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73957376, + "prompt_kind": "short", + "t_relative_s": 14166.305892582983 + }, + { + "decode_latency_s": 0.06246530340285972, + "kv_live_bytes": 462274560, + "latency_s": 2.00039237504825, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74055680, + "prompt_kind": "long", + "t_relative_s": 14171.315940208267 + }, + { + "decode_latency_s": 0.06049003906082362, + "kv_live_bytes": 462274560, + "latency_s": 1.936455124989152, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73891840, + "prompt_kind": "short", + "t_relative_s": 14176.326021333225 + }, + { + "decode_latency_s": 0.0679346184770111, + "kv_live_bytes": 462274560, + "latency_s": 2.1761719998903573, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73924608, + "prompt_kind": "long", + "t_relative_s": 14181.336065292358 + }, + { + "decode_latency_s": 0.060469472664408386, + "kv_live_bytes": 462274560, + "latency_s": 1.9359376667998731, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73334784, + "prompt_kind": "short", + "t_relative_s": 14186.34612029232 + }, + { + "decode_latency_s": 0.06809582031564787, + "kv_live_bytes": 462274560, + "latency_s": 2.181184458080679, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74203136, + "prompt_kind": "long", + "t_relative_s": 14191.356157083064 + }, + { + "decode_latency_s": 0.06044383204425685, + "kv_live_bytes": 462274560, + "latency_s": 1.9350832919590175, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74235904, + "prompt_kind": "short", + "t_relative_s": 14196.366237208247 + }, + { + "decode_latency_s": 0.06802668877935503, + "kv_live_bytes": 462274560, + "latency_s": 2.1784034171141684, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 52379648, + "prompt_kind": "long", + "t_relative_s": 14201.376340582967 + }, + { + "decode_latency_s": 0.06680991147004534, + "kv_live_bytes": 462274560, + "latency_s": 2.1398110832087696, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 72892416, + "prompt_kind": "short", + "t_relative_s": 14206.38642504206 + }, + { + "decode_latency_s": 0.06791849480941892, + "kv_live_bytes": 462274560, + "latency_s": 2.175632707774639, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73990144, + "prompt_kind": "long", + "t_relative_s": 14211.396472417284 + }, + { + "decode_latency_s": 0.06029459374258295, + "kv_live_bytes": 462274560, + "latency_s": 1.930324333254248, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73613312, + "prompt_kind": "short", + "t_relative_s": 14216.406539458316 + }, + { + "decode_latency_s": 0.06750260418630205, + "kv_live_bytes": 462274560, + "latency_s": 2.1616426669061184, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74006528, + "prompt_kind": "long", + "t_relative_s": 14221.416630708147 + }, + { + "decode_latency_s": 0.06710641668178141, + "kv_live_bytes": 462274560, + "latency_s": 2.1495372499339283, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74235904, + "prompt_kind": "short", + "t_relative_s": 14226.426683542319 + }, + { + "decode_latency_s": 0.067162050792831, + "kv_live_bytes": 462274560, + "latency_s": 2.1515570003539324, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74285056, + "prompt_kind": "long", + "t_relative_s": 14231.436754208058 + }, + { + "decode_latency_s": 0.0680847656331025, + "kv_live_bytes": 462274560, + "latency_s": 2.180059750098735, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73564160, + "prompt_kind": "short", + "t_relative_s": 14236.440743542276 + }, + { + "decode_latency_s": 0.06765415887639392, + "kv_live_bytes": 462274560, + "latency_s": 2.166797166224569, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74317824, + "prompt_kind": "long", + "t_relative_s": 14241.450839417055 + }, + { + "decode_latency_s": 0.06061476953618694, + "kv_live_bytes": 462274560, + "latency_s": 1.940455457661301, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74285056, + "prompt_kind": "short", + "t_relative_s": 14246.46093075024 + }, + { + "decode_latency_s": 0.061673687494476326, + "kv_live_bytes": 462274560, + "latency_s": 1.9749687500298023, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74334208, + "prompt_kind": "long", + "t_relative_s": 14251.470976583194 + }, + { + "decode_latency_s": 0.060504864566610195, + "kv_live_bytes": 462274560, + "latency_s": 1.9369191252626479, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74301440, + "prompt_kind": "short", + "t_relative_s": 14256.481064124964 + }, + { + "decode_latency_s": 0.0681957864726428, + "kv_live_bytes": 462274560, + "latency_s": 2.184253416955471, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73662464, + "prompt_kind": "long", + "t_relative_s": 14261.491149958223 + }, + { + "decode_latency_s": 0.06064315883850213, + "kv_live_bytes": 462274560, + "latency_s": 1.9414277919568121, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74350592, + "prompt_kind": "short", + "t_relative_s": 14266.497880042065 + }, + { + "decode_latency_s": 0.06788216016138904, + "kv_live_bytes": 462274560, + "latency_s": 2.174694749992341, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74858496, + "prompt_kind": "long", + "t_relative_s": 14271.507929500192 + }, + { + "decode_latency_s": 0.060296165363979526, + "kv_live_bytes": 462274560, + "latency_s": 1.9303388753905892, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74301440, + "prompt_kind": "short", + "t_relative_s": 14276.517981375102 + }, + { + "decode_latency_s": 0.0691175950341858, + "kv_live_bytes": 462274560, + "latency_s": 2.213015957735479, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 67452928, + "prompt_kind": "long", + "t_relative_s": 14281.528063333128 + }, + { + "decode_latency_s": 0.06091132944857236, + "kv_live_bytes": 462274560, + "latency_s": 1.9499901672825217, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71090176, + "prompt_kind": "short", + "t_relative_s": 14286.536630624905 + }, + { + "decode_latency_s": 0.06141983463021461, + "kv_live_bytes": 462274560, + "latency_s": 1.9668934172950685, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73793536, + "prompt_kind": "long", + "t_relative_s": 14291.54669004213 + }, + { + "decode_latency_s": 0.06252245443465654, + "kv_live_bytes": 462274560, + "latency_s": 2.0018027503974736, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73777152, + "prompt_kind": "short", + "t_relative_s": 14296.556764750276 + }, + { + "decode_latency_s": 0.06810366015997715, + "kv_live_bytes": 462274560, + "latency_s": 2.1816262919455767, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74416128, + "prompt_kind": "long", + "t_relative_s": 14301.56686212495 + }, + { + "decode_latency_s": 0.06040512632171158, + "kv_live_bytes": 462274560, + "latency_s": 1.9337249579839408, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74596352, + "prompt_kind": "short", + "t_relative_s": 14306.576935457997 + }, + { + "decode_latency_s": 0.06863992057333235, + "kv_live_bytes": 462274560, + "latency_s": 2.198324624914676, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74694656, + "prompt_kind": "long", + "t_relative_s": 14311.586987667251 + }, + { + "decode_latency_s": 0.060367010402842425, + "kv_live_bytes": 462274560, + "latency_s": 1.9327349998056889, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74235904, + "prompt_kind": "short", + "t_relative_s": 14316.597048792057 + }, + { + "decode_latency_s": 0.06807138804288115, + "kv_live_bytes": 462274560, + "latency_s": 2.1799823748879135, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74317824, + "prompt_kind": "long", + "t_relative_s": 14321.607132958248 + }, + { + "decode_latency_s": 0.06046865103417076, + "kv_live_bytes": 462274560, + "latency_s": 1.9359830422326922, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74891264, + "prompt_kind": "short", + "t_relative_s": 14326.617182417307 + }, + { + "decode_latency_s": 0.06782307944376953, + "kv_live_bytes": 462274560, + "latency_s": 2.1727071669884026, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74760192, + "prompt_kind": "long", + "t_relative_s": 14331.62726774998 + }, + { + "decode_latency_s": 0.060606263039517216, + "kv_live_bytes": 462274560, + "latency_s": 1.9401857918128371, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74940416, + "prompt_kind": "short", + "t_relative_s": 14336.637343292125 + }, + { + "decode_latency_s": 0.06800741536426358, + "kv_live_bytes": 462274560, + "latency_s": 2.178320167120546, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73564160, + "prompt_kind": "long", + "t_relative_s": 14341.64739412535 + }, + { + "decode_latency_s": 0.06044995444244705, + "kv_live_bytes": 462274560, + "latency_s": 1.9353296672925353, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 73908224, + "prompt_kind": "short", + "t_relative_s": 14346.657453583088 + }, + { + "decode_latency_s": 0.06922685283643659, + "kv_live_bytes": 462274560, + "latency_s": 2.2174283326603472, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 71680000, + "prompt_kind": "long", + "t_relative_s": 14351.667539375369 + }, + { + "decode_latency_s": 0.06040965497959405, + "kv_live_bytes": 462274560, + "latency_s": 1.9341657501645386, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74072064, + "prompt_kind": "short", + "t_relative_s": 14356.67759337509 + }, + { + "decode_latency_s": 0.06812563799030613, + "kv_live_bytes": 462274560, + "latency_s": 2.182012958917767, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74776576, + "prompt_kind": "long", + "t_relative_s": 14361.68766937498 + }, + { + "decode_latency_s": 0.06080827084952034, + "kv_live_bytes": 462274560, + "latency_s": 1.946680957917124, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74940416, + "prompt_kind": "short", + "t_relative_s": 14366.693770792335 + }, + { + "decode_latency_s": 0.06837962109420914, + "kv_live_bytes": 462274560, + "latency_s": 2.1907313750125468, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74465280, + "prompt_kind": "long", + "t_relative_s": 14371.703821083065 + }, + { + "decode_latency_s": 0.06065113803197164, + "kv_live_bytes": 462274560, + "latency_s": 1.94162887474522, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74481664, + "prompt_kind": "short", + "t_relative_s": 14376.713920041919 + }, + { + "decode_latency_s": 0.06867097919166554, + "kv_live_bytes": 462274560, + "latency_s": 2.199203500058502, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 66781184, + "prompt_kind": "long", + "t_relative_s": 14381.723968000151 + }, + { + "decode_latency_s": 0.06084484895109199, + "kv_live_bytes": 462274560, + "latency_s": 1.9479982499033213, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 70991872, + "prompt_kind": "short", + "t_relative_s": 14386.734054542147 + }, + { + "decode_latency_s": 0.06767613021656871, + "kv_live_bytes": 462274560, + "latency_s": 2.1681315419264138, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74072064, + "prompt_kind": "long", + "t_relative_s": 14391.744112625252 + }, + { + "decode_latency_s": 0.06035307289857883, + "kv_live_bytes": 462274560, + "latency_s": 1.9322260841727257, + "n_emitted": 32, + "ok": true, + "process_footprint_bytes": 74203136, + "prompt_kind": "short", + "t_relative_s": 14396.754187125247 + } + ] + }, + "gate": "mixed_prompt_endurance", + "metrics": { + "aggregate": { + "buckets_10min": [ + { + "bucket_index": 0, + "mean_kv_live_bytes": 399360999.0243902, + "n_turns": 41, + "p50_latency_s": 2.341742292046547, + "p95_latency_s": 52.34493345860392 + }, + { + "bucket_index": 1, + "mean_kv_live_bytes": 462274560, + "n_turns": 15, + "p50_latency_s": 3.7953444998711348, + "p95_latency_s": 148.82869383310862 + }, + { + "bucket_index": 2, + "mean_kv_live_bytes": 462274560, + "n_turns": 75, + "p50_latency_s": 2.1362737081944942, + "p95_latency_s": 2.7404998499434394 + }, + { + "bucket_index": 3, + "mean_kv_live_bytes": 462274560, + "n_turns": 120, + "p50_latency_s": 2.119702583178878, + "p95_latency_s": 2.1934768794570116 + }, + { + "bucket_index": 4, + "mean_kv_live_bytes": 462274560, + "n_turns": 119, + "p50_latency_s": 2.1287325830198824, + "p95_latency_s": 2.1952770628035068 + }, + { + "bucket_index": 5, + "mean_kv_live_bytes": 462274560, + "n_turns": 119, + "p50_latency_s": 2.0929697919636965, + "p95_latency_s": 2.198374154837802 + }, + { + "bucket_index": 6, + "mean_kv_live_bytes": 462274560, + "n_turns": 120, + "p50_latency_s": 2.0267889585811645, + "p95_latency_s": 2.190342754404992 + }, + { + "bucket_index": 7, + "mean_kv_live_bytes": 462274560, + "n_turns": 115, + "p50_latency_s": 2.030197416432202, + "p95_latency_s": 2.2249790540430694 + }, + { + "bucket_index": 8, + "mean_kv_live_bytes": 462274560, + "n_turns": 119, + "p50_latency_s": 1.9814810841344297, + "p95_latency_s": 2.186367407720536 + }, + { + "bucket_index": 9, + "mean_kv_live_bytes": 462274560, + "n_turns": 119, + "p50_latency_s": 2.133393708150834, + "p95_latency_s": 2.196558258682489 + }, + { + "bucket_index": 10, + "mean_kv_live_bytes": 462274560, + "n_turns": 119, + "p50_latency_s": 2.1431248327717185, + "p95_latency_s": 2.187859428534284 + }, + { + "bucket_index": 11, + "mean_kv_live_bytes": 462274560, + "n_turns": 119, + "p50_latency_s": 2.1319570830091834, + "p95_latency_s": 2.1923822838347404 + }, + { + "bucket_index": 12, + "mean_kv_live_bytes": 462274560, + "n_turns": 119, + "p50_latency_s": 1.9491943749599159, + "p95_latency_s": 2.1891607338096946 + }, + { + "bucket_index": 13, + "mean_kv_live_bytes": 462274560, + "n_turns": 118, + "p50_latency_s": 1.9672603332437575, + "p95_latency_s": 2.192311875009909 + }, + { + "bucket_index": 14, + "mean_kv_live_bytes": 462274560, + "n_turns": 118, + "p50_latency_s": 1.9978569999802858, + "p95_latency_s": 2.194456239347346 + }, + { + "bucket_index": 15, + "mean_kv_live_bytes": 462274560, + "n_turns": 120, + "p50_latency_s": 2.139613520586863, + "p95_latency_s": 2.187065088143572 + }, + { + "bucket_index": 16, + "mean_kv_live_bytes": 462274560, + "n_turns": 120, + "p50_latency_s": 2.1221560414414853, + "p95_latency_s": 2.1909621228231115 + }, + { + "bucket_index": 17, + "mean_kv_live_bytes": 462274560, + "n_turns": 119, + "p50_latency_s": 1.948968917131424, + "p95_latency_s": 2.194352866336703 + }, + { + "bucket_index": 18, + "mean_kv_live_bytes": 462274560, + "n_turns": 119, + "p50_latency_s": 2.1212582918815315, + "p95_latency_s": 2.192575104115531 + }, + { + "bucket_index": 19, + "mean_kv_live_bytes": 462274560, + "n_turns": 119, + "p50_latency_s": 2.0085462909191847, + "p95_latency_s": 2.1986037076916545 + }, + { + "bucket_index": 20, + "mean_kv_live_bytes": 462274560, + "n_turns": 120, + "p50_latency_s": 2.127229542005807, + "p95_latency_s": 2.193445692001842 + }, + { + "bucket_index": 21, + "mean_kv_live_bytes": 462274560, + "n_turns": 119, + "p50_latency_s": 2.055574500001967, + "p95_latency_s": 2.2043874628376217 + }, + { + "bucket_index": 22, + "mean_kv_live_bytes": 462274560, + "n_turns": 118, + "p50_latency_s": 2.0994166454765946, + "p95_latency_s": 2.1969014376401903 + }, + { + "bucket_index": 23, + "mean_kv_live_bytes": 462274560, + "n_turns": 120, + "p50_latency_s": 1.958886833395809, + "p95_latency_s": 2.1983685686718673 + } + ], + "duration_s": 14401.82984066708, + "kv_bounded": false, + "latency_drift_p50_s": 0.3687449167482555, + "max_kv_live_bytes": 462274560, + "mean_kv_live_bytes": 461293405.1882845, + "min_kv_live_bytes": 9011200, + "n_errors": 0, + "n_turns": 2629, + "p50_latency_s": 2.086497541051358, + "p95_latency_s": 2.1988716083578765, + "prefill_bounded": true + }, + "duration_s": 14401.82984066708, + "memory_bucket_medians_bytes": [ + 70516736.0, + 71139328.0, + 71909376.0, + 73891840.0, + 73285632.0, + 73760768.0, + 73023488.0, + 73588736.0 + ], + "monotonic_memory_growth": false, + "prompt_kinds": [ + "long", + "short" + ] + }, + "status": "passed", + "summary": "14401.8/14400.0s; errors=0; monotonic_memory=False; p50_drift_s=0.3687449167482555", + "thresholds": { + "max_errors": 0, + "max_p50_drift_s": 5.0, + "minimum_monotonic_growth_bytes": 67108864, + "required_prompt_kinds": [ + "short", + "long" + ], + "target_duration_s": 14400.0 + } + }, + { + "evidence": {}, + "gate": "disconnect_cancellation", + "metrics": { + "active_generations": 0, + "active_sessions": 0, + "cancellation_elapsed_s": 0.5925894170068204 + }, + "status": "passed", + "summary": "settled in 0.593s; active_generations=0; active_sessions=0", + "thresholds": { + "max_cancellation_elapsed_s": 5.0, + "required_active_generations": 0, + "required_active_sessions": 0 + } + }, + { + "evidence": {}, + "gate": "injected_hang_recycle", + "metrics": { + "injection_accepted": true, + "recycle_elapsed_s": 111.7483552498743, + "restart_count_after": 1, + "restart_count_before": 0, + "worker_pid_after": 17254, + "worker_pid_before": 54542 + }, + "status": "passed", + "summary": "accepted=True; recycled=True; elapsed=111.748s", + "thresholds": { + "max_recycle_elapsed_s": 120.0 + } + }, + { + "evidence": {}, + "gate": "kv_restore_parity", + "metrics": { + "baseline_first_token_id": 107, + "baseline_logits_sha256": "dc1085c5ae016ae2e81a717c78065a630bdd44a2f5f84e10fdb2515822686b2b", + "logits_match": true, + "restore_source": "allens_kv+proof_checkpoint", + "restored_first_token_id": 107, + "restored_logits_sha256": "dc1085c5ae016ae2e81a717c78065a630bdd44a2f5f84e10fdb2515822686b2b", + "token_match": true + }, + "status": "passed", + "summary": "token_match=True; logits_match=True; restore_source=allens_kv+proof_checkpoint", + "thresholds": { + "require_logits_sha256_match": true, + "require_token_match": true, + "required_restore_source": "allens_kv+proof_checkpoint" + } + }, + { + "evidence": {}, + "gate": "decode_latency_regression", + "metrics": { + "baseline_p50_decode_latency_s": 0.04455711807046706, + "baseline_p95_decode_latency_s": 0.04455711807046706, + "baseline_source": "bench_mlx_verifier_mean_per_token", + "long_run_p50_drift_s": 0.3687449167482555, + "p50_decode_latency_s": 0.03768789650348481, + "p50_regression_fraction": -0.1541666486624782, + "p95_decode_latency_s": 0.03850383528406383, + "p95_regression_fraction": -0.13585445038949717, + "sample_count": 20 + }, + "status": "passed", + "summary": "p50 regression=-15.42%; p95 regression=-13.59%; long-run drift=0.369s", + "thresholds": { + "max_long_run_p50_drift_s": 5.0, + "max_regression_fraction": 0.15 + } + } + ], + "git": { + "branch": "agent/primary-decode-worker-acceptance-integration-0721", + "commit": "c64bd8ec2ed92a7b2d593a92f090fa247d9ff3f0", + "dirty": false + }, + "kind": "primary_decode_mac_acceptance", + "latency": { + "p50_decode_latency_s": 0.03768789650348481, + "p95_decode_latency_s": 0.03850383528406383, + "sample_count": 20 + }, + "schema_version": 1, + "started_at": "2026-07-21T14:30:02.598234+00:00", + "status": "passed" +} diff --git a/results/platform-tests/primary_decode_acceptance.xml b/results/platform-tests/primary_decode_acceptance.xml new file mode 100644 index 00000000..969ded9d --- /dev/null +++ b/results/platform-tests/primary_decode_acceptance.xml @@ -0,0 +1 @@ +{"evidence": {}, "gate": "sequential_session_footprint", "metrics": {"active_sessions": 0, "final_process_footprint_bytes": 54394880, "fresh_process_footprint_bytes": 414597120, "growth_bytes": -360202240, "sessions_completed": 100}, "status": "passed", "summary": "100/100 sessions; footprint growth -360202240 bytes; active sessions 0", "thresholds": {"max_growth_bytes": 2147483648, "required_active_sessions": 0, "required_sessions": 100}}{"evidence": {"records": [{"decode_latency_s": 0.04194158075551968, "kv_live_bytes": 9011200, "latency_s": 1.3450685827992857, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70680576, "prompt_kind": "short", "t_relative_s": 0.003435875289142132}, {"decode_latency_s": 0.04726975780795328, "kv_live_bytes": 81776640, "latency_s": 1.514664291869849, "n_emitted": 32, "ok": true, "process_footprint_bytes": 49594368, "prompt_kind": "long", "t_relative_s": 5.008463582955301}, {"decode_latency_s": 0.19448144662601408, "kv_live_bytes": 90787840, "latency_s": 6.224768208805472, "n_emitted": 32, "ok": true, "process_footprint_bytes": 65798144, "prompt_kind": "short", "t_relative_s": 10.013505792245269}, {"decode_latency_s": 0.049309817724861205, "kv_live_bytes": 163553280, "latency_s": 1.5792533331550658, "n_emitted": 32, "ok": true, "process_footprint_bytes": 63193088, "prompt_kind": "long", "t_relative_s": 18.73200725018978}, {"decode_latency_s": 0.04958822394837625, "kv_live_bytes": 172339200, "latency_s": 1.587842124979943, "n_emitted": 32, "ok": true, "process_footprint_bytes": 54722560, "prompt_kind": "short", "t_relative_s": 23.737073583062738}, {"decode_latency_s": 1.6356374218739802, "kv_live_bytes": 245104640, "latency_s": 52.34493345860392, "n_emitted": 32, "ok": true, "process_footprint_bytes": 42876928, "prompt_kind": "long", "t_relative_s": 28.742133708205074}, {"decode_latency_s": 0.05208623046928551, "kv_live_bytes": 254115840, "latency_s": 1.6679823333397508, "n_emitted": 32, "ok": true, "process_footprint_bytes": 64307200, "prompt_kind": "short", "t_relative_s": 85.75188595801592}, {"decode_latency_s": 0.07313610415440053, "kv_live_bytes": 326881280, "latency_s": 2.341742292046547, "n_emitted": 32, "ok": true, "process_footprint_bytes": 60276736, "prompt_kind": "long", "t_relative_s": 90.75712504191324}, {"decode_latency_s": 0.05946037500689272, "kv_live_bytes": 335892480, "latency_s": 1.9039542502723634, "n_emitted": 32, "ok": true, "process_footprint_bytes": 65142784, "prompt_kind": "short", "t_relative_s": 97.20100595802069}, {"decode_latency_s": 1.7831243528489722, "kv_live_bytes": 408657920, "latency_s": 57.09534433390945, "n_emitted": 32, "ok": true, "process_footprint_bytes": 27983872, "prompt_kind": "long", "t_relative_s": 102.20604270836338}, {"decode_latency_s": 0.6393040963448584, "kv_live_bytes": 417443840, "latency_s": 20.466463500168175, "n_emitted": 32, "ok": true, "process_footprint_bytes": 57819136, "prompt_kind": "short", "t_relative_s": 168.81527866702527}, {"decode_latency_s": 0.6018479453196051, "kv_live_bytes": 462274560, "latency_s": 19.264735292177647, "n_emitted": 32, "ok": true, "process_footprint_bytes": 58245120, "prompt_kind": "long", "t_relative_s": 192.83681804221123}, {"decode_latency_s": 0.08589695965929423, "kv_live_bytes": 462274560, "latency_s": 2.755477999802679, "n_emitted": 32, "ok": true, "process_footprint_bytes": 64569344, "prompt_kind": "short", "t_relative_s": 216.8982549579814}, {"decode_latency_s": 0.07168494009238202, "kv_live_bytes": 462274560, "latency_s": 2.2953389170579612, "n_emitted": 32, "ok": true, "process_footprint_bytes": 28344320, "prompt_kind": "long", "t_relative_s": 222.6500195832923}, {"decode_latency_s": 2.6319101692788536, "kv_live_bytes": 462274560, "latency_s": 84.22491829190403, "n_emitted": 32, "ok": true, "process_footprint_bytes": 58425344, "prompt_kind": "short", "t_relative_s": 232.5751048750244}, {"decode_latency_s": 1.5243876965978416, "kv_live_bytes": 462274560, "latency_s": 48.78169579105452, "n_emitted": 32, "ok": true, "process_footprint_bytes": 58146816, "prompt_kind": "long", "t_relative_s": 322.84100687503815}, {"decode_latency_s": 0.12474652343371417, "kv_live_bytes": 462274560, "latency_s": 3.9938134998083115, "n_emitted": 32, "ok": true, "process_footprint_bytes": 30277632, "prompt_kind": "short", "t_relative_s": 379.8718310832046}, {"decode_latency_s": 0.07301290234318003, "kv_live_bytes": 462274560, "latency_s": 2.3377482080832124, "n_emitted": 32, "ok": true, "process_footprint_bytes": 57720832, "prompt_kind": "long", "t_relative_s": 388.27465179236606}, {"decode_latency_s": 0.0644523242226569, "kv_live_bytes": 462274560, "latency_s": 2.0634644581004977, "n_emitted": 32, "ok": true, "process_footprint_bytes": 54542336, "prompt_kind": "short", "t_relative_s": 398.39009783323854}, {"decode_latency_s": 0.3206847539113369, "kv_live_bytes": 462274560, "latency_s": 10.264617708977312, "n_emitted": 32, "ok": true, "process_footprint_bytes": 56082432, "prompt_kind": "long", "t_relative_s": 403.3951367922127}, {"decode_latency_s": 0.18872125518100802, "kv_live_bytes": 462274560, "latency_s": 6.040029584430158, "n_emitted": 32, "ok": true, "process_footprint_bytes": 39190528, "prompt_kind": "short", "t_relative_s": 419.06325933290645}, {"decode_latency_s": 0.06580676302837674, "kv_live_bytes": 462274560, "latency_s": 2.1071725417859852, "n_emitted": 32, "ok": true, "process_footprint_bytes": 59637760, "prompt_kind": "long", "t_relative_s": 430.4762354171835}, {"decode_latency_s": 0.24017046485096216, "kv_live_bytes": 462274560, "latency_s": 7.687336832750589, "n_emitted": 32, "ok": true, "process_footprint_bytes": 56754176, "prompt_kind": "short", "t_relative_s": 435.4812792921439}, {"decode_latency_s": 0.07155795834842138, "kv_live_bytes": 462274560, "latency_s": 2.2936204578727484, "n_emitted": 32, "ok": true, "process_footprint_bytes": 62603264, "prompt_kind": "long", "t_relative_s": 448.2242825832218}, {"decode_latency_s": 0.06366177997551858, "kv_live_bytes": 462274560, "latency_s": 2.0380771248601377, "n_emitted": 32, "ok": true, "process_footprint_bytes": 63799296, "prompt_kind": "short", "t_relative_s": 453.2293172501959}, {"decode_latency_s": 0.2601845377503196, "kv_live_bytes": 462274560, "latency_s": 8.328730124980211, "n_emitted": 32, "ok": true, "process_footprint_bytes": 55181312, "prompt_kind": "long", "t_relative_s": 458.2343935421668}, {"decode_latency_s": 0.06552486069267616, "kv_live_bytes": 462274560, "latency_s": 2.0978402919135988, "n_emitted": 32, "ok": true, "process_footprint_bytes": 62636032, "prompt_kind": "short", "t_relative_s": 473.40069587528706}, {"decode_latency_s": 0.06835086199862417, "kv_live_bytes": 462274560, "latency_s": 2.188219875097275, "n_emitted": 32, "ok": true, "process_footprint_bytes": 64012288, "prompt_kind": "long", "t_relative_s": 478.40577000007033}, {"decode_latency_s": 0.06286907158209942, "kv_live_bytes": 462274560, "latency_s": 2.0126317078247666, "n_emitted": 32, "ok": true, "process_footprint_bytes": 62914560, "prompt_kind": "short", "t_relative_s": 489.8329248330556}, {"decode_latency_s": 0.10900253646832425, "kv_live_bytes": 462274560, "latency_s": 3.489047249779105, "n_emitted": 32, "ok": true, "process_footprint_bytes": 58638336, "prompt_kind": "long", "t_relative_s": 494.8380233752541}, {"decode_latency_s": 0.06553762759722304, "kv_live_bytes": 462274560, "latency_s": 2.098435875028372, "n_emitted": 32, "ok": true, "process_footprint_bytes": 63455232, "prompt_kind": "short", "t_relative_s": 505.06951637519524}, {"decode_latency_s": 0.06605861718708184, "kv_live_bytes": 462274560, "latency_s": 2.114868250209838, "n_emitted": 32, "ok": true, "process_footprint_bytes": 64389120, "prompt_kind": "long", "t_relative_s": 510.0703265829943}, {"decode_latency_s": 0.07553266405011527, "kv_live_bytes": 462274560, "latency_s": 2.421180416829884, "n_emitted": 32, "ok": true, "process_footprint_bytes": 63619072, "prompt_kind": "short", "t_relative_s": 515.5636366251856}, {"decode_latency_s": 0.10975640753167681, "kv_live_bytes": 462274560, "latency_s": 3.513442291878164, "n_emitted": 32, "ok": true, "process_footprint_bytes": 57294848, "prompt_kind": "long", "t_relative_s": 520.5657405829988}, {"decode_latency_s": 0.06548752343223896, "kv_live_bytes": 462274560, "latency_s": 2.0968259163200855, "n_emitted": 32, "ok": true, "process_footprint_bytes": 60456960, "prompt_kind": "short", "t_relative_s": 531.8434088751674}, {"decode_latency_s": 0.10898987759719603, "kv_live_bytes": 462274560, "latency_s": 3.489520125091076, "n_emitted": 32, "ok": true, "process_footprint_bytes": 59326464, "prompt_kind": "long", "t_relative_s": 536.9728945000097}, {"decode_latency_s": 0.069050514313858, "kv_live_bytes": 462274560, "latency_s": 2.2117402497678995, "n_emitted": 32, "ok": true, "process_footprint_bytes": 63569920, "prompt_kind": "short", "t_relative_s": 545.129778875038}, {"decode_latency_s": 0.33017402472614776, "kv_live_bytes": 462274560, "latency_s": 10.56664541689679, "n_emitted": 32, "ok": true, "process_footprint_bytes": 34750464, "prompt_kind": "long", "t_relative_s": 550.1348357922398}, {"decode_latency_s": 0.0631501640600618, "kv_live_bytes": 462274560, "latency_s": 2.0221756659448147, "n_emitted": 32, "ok": true, "process_footprint_bytes": 62226432, "prompt_kind": "short", "t_relative_s": 567.10911016725}, {"decode_latency_s": 0.3369332591246348, "kv_live_bytes": 462274560, "latency_s": 10.782905833330005, "n_emitted": 32, "ok": true, "process_footprint_bytes": 32407552, "prompt_kind": "long", "t_relative_s": 572.1091809170321}, {"decode_latency_s": 0.3492901536228601, "kv_live_bytes": 462274560, "latency_s": 11.182629541959614, "n_emitted": 32, "ok": true, "process_footprint_bytes": 52183040, "prompt_kind": "short", "t_relative_s": 591.8658308750018}, {"decode_latency_s": 0.11856966925552115, "kv_live_bytes": 462274560, "latency_s": 3.7953444998711348, "n_emitted": 32, "ok": true, "process_footprint_bytes": 62488576, "prompt_kind": "long", "t_relative_s": 609.2900544581935}, {"decode_latency_s": 0.06353884634154383, "kv_live_bytes": 462274560, "latency_s": 2.035615958739072, "n_emitted": 32, "ok": true, "process_footprint_bytes": 63668224, "prompt_kind": "short", "t_relative_s": 623.6584696671925}, {"decode_latency_s": 1.5098980716138612, "kv_live_bytes": 462274560, "latency_s": 48.318434208165854, "n_emitted": 32, "ok": true, "process_footprint_bytes": 58392576, "prompt_kind": "long", "t_relative_s": 628.6685086670332}, {"decode_latency_s": 0.3939237994636642, "kv_live_bytes": 462274560, "latency_s": 12.60687604220584, "n_emitted": 32, "ok": true, "process_footprint_bytes": 38125568, "prompt_kind": "short", "t_relative_s": 683.3131812503561}, {"decode_latency_s": 0.12754465625039302, "kv_live_bytes": 462274560, "latency_s": 4.083868416026235, "n_emitted": 32, "ok": true, "process_footprint_bytes": 60669952, "prompt_kind": "long", "t_relative_s": 700.8209411669523}, {"decode_latency_s": 0.09734933462459594, "kv_live_bytes": 462274560, "latency_s": 3.1164666661061347, "n_emitted": 32, "ok": true, "process_footprint_bytes": 58245120, "prompt_kind": "short", "t_relative_s": 714.1765475003049}, {"decode_latency_s": 0.07417377083038446, "kv_live_bytes": 462274560, "latency_s": 2.375381208024919, "n_emitted": 32, "ok": true, "process_footprint_bytes": 62406656, "prompt_kind": "long", "t_relative_s": 722.0384831251577}, {"decode_latency_s": 0.526508941402426, "kv_live_bytes": 462274560, "latency_s": 16.849341417197138, "n_emitted": 32, "ok": true, "process_footprint_bytes": 58294272, "prompt_kind": "short", "t_relative_s": 727.0485808332451}, {"decode_latency_s": 0.0677816380339209, "kv_live_bytes": 462274560, "latency_s": 2.1703126253560185, "n_emitted": 32, "ok": true, "process_footprint_bytes": 61718528, "prompt_kind": "long", "t_relative_s": 748.0128453751095}, {"decode_latency_s": 0.06548524477693718, "kv_live_bytes": 462274560, "latency_s": 2.096369792241603, "n_emitted": 32, "ok": true, "process_footprint_bytes": 62865408, "prompt_kind": "short", "t_relative_s": 753.0228761672042}, {"decode_latency_s": 0.0662957070308039, "kv_live_bytes": 462274560, "latency_s": 2.122789291664958, "n_emitted": 32, "ok": true, "process_footprint_bytes": 55820288, "prompt_kind": "long", "t_relative_s": 758.0329328333028}, {"decode_latency_s": 1.4323664895928232, "kv_live_bytes": 462274560, "latency_s": 45.85464516701177, "n_emitted": 32, "ok": true, "process_footprint_bytes": 60178432, "prompt_kind": "short", "t_relative_s": 763.103163999971}, {"decode_latency_s": 0.20300411459174938, "kv_live_bytes": 462274560, "latency_s": 6.501160624902695, "n_emitted": 32, "ok": true, "process_footprint_bytes": 65388544, "prompt_kind": "long", "t_relative_s": 817.8108744169585}, {"decode_latency_s": 0.07539148307114374, "kv_live_bytes": 462274560, "latency_s": 2.4134672917425632, "n_emitted": 32, "ok": true, "process_footprint_bytes": 64487424, "prompt_kind": "short", "t_relative_s": 846.1153635829687}, {"decode_latency_s": 11.979533445308334, "kv_live_bytes": 462274560, "latency_s": 383.3526329579763, "n_emitted": 32, "ok": true, "process_footprint_bytes": 52494336, "prompt_kind": "long", "t_relative_s": 851.1254146252759}, {"decode_latency_s": 3.6457390859432053, "kv_live_bytes": 462274560, "latency_s": 116.68958675023168, "n_emitted": 32, "ok": true, "process_footprint_bytes": 61210624, "prompt_kind": "short", "t_relative_s": 1241.3207086250186}, {"decode_latency_s": 0.12497417446866166, "kv_live_bytes": 462274560, "latency_s": 4.002859834115952, "n_emitted": 32, "ok": true, "process_footprint_bytes": 63176704, "prompt_kind": "long", "t_relative_s": 1403.8526265830733}, {"decode_latency_s": 0.39175537499249913, "kv_live_bytes": 462274560, "latency_s": 12.538035833276808, "n_emitted": 32, "ok": true, "process_footprint_bytes": 63258624, "prompt_kind": "short", "t_relative_s": 1414.6154155833647}, {"decode_latency_s": 0.12239820053218864, "kv_live_bytes": 462274560, "latency_s": 3.920055208262056, "n_emitted": 32, "ok": true, "process_footprint_bytes": 62996480, "prompt_kind": "long", "t_relative_s": 1433.3145287921652}, {"decode_latency_s": 0.06380257030832581, "kv_live_bytes": 462274560, "latency_s": 2.0425695003941655, "n_emitted": 32, "ok": true, "process_footprint_bytes": 64045056, "prompt_kind": "short", "t_relative_s": 1445.099639333319}, {"decode_latency_s": 0.06976525258505717, "kv_live_bytes": 462274560, "latency_s": 2.2349761249497533, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66355200, "prompt_kind": "long", "t_relative_s": 1450.1097154170275}, {"decode_latency_s": 0.06197110157518182, "kv_live_bytes": 462274560, "latency_s": 1.9838502081111073, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66912256, "prompt_kind": "short", "t_relative_s": 1455.1197580420412}, {"decode_latency_s": 0.06964224218972959, "kv_live_bytes": 462274560, "latency_s": 2.2310369997285306, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67010560, "prompt_kind": "long", "t_relative_s": 1460.1297835833393}, {"decode_latency_s": 0.061490838532336056, "kv_live_bytes": 462274560, "latency_s": 1.9685160829685628, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67567616, "prompt_kind": "short", "t_relative_s": 1465.1398555422202}, {"decode_latency_s": 0.06939648046682123, "kv_live_bytes": 462274560, "latency_s": 2.222664291970432, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67682304, "prompt_kind": "long", "t_relative_s": 1470.1499116253108}, {"decode_latency_s": 0.06110750522930175, "kv_live_bytes": 462274560, "latency_s": 1.9563647089526057, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67895296, "prompt_kind": "short", "t_relative_s": 1475.159966125153}, {"decode_latency_s": 0.06878758331004065, "kv_live_bytes": 462274560, "latency_s": 2.2033472917973995, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67944448, "prompt_kind": "long", "t_relative_s": 1480.170044875238}, {"decode_latency_s": 0.06390035156800877, "kv_live_bytes": 462274560, "latency_s": 2.045735416933894, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67993600, "prompt_kind": "short", "t_relative_s": 1485.1801037923433}, {"decode_latency_s": 0.06744628775049932, "kv_live_bytes": 462274560, "latency_s": 2.1602202500216663, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67239936, "prompt_kind": "long", "t_relative_s": 1490.1901762080379}, {"decode_latency_s": 0.06029493099777028, "kv_live_bytes": 462274560, "latency_s": 1.9303322918713093, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68337664, "prompt_kind": "short", "t_relative_s": 1495.2002274999395}, {"decode_latency_s": 0.06809578387765214, "kv_live_bytes": 462274560, "latency_s": 2.18114379234612, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68403200, "prompt_kind": "long", "t_relative_s": 1500.210306667257}, {"decode_latency_s": 0.06417240365408361, "kv_live_bytes": 462274560, "latency_s": 2.0544218332506716, "n_emitted": 32, "ok": true, "process_footprint_bytes": 65519616, "prompt_kind": "short", "t_relative_s": 1505.2203742079437}, {"decode_latency_s": 0.062218934908742085, "kv_live_bytes": 462274560, "latency_s": 1.9919655830599368, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66912256, "prompt_kind": "long", "t_relative_s": 1510.2304633753374}, {"decode_latency_s": 0.060914585934369825, "kv_live_bytes": 462274560, "latency_s": 1.9500990421511233, "n_emitted": 32, "ok": true, "process_footprint_bytes": 64782336, "prompt_kind": "short", "t_relative_s": 1515.2405150830746}, {"decode_latency_s": 0.06787935027386993, "kv_live_bytes": 462274560, "latency_s": 2.1750666247680783, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67846144, "prompt_kind": "long", "t_relative_s": 1520.2505790423602}, {"decode_latency_s": 0.06037422006193083, "kv_live_bytes": 462274560, "latency_s": 1.932741791009903, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68468736, "prompt_kind": "short", "t_relative_s": 1525.260664250236}, {"decode_latency_s": 0.06774625390244182, "kv_live_bytes": 462274560, "latency_s": 2.1700409171171486, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68665344, "prompt_kind": "long", "t_relative_s": 1530.2707407921553}, {"decode_latency_s": 0.060579565120860934, "kv_live_bytes": 462274560, "latency_s": 1.9394515831954777, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68714496, "prompt_kind": "short", "t_relative_s": 1535.2807750003412}, {"decode_latency_s": 0.06833879426994827, "kv_live_bytes": 462274560, "latency_s": 2.1888357917778194, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68960256, "prompt_kind": "long", "t_relative_s": 1540.2908508749679}, {"decode_latency_s": 0.06286633203853853, "kv_live_bytes": 462274560, "latency_s": 2.012590333353728, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67272704, "prompt_kind": "short", "t_relative_s": 1545.3009446249343}, {"decode_latency_s": 0.06788635415432509, "kv_live_bytes": 462274560, "latency_s": 2.1742948330938816, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68763648, "prompt_kind": "long", "t_relative_s": 1550.305206167046}, {"decode_latency_s": 0.06042916928709019, "kv_live_bytes": 462274560, "latency_s": 1.934629333205521, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69222400, "prompt_kind": "short", "t_relative_s": 1555.3152527082711}, {"decode_latency_s": 0.06745064971619286, "kv_live_bytes": 462274560, "latency_s": 2.1609560842625797, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69550080, "prompt_kind": "long", "t_relative_s": 1560.325327792205}, {"decode_latency_s": 0.06055718100105878, "kv_live_bytes": 462274560, "latency_s": 1.9386349166743457, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69566464, "prompt_kind": "short", "t_relative_s": 1565.3354199579917}, {"decode_latency_s": 0.06850576824217569, "kv_live_bytes": 462274560, "latency_s": 2.194456750061363, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69074944, "prompt_kind": "long", "t_relative_s": 1570.3454619171098}, {"decode_latency_s": 0.060767669274355285, "kv_live_bytes": 462274560, "latency_s": 1.9454601253382862, "n_emitted": 32, "ok": true, "process_footprint_bytes": 60260352, "prompt_kind": "short", "t_relative_s": 1575.3555335421115}, {"decode_latency_s": 0.06772953906329349, "kv_live_bytes": 462274560, "latency_s": 2.169820624869317, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68927488, "prompt_kind": "long", "t_relative_s": 1580.365621500183}, {"decode_latency_s": 0.06059357163030654, "kv_live_bytes": 462274560, "latency_s": 1.939764834009111, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69550080, "prompt_kind": "short", "t_relative_s": 1585.3756862501614}, {"decode_latency_s": 0.06800447915156838, "kv_live_bytes": 462274560, "latency_s": 2.178376874886453, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69877760, "prompt_kind": "long", "t_relative_s": 1590.3857313329354}, {"decode_latency_s": 0.06058308725187089, "kv_live_bytes": 462274560, "latency_s": 1.9395621661096811, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69746688, "prompt_kind": "short", "t_relative_s": 1595.395796125289}, {"decode_latency_s": 0.06819628775701858, "kv_live_bytes": 462274560, "latency_s": 2.183902082964778, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70074368, "prompt_kind": "long", "t_relative_s": 1600.4058715002611}, {"decode_latency_s": 0.06073883202043362, "kv_live_bytes": 462274560, "latency_s": 1.9444553749635816, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70057984, "prompt_kind": "short", "t_relative_s": 1605.415924708359}, {"decode_latency_s": 0.060563320308574475, "kv_live_bytes": 462274560, "latency_s": 1.9392815409228206, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69107712, "prompt_kind": "long", "t_relative_s": 1610.4260107502341}, {"decode_latency_s": 0.06670996484172065, "kv_live_bytes": 462274560, "latency_s": 2.1362737081944942, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69730304, "prompt_kind": "short", "t_relative_s": 1615.4361197082326}, {"decode_latency_s": 0.06796755990944803, "kv_live_bytes": 462274560, "latency_s": 2.1773668327368796, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70139904, "prompt_kind": "long", "t_relative_s": 1620.4445876250975}, {"decode_latency_s": 0.060543020837940276, "kv_live_bytes": 462274560, "latency_s": 1.9382731658406556, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70516736, "prompt_kind": "short", "t_relative_s": 1625.4546589581296}, {"decode_latency_s": 0.0680649531132076, "kv_live_bytes": 462274560, "latency_s": 2.1795728341676295, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70582272, "prompt_kind": "long", "t_relative_s": 1630.4647497502156}, {"decode_latency_s": 0.061579805973451585, "kv_live_bytes": 462274560, "latency_s": 1.971954165957868, "n_emitted": 32, "ok": true, "process_footprint_bytes": 61931520, "prompt_kind": "short", "t_relative_s": 1635.4747808333486}, {"decode_latency_s": 0.06839418750314508, "kv_live_bytes": 462274560, "latency_s": 2.191048333887011, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70270976, "prompt_kind": "long", "t_relative_s": 1640.4848520001397}, {"decode_latency_s": 0.06030623828701209, "kv_live_bytes": 462274560, "latency_s": 1.9306882498785853, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70778880, "prompt_kind": "short", "t_relative_s": 1645.4898629579693}, {"decode_latency_s": 0.06820715362846386, "kv_live_bytes": 462274560, "latency_s": 2.1842903341166675, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70877184, "prompt_kind": "long", "t_relative_s": 1650.499942583032}, {"decode_latency_s": 0.060544937499798834, "kv_live_bytes": 462274560, "latency_s": 1.9382719998247921, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70942720, "prompt_kind": "short", "t_relative_s": 1655.5099880420603}, {"decode_latency_s": 0.06755092968523968, "kv_live_bytes": 462274560, "latency_s": 2.1640255418606102, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71024640, "prompt_kind": "long", "t_relative_s": 1660.520058083348}, {"decode_latency_s": 0.06069472525268793, "kv_live_bytes": 462274560, "latency_s": 1.9430142911151052, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71663616, "prompt_kind": "short", "t_relative_s": 1665.5301331672817}, {"decode_latency_s": 0.06778555859636981, "kv_live_bytes": 462274560, "latency_s": 2.171236792113632, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70844416, "prompt_kind": "long", "t_relative_s": 1670.54019012535}, {"decode_latency_s": 0.060307847656076774, "kv_live_bytes": 462274560, "latency_s": 1.9307794170454144, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70762496, "prompt_kind": "short", "t_relative_s": 1675.5502710831352}, {"decode_latency_s": 0.0674680833471939, "kv_live_bytes": 462274560, "latency_s": 2.161213458981365, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71467008, "prompt_kind": "long", "t_relative_s": 1680.5603376249783}, {"decode_latency_s": 0.06093237368622795, "kv_live_bytes": 462274560, "latency_s": 1.9506503329612315, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71499776, "prompt_kind": "short", "t_relative_s": 1685.570397167001}, {"decode_latency_s": 0.06794419010111596, "kv_live_bytes": 462274560, "latency_s": 2.175957459025085, "n_emitted": 32, "ok": true, "process_footprint_bytes": 64995328, "prompt_kind": "long", "t_relative_s": 1690.580472792033}, {"decode_latency_s": 0.06097813934320584, "kv_live_bytes": 462274560, "latency_s": 1.9522199998609722, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67584000, "prompt_kind": "short", "t_relative_s": 1695.5905627501197}, {"decode_latency_s": 0.06792411590868142, "kv_live_bytes": 462274560, "latency_s": 2.175510459113866, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71090176, "prompt_kind": "long", "t_relative_s": 1700.600634417031}, {"decode_latency_s": 0.06055870311683975, "kv_live_bytes": 462274560, "latency_s": 1.9388122092932463, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71712768, "prompt_kind": "short", "t_relative_s": 1705.610688582994}, {"decode_latency_s": 0.06800493490300141, "kv_live_bytes": 462274560, "latency_s": 2.1785297081805766, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71811072, "prompt_kind": "long", "t_relative_s": 1710.6207912499085}, {"decode_latency_s": 0.06071289842657279, "kv_live_bytes": 462274560, "latency_s": 1.9435736248269677, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71860224, "prompt_kind": "short", "t_relative_s": 1715.6308932919055}, {"decode_latency_s": 0.06834724350483157, "kv_live_bytes": 462274560, "latency_s": 2.1892688339576125, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71811072, "prompt_kind": "long", "t_relative_s": 1720.6371674579568}, {"decode_latency_s": 0.06109645703691058, "kv_live_bytes": 462274560, "latency_s": 1.9560174997895956, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72040448, "prompt_kind": "short", "t_relative_s": 1725.6472457922064}, {"decode_latency_s": 0.06814573047449812, "kv_live_bytes": 462274560, "latency_s": 2.183418999891728, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71925760, "prompt_kind": "long", "t_relative_s": 1730.6572999581695}, {"decode_latency_s": 0.06035154296841938, "kv_live_bytes": 462274560, "latency_s": 1.9320242921821773, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72515584, "prompt_kind": "short", "t_relative_s": 1735.6673743752763}, {"decode_latency_s": 0.0682242070324719, "kv_live_bytes": 462274560, "latency_s": 2.1847572079859674, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69500928, "prompt_kind": "long", "t_relative_s": 1740.677415125072}, {"decode_latency_s": 0.06051239580847323, "kv_live_bytes": 462274560, "latency_s": 1.9373468337580562, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71041024, "prompt_kind": "short", "t_relative_s": 1745.6874622502364}, {"decode_latency_s": 0.06788441014941782, "kv_live_bytes": 462274560, "latency_s": 2.1747739580459893, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71843840, "prompt_kind": "long", "t_relative_s": 1750.697519833222}, {"decode_latency_s": 0.06088101952627767, "kv_live_bytes": 462274560, "latency_s": 1.9491099170409143, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68206592, "prompt_kind": "short", "t_relative_s": 1755.7076048329473}, {"decode_latency_s": 0.06775340234162286, "kv_live_bytes": 462274560, "latency_s": 2.170519624836743, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71516160, "prompt_kind": "long", "t_relative_s": 1760.7176568750292}, {"decode_latency_s": 0.06677685416070744, "kv_live_bytes": 462274560, "latency_s": 2.138525791000575, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72089600, "prompt_kind": "short", "t_relative_s": 1765.7277350421064}, {"decode_latency_s": 0.06794726171938237, "kv_live_bytes": 462274560, "latency_s": 2.176390249747783, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72351744, "prompt_kind": "long", "t_relative_s": 1770.737786917016}, {"decode_latency_s": 0.060487596347229555, "kv_live_bytes": 462274560, "latency_s": 1.9365250002592802, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72400896, "prompt_kind": "short", "t_relative_s": 1775.7478611250408}, {"decode_latency_s": 0.06899679428897798, "kv_live_bytes": 462274560, "latency_s": 2.2097465000115335, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71516160, "prompt_kind": "long", "t_relative_s": 1780.7579247499816}, {"decode_latency_s": 0.0615046588354744, "kv_live_bytes": 462274560, "latency_s": 1.969066542107612, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71614464, "prompt_kind": "short", "t_relative_s": 1785.7636851249263}, {"decode_latency_s": 0.06056658203306142, "kv_live_bytes": 462274560, "latency_s": 1.9391940422356129, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71860224, "prompt_kind": "long", "t_relative_s": 1790.7737428331748}, {"decode_latency_s": 0.06044264194497373, "kv_live_bytes": 462274560, "latency_s": 1.9350872919894755, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71794688, "prompt_kind": "short", "t_relative_s": 1795.7838294170797}, {"decode_latency_s": 0.06797263280896004, "kv_live_bytes": 462274560, "latency_s": 2.1771807922050357, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72515584, "prompt_kind": "long", "t_relative_s": 1800.7939154170454}, {"decode_latency_s": 0.060891898450790904, "kv_live_bytes": 462274560, "latency_s": 1.9494722918607295, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72613888, "prompt_kind": "short", "t_relative_s": 1805.8039679583162}, {"decode_latency_s": 0.06778315624978859, "kv_live_bytes": 462274560, "latency_s": 2.171589750330895, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72663040, "prompt_kind": "long", "t_relative_s": 1810.8140465002507}, {"decode_latency_s": 0.06323009896732401, "kv_live_bytes": 462274560, "latency_s": 2.0251695830374956, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69238784, "prompt_kind": "short", "t_relative_s": 1815.8241708329879}, {"decode_latency_s": 0.06745718487945851, "kv_live_bytes": 462274560, "latency_s": 2.161487374920398, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71778304, "prompt_kind": "long", "t_relative_s": 1820.834213042166}, {"decode_latency_s": 0.06025869531731587, "kv_live_bytes": 462274560, "latency_s": 1.9291422087699175, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72433664, "prompt_kind": "short", "t_relative_s": 1825.844289583154}, {"decode_latency_s": 0.06785117837716825, "kv_live_bytes": 462274560, "latency_s": 2.1732110837474465, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72613888, "prompt_kind": "long", "t_relative_s": 1830.8543591252528}, {"decode_latency_s": 0.060498847655253485, "kv_live_bytes": 462274560, "latency_s": 1.9382116249762475, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72761344, "prompt_kind": "short", "t_relative_s": 1835.8644121252}, {"decode_latency_s": 0.06807242319337092, "kv_live_bytes": 462274560, "latency_s": 2.1806609169580042, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72794112, "prompt_kind": "long", "t_relative_s": 1840.8744786670431}, {"decode_latency_s": 0.06792516278801486, "kv_live_bytes": 462274560, "latency_s": 2.175082332920283, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70090752, "prompt_kind": "short", "t_relative_s": 1845.884611458052}, {"decode_latency_s": 0.06083390624553431, "kv_live_bytes": 462274560, "latency_s": 1.9480622918345034, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71630848, "prompt_kind": "long", "t_relative_s": 1850.8947176672518}, {"decode_latency_s": 0.06534662758349441, "kv_live_bytes": 462274560, "latency_s": 2.092951124999672, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72237056, "prompt_kind": "short", "t_relative_s": 1855.9017727919854}, {"decode_latency_s": 0.06761543618631549, "kv_live_bytes": 462274560, "latency_s": 2.166257250122726, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72794112, "prompt_kind": "long", "t_relative_s": 1860.9118597921915}, {"decode_latency_s": 0.06725896224088501, "kv_live_bytes": 462274560, "latency_s": 2.1541982502676547, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71647232, "prompt_kind": "short", "t_relative_s": 1865.921930667013}, {"decode_latency_s": 0.06818767187360208, "kv_live_bytes": 462274560, "latency_s": 2.1843582089059055, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71974912, "prompt_kind": "long", "t_relative_s": 1870.93198016705}, {"decode_latency_s": 0.06104045834217686, "kv_live_bytes": 462274560, "latency_s": 1.9542069998569787, "n_emitted": 32, "ok": true, "process_footprint_bytes": 61702144, "prompt_kind": "short", "t_relative_s": 1875.9420476253144}, {"decode_latency_s": 0.06788143618905451, "kv_live_bytes": 462274560, "latency_s": 2.175105709116906, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71843840, "prompt_kind": "long", "t_relative_s": 1880.952098458074}, {"decode_latency_s": 0.06040409894194454, "kv_live_bytes": 462274560, "latency_s": 1.9336960832588375, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72499200, "prompt_kind": "short", "t_relative_s": 1885.9621815420687}, {"decode_latency_s": 0.0684048294351669, "kv_live_bytes": 462274560, "latency_s": 2.190417249687016, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72351744, "prompt_kind": "long", "t_relative_s": 1890.9722335832193}, {"decode_latency_s": 0.060702587259584107, "kv_live_bytes": 462274560, "latency_s": 1.9434289578348398, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72548352, "prompt_kind": "short", "t_relative_s": 1895.9822907499038}, {"decode_latency_s": 0.06821351694816258, "kv_live_bytes": 462274560, "latency_s": 2.1845196252688766, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72564736, "prompt_kind": "long", "t_relative_s": 1900.9923702920787}, {"decode_latency_s": 0.06760286977805663, "kv_live_bytes": 462274560, "latency_s": 2.1647273330017924, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70762496, "prompt_kind": "short", "t_relative_s": 1906.002415667288}, {"decode_latency_s": 0.06769464453100227, "kv_live_bytes": 462274560, "latency_s": 2.1683454159647226, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71909376, "prompt_kind": "long", "t_relative_s": 1911.012500333134}, {"decode_latency_s": 0.06069512109388597, "kv_live_bytes": 462274560, "latency_s": 1.943179083056748, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72679424, "prompt_kind": "short", "t_relative_s": 1916.022547250148}, {"decode_latency_s": 0.06782105467573274, "kv_live_bytes": 462274560, "latency_s": 2.1730297077447176, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72794112, "prompt_kind": "long", "t_relative_s": 1921.032626207918}, {"decode_latency_s": 0.06091237367945723, "kv_live_bytes": 462274560, "latency_s": 1.9499806673265994, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72859648, "prompt_kind": "short", "t_relative_s": 1926.0427029170096}, {"decode_latency_s": 0.06789897002454381, "kv_live_bytes": 462274560, "latency_s": 2.174862415995449, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "long", "t_relative_s": 1931.0527466251515}, {"decode_latency_s": 0.06096967056510039, "kv_live_bytes": 462274560, "latency_s": 1.9519523340277374, "n_emitted": 32, "ok": true, "process_footprint_bytes": 58834944, "prompt_kind": "short", "t_relative_s": 1936.0628281673416}, {"decode_latency_s": 0.06846514712378848, "kv_live_bytes": 462274560, "latency_s": 2.1933315419591963, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70303744, "prompt_kind": "long", "t_relative_s": 1941.0729217920452}, {"decode_latency_s": 0.06028796485043131, "kv_live_bytes": 462274560, "latency_s": 1.9300916250795126, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72630272, "prompt_kind": "short", "t_relative_s": 1946.0829816251062}, {"decode_latency_s": 0.06801981640455779, "kv_live_bytes": 462274560, "latency_s": 2.178756915964186, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72728576, "prompt_kind": "long", "t_relative_s": 1951.0930411671288}, {"decode_latency_s": 0.06087672393186949, "kv_live_bytes": 462274560, "latency_s": 1.9488792079500854, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72761344, "prompt_kind": "short", "t_relative_s": 1956.1031152079813}, {"decode_latency_s": 0.06079816406418104, "kv_live_bytes": 462274560, "latency_s": 1.9465238749980927, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72843264, "prompt_kind": "long", "t_relative_s": 1961.1131708333269}, {"decode_latency_s": 0.06047411328472663, "kv_live_bytes": 462274560, "latency_s": 1.935925125144422, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72843264, "prompt_kind": "short", "t_relative_s": 1966.123244999908}, {"decode_latency_s": 0.06874520440760534, "kv_live_bytes": 462274560, "latency_s": 2.202666707802564, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70254592, "prompt_kind": "long", "t_relative_s": 1971.1333146672696}, {"decode_latency_s": 0.06282107293372974, "kv_live_bytes": 462274560, "latency_s": 2.0113020832650363, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69730304, "prompt_kind": "short", "t_relative_s": 1976.1434309580363}, {"decode_latency_s": 0.06815806639497168, "kv_live_bytes": 462274560, "latency_s": 2.183096833061427, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71843840, "prompt_kind": "long", "t_relative_s": 1981.1535049169324}, {"decode_latency_s": 0.06067771225934848, "kv_live_bytes": 462274560, "latency_s": 1.9425205830484629, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71909376, "prompt_kind": "short", "t_relative_s": 1986.1635585422628}, {"decode_latency_s": 0.06794416147749871, "kv_live_bytes": 462274560, "latency_s": 2.176565082743764, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71991296, "prompt_kind": "long", "t_relative_s": 1991.1736097079702}, {"decode_latency_s": 0.06296080209722277, "kv_live_bytes": 462274560, "latency_s": 2.015626832842827, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71417856, "prompt_kind": "short", "t_relative_s": 1996.1837067082524}, {"decode_latency_s": 0.06806397525360808, "kv_live_bytes": 462274560, "latency_s": 2.180029584094882, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72105984, "prompt_kind": "long", "t_relative_s": 2001.193786750082}, {"decode_latency_s": 0.060763690096791834, "kv_live_bytes": 462274560, "latency_s": 1.9452970419079065, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "short", "t_relative_s": 2006.1981925829314}, {"decode_latency_s": 0.06794467056170106, "kv_live_bytes": 462274560, "latency_s": 2.1764059588313103, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72695808, "prompt_kind": "long", "t_relative_s": 2011.208236375358}, {"decode_latency_s": 0.0606741549418075, "kv_live_bytes": 462274560, "latency_s": 1.9423540839925408, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72777728, "prompt_kind": "short", "t_relative_s": 2016.2183205001056}, {"decode_latency_s": 0.06814986196695827, "kv_live_bytes": 462274560, "latency_s": 2.182687332853675, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72056832, "prompt_kind": "long", "t_relative_s": 2021.2283797082491}, {"decode_latency_s": 0.06042550000711344, "kv_live_bytes": 462274560, "latency_s": 1.9345734999515116, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70746112, "prompt_kind": "short", "t_relative_s": 2026.2384292921051}, {"decode_latency_s": 0.0679433801851701, "kv_live_bytes": 462274560, "latency_s": 2.1759172920137644, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72187904, "prompt_kind": "long", "t_relative_s": 2031.248514083214}, {"decode_latency_s": 0.060490044284961186, "kv_live_bytes": 462274560, "latency_s": 1.936506707686931, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72712192, "prompt_kind": "short", "t_relative_s": 2036.2585586672649}, {"decode_latency_s": 0.06038169794192072, "kv_live_bytes": 462274560, "latency_s": 1.9334554160013795, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72728576, "prompt_kind": "long", "t_relative_s": 2041.2686271253042}, {"decode_latency_s": 0.060685393211315386, "kv_live_bytes": 462274560, "latency_s": 1.9427237082272768, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72810496, "prompt_kind": "short", "t_relative_s": 2046.2737797079608}, {"decode_latency_s": 0.06818147006561048, "kv_live_bytes": 462274560, "latency_s": 2.1838472499512136, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71974912, "prompt_kind": "long", "t_relative_s": 2051.283832750283}, {"decode_latency_s": 0.06098677216505166, "kv_live_bytes": 462274560, "latency_s": 1.9525262918323278, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71221248, "prompt_kind": "short", "t_relative_s": 2056.293911250308}, {"decode_latency_s": 0.06796326172479894, "kv_live_bytes": 462274560, "latency_s": 2.1764466669410467, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72024064, "prompt_kind": "long", "t_relative_s": 2061.3039859579876}, {"decode_latency_s": 0.06051109507097863, "kv_live_bytes": 462274560, "latency_s": 1.9371712910942733, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72581120, "prompt_kind": "short", "t_relative_s": 2066.31403445825}, {"decode_latency_s": 0.06778459636552725, "kv_live_bytes": 462274560, "latency_s": 2.1727698747999966, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72630272, "prompt_kind": "long", "t_relative_s": 2071.324111542199}, {"decode_latency_s": 0.06076252993079834, "kv_live_bytes": 462274560, "latency_s": 1.945192875340581, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72777728, "prompt_kind": "short", "t_relative_s": 2076.3342170421965}, {"decode_latency_s": 0.06795046615297906, "kv_live_bytes": 462274560, "latency_s": 2.1768742501735687, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72859648, "prompt_kind": "long", "t_relative_s": 2081.3442560001276}, {"decode_latency_s": 0.06180050912371371, "kv_live_bytes": 462274560, "latency_s": 1.9788226671516895, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71417856, "prompt_kind": "short", "t_relative_s": 2086.3543121670373}, {"decode_latency_s": 0.06815846743120346, "kv_live_bytes": 462274560, "latency_s": 2.1828344156965613, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69173248, "prompt_kind": "long", "t_relative_s": 2091.364367207978}, {"decode_latency_s": 0.06649683984869625, "kv_live_bytes": 462274560, "latency_s": 2.1296003330498934, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71745536, "prompt_kind": "short", "t_relative_s": 2096.37444412522}, {"decode_latency_s": 0.06858281508903019, "kv_live_bytes": 462274560, "latency_s": 2.1968507920391858, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "long", "t_relative_s": 2101.384518583305}, {"decode_latency_s": 0.060613234381889924, "kv_live_bytes": 462274560, "latency_s": 1.9404157921671867, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72761344, "prompt_kind": "short", "t_relative_s": 2106.3946230830625}, {"decode_latency_s": 0.06856772786704823, "kv_live_bytes": 462274560, "latency_s": 2.196238291915506, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72122368, "prompt_kind": "long", "t_relative_s": 2111.4046715423465}, {"decode_latency_s": 0.06589160936709959, "kv_live_bytes": 462274560, "latency_s": 2.1098048333078623, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71106560, "prompt_kind": "short", "t_relative_s": 2116.414737583138}, {"decode_latency_s": 0.06837312108837068, "kv_live_bytes": 462274560, "latency_s": 2.1903833341784775, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72040448, "prompt_kind": "long", "t_relative_s": 2121.424785000272}, {"decode_latency_s": 0.060407764322008006, "kv_live_bytes": 462274560, "latency_s": 1.933817416895181, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "short", "t_relative_s": 2126.434856292326}, {"decode_latency_s": 0.06778889062115923, "kv_live_bytes": 462274560, "latency_s": 2.1713567501865327, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72564736, "prompt_kind": "long", "t_relative_s": 2131.4448943752795}, {"decode_latency_s": 0.06047181382018607, "kv_live_bytes": 462274560, "latency_s": 1.9360209172591567, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72269824, "prompt_kind": "short", "t_relative_s": 2136.454947750084}, {"decode_latency_s": 0.06956989715399686, "kv_live_bytes": 462274560, "latency_s": 2.228588374797255, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71778304, "prompt_kind": "long", "t_relative_s": 2141.4650292503648}, {"decode_latency_s": 0.060421160160331056, "kv_live_bytes": 462274560, "latency_s": 1.9343499997630715, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71663616, "prompt_kind": "short", "t_relative_s": 2146.4751159171574}, {"decode_latency_s": 0.06710695312358439, "kv_live_bytes": 462274560, "latency_s": 2.149330750107765, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71712768, "prompt_kind": "long", "t_relative_s": 2151.485223833006}, {"decode_latency_s": 0.06071485416032374, "kv_live_bytes": 462274560, "latency_s": 1.9437371250241995, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67567616, "prompt_kind": "short", "t_relative_s": 2156.4952562502585}, {"decode_latency_s": 0.06763348959793802, "kv_live_bytes": 462274560, "latency_s": 2.1670751250348985, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70303744, "prompt_kind": "long", "t_relative_s": 2161.5052988333628}, {"decode_latency_s": 0.06080800521885976, "kv_live_bytes": 462274560, "latency_s": 1.9468324580229819, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70877184, "prompt_kind": "short", "t_relative_s": 2166.515368500259}, {"decode_latency_s": 0.0609132122626761, "kv_live_bytes": 462274560, "latency_s": 1.9501121249049902, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71696384, "prompt_kind": "long", "t_relative_s": 2171.52543991711}, {"decode_latency_s": 0.06661226562573574, "kv_live_bytes": 462274560, "latency_s": 2.1333864997141063, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71286784, "prompt_kind": "short", "t_relative_s": 2176.535493250005}, {"decode_latency_s": 0.0679944010626059, "kv_live_bytes": 462274560, "latency_s": 2.1779697500169277, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72187904, "prompt_kind": "long", "t_relative_s": 2181.5455449582078}, {"decode_latency_s": 0.06032263413362671, "kv_live_bytes": 462274560, "latency_s": 1.93118058424443, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72777728, "prompt_kind": "short", "t_relative_s": 2186.555616958067}, {"decode_latency_s": 0.06801215757150203, "kv_live_bytes": 462274560, "latency_s": 2.177969792392105, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72237056, "prompt_kind": "long", "t_relative_s": 2191.565686458256}, {"decode_latency_s": 0.06060220312792808, "kv_live_bytes": 462274560, "latency_s": 1.9401437500491738, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71778304, "prompt_kind": "short", "t_relative_s": 2196.5757410419174}, {"decode_latency_s": 0.06882680469425395, "kv_live_bytes": 462274560, "latency_s": 2.2047631251625717, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72417280, "prompt_kind": "long", "t_relative_s": 2201.58581583295}, {"decode_latency_s": 0.06200250258552842, "kv_live_bytes": 462274560, "latency_s": 1.9850927917286754, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68714496, "prompt_kind": "short", "t_relative_s": 2206.5958785829134}, {"decode_latency_s": 0.06829261197708547, "kv_live_bytes": 462274560, "latency_s": 2.187285208143294, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71761920, "prompt_kind": "long", "t_relative_s": 2211.606047499925}, {"decode_latency_s": 0.0604536198079586, "kv_live_bytes": 462274560, "latency_s": 1.93554054107517, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72417280, "prompt_kind": "short", "t_relative_s": 2216.616096457932}, {"decode_latency_s": 0.06783333983912598, "kv_live_bytes": 462274560, "latency_s": 2.1729906662367284, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73138176, "prompt_kind": "long", "t_relative_s": 2221.6261602500454}, {"decode_latency_s": 0.060807201807620004, "kv_live_bytes": 462274560, "latency_s": 1.9466360001824796, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73154560, "prompt_kind": "short", "t_relative_s": 2226.6362675833516}, {"decode_latency_s": 0.06898434375762008, "kv_live_bytes": 462274560, "latency_s": 2.209586290642619, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72630272, "prompt_kind": "long", "t_relative_s": 2231.646324167028}, {"decode_latency_s": 0.0674438724090578, "kv_live_bytes": 462274560, "latency_s": 2.160233417060226, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71483392, "prompt_kind": "short", "t_relative_s": 2236.6563998330384}, {"decode_latency_s": 0.06075381771370303, "kv_live_bytes": 462274560, "latency_s": 1.9450717498548329, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72318976, "prompt_kind": "long", "t_relative_s": 2241.666467042174}, {"decode_latency_s": 0.060531139344675466, "kv_live_bytes": 462274560, "latency_s": 1.9377922089770436, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72892416, "prompt_kind": "short", "t_relative_s": 2246.676550167147}, {"decode_latency_s": 0.06806225390755571, "kv_live_bytes": 462274560, "latency_s": 2.1801820411346853, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73039872, "prompt_kind": "long", "t_relative_s": 2251.6866057501175}, {"decode_latency_s": 0.06067408721719403, "kv_live_bytes": 462274560, "latency_s": 1.9425032916478813, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73138176, "prompt_kind": "short", "t_relative_s": 2256.6966630001552}, {"decode_latency_s": 0.06847732422465924, "kv_live_bytes": 462274560, "latency_s": 2.1930442498996854, "n_emitted": 32, "ok": true, "process_footprint_bytes": 51445760, "prompt_kind": "long", "t_relative_s": 2261.7067375420593}, {"decode_latency_s": 0.06052622133574914, "kv_live_bytes": 462274560, "latency_s": 1.937839874997735, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71712768, "prompt_kind": "short", "t_relative_s": 2266.716984833125}, {"decode_latency_s": 0.06766675389371812, "kv_live_bytes": 462274560, "latency_s": 2.1673479578457773, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71974912, "prompt_kind": "long", "t_relative_s": 2271.727052875329}, {"decode_latency_s": 0.06026338801893871, "kv_live_bytes": 462274560, "latency_s": 1.92940125009045, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72515584, "prompt_kind": "short", "t_relative_s": 2276.7371020829305}, {"decode_latency_s": 0.06756020702596288, "kv_live_bytes": 462274560, "latency_s": 2.1642151246778667, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73089024, "prompt_kind": "long", "t_relative_s": 2281.747189000249}, {"decode_latency_s": 0.06087866016605403, "kv_live_bytes": 462274560, "latency_s": 1.9489317922852933, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73302016, "prompt_kind": "short", "t_relative_s": 2286.757259833161}, {"decode_latency_s": 0.06846426299307495, "kv_live_bytes": 462274560, "latency_s": 2.1930574579164386, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73138176, "prompt_kind": "long", "t_relative_s": 2291.7673141672276}, {"decode_latency_s": 0.06319523956335615, "kv_live_bytes": 462274560, "latency_s": 2.02318162471056, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71303168, "prompt_kind": "short", "t_relative_s": 2296.777416958008}, {"decode_latency_s": 0.06788892447366379, "kv_live_bytes": 462274560, "latency_s": 2.174705000128597, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72400896, "prompt_kind": "long", "t_relative_s": 2301.78749662498}, {"decode_latency_s": 0.06058566666615661, "kv_live_bytes": 462274560, "latency_s": 1.9395244577899575, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72990720, "prompt_kind": "short", "t_relative_s": 2306.7975960420445}, {"decode_latency_s": 0.06036817577842157, "kv_live_bytes": 462274560, "latency_s": 1.9330954174511135, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73023488, "prompt_kind": "long", "t_relative_s": 2311.807639125269}, {"decode_latency_s": 0.0618202447949443, "kv_live_bytes": 462274560, "latency_s": 1.979215290863067, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73039872, "prompt_kind": "short", "t_relative_s": 2316.817718292121}, {"decode_latency_s": 0.06188895311788656, "kv_live_bytes": 462274560, "latency_s": 1.9813696253113449, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72384512, "prompt_kind": "long", "t_relative_s": 2321.827795207966}, {"decode_latency_s": 0.060985196629189886, "kv_live_bytes": 462274560, "latency_s": 1.952390959020704, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72040448, "prompt_kind": "short", "t_relative_s": 2326.8378718751483}, {"decode_latency_s": 0.06791804425301962, "kv_live_bytes": 462274560, "latency_s": 2.1755422088317573, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72433664, "prompt_kind": "long", "t_relative_s": 2331.8479261249304}, {"decode_latency_s": 0.06060562368656974, "kv_live_bytes": 462274560, "latency_s": 1.9401909578591585, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71958528, "prompt_kind": "short", "t_relative_s": 2336.8580053751357}, {"decode_latency_s": 0.06805175261979457, "kv_live_bytes": 462274560, "latency_s": 2.179639416281134, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73023488, "prompt_kind": "long", "t_relative_s": 2341.8680610000156}, {"decode_latency_s": 0.060505798159283586, "kv_live_bytes": 462274560, "latency_s": 1.9371342500671744, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72925184, "prompt_kind": "short", "t_relative_s": 2346.878145792056}, {"decode_latency_s": 0.0678695729438914, "kv_live_bytes": 462274560, "latency_s": 2.1736599169671535, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73056256, "prompt_kind": "long", "t_relative_s": 2351.8882205830887}, {"decode_latency_s": 0.06760681640298571, "kv_live_bytes": 462274560, "latency_s": 2.1651767501607537, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71286784, "prompt_kind": "short", "t_relative_s": 2356.8982814173214}, {"decode_latency_s": 0.06789383331488352, "kv_live_bytes": 462274560, "latency_s": 2.175034458283335, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72466432, "prompt_kind": "long", "t_relative_s": 2361.9083653753623}, {"decode_latency_s": 0.0608592734351987, "kv_live_bytes": 462274560, "latency_s": 1.9482890837825835, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73007104, "prompt_kind": "short", "t_relative_s": 2366.918480833061}, {"decode_latency_s": 0.06807060027495027, "kv_live_bytes": 462274560, "latency_s": 2.180694750044495, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73089024, "prompt_kind": "long", "t_relative_s": 2371.9285275419243}, {"decode_latency_s": 0.06041717187326867, "kv_live_bytes": 462274560, "latency_s": 1.934191457927227, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73187328, "prompt_kind": "short", "t_relative_s": 2376.938597917091}, {"decode_latency_s": 0.0684563333488768, "kv_live_bytes": 462274560, "latency_s": 2.192115291953087, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71925760, "prompt_kind": "long", "t_relative_s": 2381.9486948330887}, {"decode_latency_s": 0.06492669403087348, "kv_live_bytes": 462274560, "latency_s": 2.079634541179985, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70696960, "prompt_kind": "short", "t_relative_s": 2386.950455875136}, {"decode_latency_s": 0.0680259987275349, "kv_live_bytes": 462274560, "latency_s": 2.178735540714115, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72073216, "prompt_kind": "long", "t_relative_s": 2391.960510833189}, {"decode_latency_s": 0.0606039361882722, "kv_live_bytes": 462274560, "latency_s": 1.9402524158358574, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71532544, "prompt_kind": "short", "t_relative_s": 2396.9705729172565}, {"decode_latency_s": 0.06514013670675922, "kv_live_bytes": 462274560, "latency_s": 2.0858104578219354, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67305472, "prompt_kind": "long", "t_relative_s": 2401.9806201253086}, {"decode_latency_s": 0.060837449214886874, "kv_live_bytes": 462274560, "latency_s": 1.947667459025979, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69779456, "prompt_kind": "short", "t_relative_s": 2406.990701749921}, {"decode_latency_s": 0.06855314194399398, "kv_live_bytes": 462274560, "latency_s": 2.1951927919872105, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70713344, "prompt_kind": "long", "t_relative_s": 2412.000752333086}, {"decode_latency_s": 0.06303739713621326, "kv_live_bytes": 462274560, "latency_s": 2.018073834013194, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70254592, "prompt_kind": "short", "t_relative_s": 2417.010814375244}, {"decode_latency_s": 0.06043339581810869, "kv_live_bytes": 462274560, "latency_s": 1.9350458751432598, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69566464, "prompt_kind": "long", "t_relative_s": 2422.0208647502586}, {"decode_latency_s": 0.06052679818822071, "kv_live_bytes": 462274560, "latency_s": 1.9376516249030828, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70189056, "prompt_kind": "short", "t_relative_s": 2427.030937250238}, {"decode_latency_s": 0.06814521484193392, "kv_live_bytes": 462274560, "latency_s": 2.18235683394596, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70221824, "prompt_kind": "long", "t_relative_s": 2432.041015041992}, {"decode_latency_s": 0.0606170560058672, "kv_live_bytes": 462274560, "latency_s": 1.9407069589942694, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70270976, "prompt_kind": "short", "t_relative_s": 2437.0510620833375}, {"decode_latency_s": 0.06817932812555227, "kv_live_bytes": 462274560, "latency_s": 2.1835262500680983, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71008256, "prompt_kind": "long", "t_relative_s": 2442.0611350000836}, {"decode_latency_s": 0.061429859371855855, "kv_live_bytes": 462274560, "latency_s": 1.9665376250632107, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69386240, "prompt_kind": "short", "t_relative_s": 2447.0711906249635}, {"decode_latency_s": 0.06581366015598178, "kv_live_bytes": 462274560, "latency_s": 2.108117625117302, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70402048, "prompt_kind": "long", "t_relative_s": 2452.0812528333627}, {"decode_latency_s": 0.06061456771567464, "kv_live_bytes": 462274560, "latency_s": 1.9404036672785878, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70909952, "prompt_kind": "short", "t_relative_s": 2457.0913189579733}, {"decode_latency_s": 0.06758460415585432, "kv_live_bytes": 462274560, "latency_s": 2.16500825015828, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70959104, "prompt_kind": "long", "t_relative_s": 2462.1013822080567}, {"decode_latency_s": 0.060401380207622424, "kv_live_bytes": 462274560, "latency_s": 1.9337861253879964, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69648384, "prompt_kind": "short", "t_relative_s": 2467.1114567080513}, {"decode_latency_s": 0.06801823437854182, "kv_live_bytes": 462274560, "latency_s": 2.1783165829256177, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70418432, "prompt_kind": "long", "t_relative_s": 2472.121535083279}, {"decode_latency_s": 0.06119620834942907, "kv_live_bytes": 462274560, "latency_s": 1.9591498747467995, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69746688, "prompt_kind": "short", "t_relative_s": 2477.131584917195}, {"decode_latency_s": 0.06787326952326111, "kv_live_bytes": 462274560, "latency_s": 2.1735981251113117, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69042176, "prompt_kind": "long", "t_relative_s": 2482.141635583248}, {"decode_latency_s": 0.06050372656318359, "kv_live_bytes": 462274560, "latency_s": 1.9371052500791848, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69238784, "prompt_kind": "short", "t_relative_s": 2487.1516871252097}, {"decode_latency_s": 0.06814316277450416, "kv_live_bytes": 462274560, "latency_s": 2.18243029108271, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70959104, "prompt_kind": "long", "t_relative_s": 2492.161755166948}, {"decode_latency_s": 0.0604976393369725, "kv_live_bytes": 462274560, "latency_s": 1.936729250010103, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70991872, "prompt_kind": "short", "t_relative_s": 2497.1718272082508}, {"decode_latency_s": 0.06197552607045509, "kv_live_bytes": 462274560, "latency_s": 1.984041666612029, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70942720, "prompt_kind": "long", "t_relative_s": 2502.1818875833414}, {"decode_latency_s": 0.08938437369943131, "kv_live_bytes": 462274560, "latency_s": 2.8611742919310927, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66748416, "prompt_kind": "short", "t_relative_s": 2507.1839160830714}, {"decode_latency_s": 0.0675552395550767, "kv_live_bytes": 462274560, "latency_s": 2.164108084049076, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67371008, "prompt_kind": "long", "t_relative_s": 2514.563008583151}, {"decode_latency_s": 0.06680906769179273, "kv_live_bytes": 462274560, "latency_s": 2.1394009590148926, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67977216, "prompt_kind": "short", "t_relative_s": 2519.5731230000965}, {"decode_latency_s": 0.06095748697407544, "kv_live_bytes": 462274560, "latency_s": 1.9513886249624193, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68009984, "prompt_kind": "long", "t_relative_s": 2524.5831935000606}, {"decode_latency_s": 0.06059767316037323, "kv_live_bytes": 462274560, "latency_s": 1.9400095003657043, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67911680, "prompt_kind": "short", "t_relative_s": 2529.5932622500695}, {"decode_latency_s": 0.06846305467479397, "kv_live_bytes": 462274560, "latency_s": 2.1929673748090863, "n_emitted": 32, "ok": true, "process_footprint_bytes": 60030976, "prompt_kind": "long", "t_relative_s": 2534.603335833177}, {"decode_latency_s": 0.060713808590662666, "kv_live_bytes": 462274560, "latency_s": 1.9436084171757102, "n_emitted": 32, "ok": true, "process_footprint_bytes": 65060864, "prompt_kind": "short", "t_relative_s": 2539.6134069580585}, {"decode_latency_s": 0.06841148568491917, "kv_live_bytes": 462274560, "latency_s": 2.191669291816652, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67911680, "prompt_kind": "long", "t_relative_s": 2544.6234619999304}, {"decode_latency_s": 0.06035246353712864, "kv_live_bytes": 462274560, "latency_s": 1.9321657503023744, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68452352, "prompt_kind": "short", "t_relative_s": 2549.6335194171406}, {"decode_latency_s": 0.06773687110398896, "kv_live_bytes": 462274560, "latency_s": 2.1692857081070542, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68616192, "prompt_kind": "long", "t_relative_s": 2554.643592624925}, {"decode_latency_s": 0.0665502629999537, "kv_live_bytes": 462274560, "latency_s": 2.131417542230338, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68763648, "prompt_kind": "short", "t_relative_s": 2559.653641208075}, {"decode_latency_s": 0.06862719921628013, "kv_live_bytes": 462274560, "latency_s": 2.1985412081703544, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68173824, "prompt_kind": "long", "t_relative_s": 2564.663738375064}, {"decode_latency_s": 0.06091420180746354, "kv_live_bytes": 462274560, "latency_s": 1.9501002500765026, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68386816, "prompt_kind": "short", "t_relative_s": 2569.6738178329542}, {"decode_latency_s": 0.0679313320433721, "kv_live_bytes": 462274560, "latency_s": 2.1758236247114837, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68829184, "prompt_kind": "long", "t_relative_s": 2574.683870833367}, {"decode_latency_s": 0.060425733056035824, "kv_live_bytes": 462274560, "latency_s": 1.9345247498713434, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69353472, "prompt_kind": "short", "t_relative_s": 2579.69394012494}, {"decode_latency_s": 0.06838295052875765, "kv_live_bytes": 462274560, "latency_s": 2.1894746250472963, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69386240, "prompt_kind": "long", "t_relative_s": 2584.704026750289}, {"decode_latency_s": 0.06658519789925776, "kv_live_bytes": 462274560, "latency_s": 2.132905332837254, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69419008, "prompt_kind": "short", "t_relative_s": 2589.7140768752433}, {"decode_latency_s": 0.06950667320052162, "kv_live_bytes": 462274560, "latency_s": 2.226425249595195, "n_emitted": 32, "ok": true, "process_footprint_bytes": 61718528, "prompt_kind": "long", "t_relative_s": 2594.724163208157}, {"decode_latency_s": 0.06125021225307137, "kv_live_bytes": 462274560, "latency_s": 1.9609932089224458, "n_emitted": 32, "ok": true, "process_footprint_bytes": 64962560, "prompt_kind": "short", "t_relative_s": 2599.734238583129}, {"decode_latency_s": 0.06762338930275291, "kv_live_bytes": 462274560, "latency_s": 2.165542332921177, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68435968, "prompt_kind": "long", "t_relative_s": 2604.7443277081475}, {"decode_latency_s": 0.060638371083769016, "kv_live_bytes": 462274560, "latency_s": 1.9412924167700112, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69124096, "prompt_kind": "short", "t_relative_s": 2609.7543727499433}, {"decode_latency_s": 0.06827897006587591, "kv_live_bytes": 462274560, "latency_s": 2.18707637488842, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69566464, "prompt_kind": "long", "t_relative_s": 2614.7644443330355}, {"decode_latency_s": 0.06668726303905714, "kv_live_bytes": 462274560, "latency_s": 2.1356739164330065, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69681152, "prompt_kind": "short", "t_relative_s": 2619.7745320000686}, {"decode_latency_s": 0.06818250780634116, "kv_live_bytes": 462274560, "latency_s": 2.184393874835223, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68960256, "prompt_kind": "long", "t_relative_s": 2624.7845898331143}, {"decode_latency_s": 0.06049900781363249, "kv_live_bytes": 462274560, "latency_s": 1.9368225000798702, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69599232, "prompt_kind": "short", "t_relative_s": 2629.7946863332763}, {"decode_latency_s": 0.06511200782551896, "kv_live_bytes": 462274560, "latency_s": 2.084859250113368, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69353472, "prompt_kind": "long", "t_relative_s": 2634.8047800422646}, {"decode_latency_s": 0.06047232162381988, "kv_live_bytes": 462274560, "latency_s": 1.936022374778986, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68272128, "prompt_kind": "short", "t_relative_s": 2639.814827958122}, {"decode_latency_s": 0.06487816665321589, "kv_live_bytes": 462274560, "latency_s": 2.0773811251856387, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68763648, "prompt_kind": "long", "t_relative_s": 2644.8248500423506}, {"decode_latency_s": 0.06552805077808443, "kv_live_bytes": 462274560, "latency_s": 2.0986275421455503, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67780608, "prompt_kind": "short", "t_relative_s": 2649.834905249998}, {"decode_latency_s": 0.06791826304106507, "kv_live_bytes": 462274560, "latency_s": 2.1761467093601823, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69320704, "prompt_kind": "long", "t_relative_s": 2654.8449790831655}, {"decode_latency_s": 0.0606640599144157, "kv_live_bytes": 462274560, "latency_s": 1.9420542907901108, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69959680, "prompt_kind": "short", "t_relative_s": 2659.855081083253}, {"decode_latency_s": 0.06854122393997386, "kv_live_bytes": 462274560, "latency_s": 2.196035500150174, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70057984, "prompt_kind": "long", "t_relative_s": 2664.8651256249286}, {"decode_latency_s": 0.060436481740907766, "kv_live_bytes": 462274560, "latency_s": 1.9348557502962649, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70500352, "prompt_kind": "short", "t_relative_s": 2669.875200957991}, {"decode_latency_s": 0.06829017706331797, "kv_live_bytes": 462274560, "latency_s": 2.1867584590800107, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70696960, "prompt_kind": "long", "t_relative_s": 2674.8852830831893}, {"decode_latency_s": 0.060538582038134336, "kv_live_bytes": 462274560, "latency_s": 1.9381156670860946, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70844416, "prompt_kind": "short", "t_relative_s": 2679.8953270423226}, {"decode_latency_s": 0.06783645311952569, "kv_live_bytes": 462274560, "latency_s": 2.172951125074178, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69058560, "prompt_kind": "long", "t_relative_s": 2684.9053881252185}, {"decode_latency_s": 0.06053589843213558, "kv_live_bytes": 462274560, "latency_s": 1.9380185841582716, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67993600, "prompt_kind": "short", "t_relative_s": 2689.9154611672275}, {"decode_latency_s": 0.06803127865714487, "kv_live_bytes": 462274560, "latency_s": 2.179083624854684, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70778880, "prompt_kind": "long", "t_relative_s": 2694.9255232079886}, {"decode_latency_s": 0.060625790370977484, "kv_live_bytes": 462274560, "latency_s": 1.940855666063726, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71385088, "prompt_kind": "short", "t_relative_s": 2699.9355722079054}, {"decode_latency_s": 0.06818326041684486, "kv_live_bytes": 462274560, "latency_s": 2.184345708694309, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71516160, "prompt_kind": "long", "t_relative_s": 2704.9456612919457}, {"decode_latency_s": 0.060846368491183966, "kv_live_bytes": 462274560, "latency_s": 1.9478280418552458, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71532544, "prompt_kind": "short", "t_relative_s": 2709.9557314580306}, {"decode_latency_s": 0.06817817577393726, "kv_live_bytes": 462274560, "latency_s": 2.1840185415931046, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70877184, "prompt_kind": "long", "t_relative_s": 2714.96578541724}, {"decode_latency_s": 0.06041672137507703, "kv_live_bytes": 462274560, "latency_s": 1.9342200420796871, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70975488, "prompt_kind": "short", "t_relative_s": 2719.9758609170094}, {"decode_latency_s": 0.0682867499999702, "kv_live_bytes": 462274560, "latency_s": 2.1868697092868388, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71598080, "prompt_kind": "long", "t_relative_s": 2724.9859518329613}, {"decode_latency_s": 0.06138890885631554, "kv_live_bytes": 462274560, "latency_s": 1.9657960841432214, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72138752, "prompt_kind": "short", "t_relative_s": 2729.9960164581425}, {"decode_latency_s": 0.06845481118944008, "kv_live_bytes": 462274560, "latency_s": 2.1929053752683103, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72073216, "prompt_kind": "long", "t_relative_s": 2735.006104625296}, {"decode_latency_s": 0.0610607604321558, "kv_live_bytes": 462274560, "latency_s": 1.9547017919830978, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72286208, "prompt_kind": "short", "t_relative_s": 2740.016138625331}, {"decode_latency_s": 0.06789723696419969, "kv_live_bytes": 462274560, "latency_s": 2.1748552499338984, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72286208, "prompt_kind": "long", "t_relative_s": 2745.0261842082255}, {"decode_latency_s": 0.06035337240609806, "kv_live_bytes": 462274560, "latency_s": 1.9322142084129155, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72187904, "prompt_kind": "short", "t_relative_s": 2750.0362640419044}, {"decode_latency_s": 0.06818289193324745, "kv_live_bytes": 462274560, "latency_s": 2.1836080830544233, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72302592, "prompt_kind": "long", "t_relative_s": 2755.04634379223}, {"decode_latency_s": 0.06646100521902554, "kv_live_bytes": 462274560, "latency_s": 2.128519124817103, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72892416, "prompt_kind": "short", "t_relative_s": 2760.0563969169743}, {"decode_latency_s": 0.06811908853705972, "kv_live_bytes": 462274560, "latency_s": 2.1832982092164457, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72974336, "prompt_kind": "long", "t_relative_s": 2765.066492666956}, {"decode_latency_s": 0.06072622397914529, "kv_live_bytes": 462274560, "latency_s": 1.9440167909488082, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72957952, "prompt_kind": "short", "t_relative_s": 2770.0765698752366}, {"decode_latency_s": 0.06795740366214886, "kv_live_bytes": 462274560, "latency_s": 2.1770694158039987, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66682880, "prompt_kind": "long", "t_relative_s": 2775.086604082957}, {"decode_latency_s": 0.0605945195275126, "kv_live_bytes": 462274560, "latency_s": 1.940013958606869, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68616192, "prompt_kind": "short", "t_relative_s": 2780.096700707916}, {"decode_latency_s": 0.0682522343704477, "kv_live_bytes": 462274560, "latency_s": 2.185770457610488, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72581120, "prompt_kind": "long", "t_relative_s": 2785.106771042105}, {"decode_latency_s": 0.06048222265962977, "kv_live_bytes": 462274560, "latency_s": 1.936314499936998, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73138176, "prompt_kind": "short", "t_relative_s": 2790.1168124172837}, {"decode_latency_s": 0.06795369272003882, "kv_live_bytes": 462274560, "latency_s": 2.1768485829234123, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73170944, "prompt_kind": "long", "t_relative_s": 2795.126892332919}, {"decode_latency_s": 0.06055398048192728, "kv_live_bytes": 462274560, "latency_s": 1.9384595407173038, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73203712, "prompt_kind": "short", "t_relative_s": 2800.1369676250033}, {"decode_latency_s": 0.06839508333359845, "kv_live_bytes": 462274560, "latency_s": 2.190771209076047, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73039872, "prompt_kind": "long", "t_relative_s": 2805.1470225830562}, {"decode_latency_s": 0.060440079410909675, "kv_live_bytes": 462274560, "latency_s": 1.9349780841730535, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73072640, "prompt_kind": "short", "t_relative_s": 2810.157079708297}, {"decode_latency_s": 0.06828288543329109, "kv_live_bytes": 462274560, "latency_s": 2.1868920838460326, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73252864, "prompt_kind": "long", "t_relative_s": 2815.167168417014}, {"decode_latency_s": 0.06055228515469935, "kv_live_bytes": 462274560, "latency_s": 1.938529749866575, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73793536, "prompt_kind": "short", "t_relative_s": 2820.177229083143}, {"decode_latency_s": 0.06841699349752162, "kv_live_bytes": 462274560, "latency_s": 2.1917193331755698, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73842688, "prompt_kind": "long", "t_relative_s": 2825.1873016669415}, {"decode_latency_s": 0.06054487889923621, "kv_live_bytes": 462274560, "latency_s": 1.9382549999281764, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73924608, "prompt_kind": "short", "t_relative_s": 2830.197385125328}, {"decode_latency_s": 0.06761664974328596, "kv_live_bytes": 462274560, "latency_s": 2.165926791727543, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73187328, "prompt_kind": "long", "t_relative_s": 2835.207440542057}, {"decode_latency_s": 0.06044286327960435, "kv_live_bytes": 462274560, "latency_s": 1.9350350419990718, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73220096, "prompt_kind": "short", "t_relative_s": 2840.217510000337}, {"decode_latency_s": 0.0603405481815571, "kv_live_bytes": 462274560, "latency_s": 1.9318105420097709, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73416704, "prompt_kind": "long", "t_relative_s": 2845.2275952920318}, {"decode_latency_s": 0.06044399739766959, "kv_live_bytes": 462274560, "latency_s": 1.9350703749805689, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73940992, "prompt_kind": "short", "t_relative_s": 2850.237648332957}, {"decode_latency_s": 0.06860232811595779, "kv_live_bytes": 462274560, "latency_s": 2.1973955002613366, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73990144, "prompt_kind": "long", "t_relative_s": 2855.247719042003}, {"decode_latency_s": 0.06658005858480465, "kv_live_bytes": 462274560, "latency_s": 2.132236875128001, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74022912, "prompt_kind": "short", "t_relative_s": 2860.257800917141}, {"decode_latency_s": 0.060444192684371956, "kv_live_bytes": 462274560, "latency_s": 1.9349620421417058, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73826304, "prompt_kind": "long", "t_relative_s": 2865.2678572922014}, {"decode_latency_s": 0.060374438820872456, "kv_live_bytes": 462274560, "latency_s": 1.932901583146304, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72613888, "prompt_kind": "short", "t_relative_s": 2870.277907792013}, {"decode_latency_s": 0.06838487631466705, "kv_live_bytes": 462274560, "latency_s": 2.1903539588674903, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73302016, "prompt_kind": "long", "t_relative_s": 2875.287987458054}, {"decode_latency_s": 0.06642931641545147, "kv_live_bytes": 462274560, "latency_s": 2.1273969169706106, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73875456, "prompt_kind": "short", "t_relative_s": 2880.2967965421267}, {"decode_latency_s": 0.06814110808772966, "kv_live_bytes": 462274560, "latency_s": 2.1834060410037637, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73924608, "prompt_kind": "long", "t_relative_s": 2885.306875000242}, {"decode_latency_s": 0.06061867318931036, "kv_live_bytes": 462274560, "latency_s": 1.9405330419540405, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73990144, "prompt_kind": "short", "t_relative_s": 2890.3169611669146}, {"decode_latency_s": 0.06767214584397152, "kv_live_bytes": 462274560, "latency_s": 2.167483499739319, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70025216, "prompt_kind": "long", "t_relative_s": 2895.3270136252977}, {"decode_latency_s": 0.060447350289905444, "kv_live_bytes": 462274560, "latency_s": 1.9352482915855944, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69074944, "prompt_kind": "short", "t_relative_s": 2900.3370729582384}, {"decode_latency_s": 0.0677968841191614, "kv_live_bytes": 462274560, "latency_s": 2.1716394582763314, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73203712, "prompt_kind": "long", "t_relative_s": 2905.3471519169398}, {"decode_latency_s": 0.06668044271646068, "kv_live_bytes": 462274560, "latency_s": 2.135425833053887, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73777152, "prompt_kind": "short", "t_relative_s": 2910.3572235419415}, {"decode_latency_s": 0.06830880595953204, "kv_live_bytes": 462274560, "latency_s": 2.188519374933094, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73793536, "prompt_kind": "long", "t_relative_s": 2915.367278708145}, {"decode_latency_s": 0.06050122265878599, "kv_live_bytes": 462274560, "latency_s": 1.9368119579739869, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73842688, "prompt_kind": "short", "t_relative_s": 2920.3773639579304}, {"decode_latency_s": 0.0680674544419162, "kv_live_bytes": 462274560, "latency_s": 2.1802205410785973, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73678848, "prompt_kind": "long", "t_relative_s": 2925.3874224172905}, {"decode_latency_s": 0.06033238803502172, "kv_live_bytes": 462274560, "latency_s": 1.9315590001642704, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73646080, "prompt_kind": "short", "t_relative_s": 2930.397475332953}, {"decode_latency_s": 0.06779477083182428, "kv_live_bytes": 462274560, "latency_s": 2.1715497910045087, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73809920, "prompt_kind": "long", "t_relative_s": 2935.4075500420295}, {"decode_latency_s": 0.0664679505280219, "kv_live_bytes": 462274560, "latency_s": 2.1287325830198824, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73859072, "prompt_kind": "short", "t_relative_s": 2940.417606708128}, {"decode_latency_s": 0.06811999477213249, "kv_live_bytes": 462274560, "latency_s": 2.182383374776691, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73875456, "prompt_kind": "long", "t_relative_s": 2945.427649416961}, {"decode_latency_s": 0.06048641925735865, "kv_live_bytes": 462274560, "latency_s": 1.9363510003313422, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73924608, "prompt_kind": "short", "t_relative_s": 2950.4377290420234}, {"decode_latency_s": 0.06799773174861912, "kv_live_bytes": 462274560, "latency_s": 2.1776782078668475, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72613888, "prompt_kind": "long", "t_relative_s": 2955.4477580832317}, {"decode_latency_s": 0.060441067689680494, "kv_live_bytes": 462274560, "latency_s": 1.9351172908209264, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72663040, "prompt_kind": "short", "t_relative_s": 2960.457792292349}, {"decode_latency_s": 0.06827686197357252, "kv_live_bytes": 462274560, "latency_s": 2.186856208834797, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73416704, "prompt_kind": "long", "t_relative_s": 2965.467882832978}, {"decode_latency_s": 0.06664539975463413, "kv_live_bytes": 462274560, "latency_s": 2.1343881669454277, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73924608, "prompt_kind": "short", "t_relative_s": 2970.4779487499036}, {"decode_latency_s": 0.06816869793692604, "kv_live_bytes": 462274560, "latency_s": 2.1837023752741516, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73990144, "prompt_kind": "long", "t_relative_s": 2975.4880230831914}, {"decode_latency_s": 0.06067421221814584, "kv_live_bytes": 462274560, "latency_s": 1.942350750323385, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74039296, "prompt_kind": "short", "t_relative_s": 2980.4981082081795}, {"decode_latency_s": 0.06900908071838785, "kv_live_bytes": 462274560, "latency_s": 2.2091679158620536, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70811648, "prompt_kind": "long", "t_relative_s": 2985.5004010000266}, {"decode_latency_s": 0.06026842709979974, "kv_live_bytes": 462274560, "latency_s": 1.9295442500151694, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70860800, "prompt_kind": "short", "t_relative_s": 2992.0980941252783}, {"decode_latency_s": 0.06761970443767495, "kv_live_bytes": 462274560, "latency_s": 2.1657490422949195, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70926336, "prompt_kind": "long", "t_relative_s": 2997.1081738332286}, {"decode_latency_s": 0.06053645831707399, "kv_live_bytes": 462274560, "latency_s": 1.9379031667485833, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70942720, "prompt_kind": "short", "t_relative_s": 3002.118259458337}, {"decode_latency_s": 0.06797898178047035, "kv_live_bytes": 462274560, "latency_s": 2.177660208195448, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70991872, "prompt_kind": "long", "t_relative_s": 3007.128315458074}, {"decode_latency_s": 0.060302197918645106, "kv_live_bytes": 462274560, "latency_s": 1.9305458329617977, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71024640, "prompt_kind": "short", "t_relative_s": 3012.138391250279}, {"decode_latency_s": 0.06850798046798445, "kv_live_bytes": 462274560, "latency_s": 2.194347749929875, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69451776, "prompt_kind": "long", "t_relative_s": 3017.1484757079743}, {"decode_latency_s": 0.06034547265153378, "kv_live_bytes": 462274560, "latency_s": 1.9319065832532942, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69615616, "prompt_kind": "short", "t_relative_s": 3022.158503250219}, {"decode_latency_s": 0.06827813280688133, "kv_live_bytes": 462274560, "latency_s": 2.1869594999589026, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70238208, "prompt_kind": "long", "t_relative_s": 3027.1685834582895}, {"decode_latency_s": 0.0606798033695668, "kv_live_bytes": 462274560, "latency_s": 1.94252908276394, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70647808, "prompt_kind": "short", "t_relative_s": 3032.178709917236}, {"decode_latency_s": 0.06788707162195351, "kv_live_bytes": 462274560, "latency_s": 2.1749161249026656, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70795264, "prompt_kind": "long", "t_relative_s": 3037.1887586670928}, {"decode_latency_s": 0.060530983057105914, "kv_live_bytes": 462274560, "latency_s": 1.93782962532714, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70647808, "prompt_kind": "short", "t_relative_s": 3042.1988306250423}, {"decode_latency_s": 0.0673741588689154, "kv_live_bytes": 462274560, "latency_s": 2.157914541196078, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70680576, "prompt_kind": "long", "t_relative_s": 3047.208904375322}, {"decode_latency_s": 0.0708355859387666, "kv_live_bytes": 462274560, "latency_s": 2.273103083949536, "n_emitted": 32, "ok": true, "process_footprint_bytes": 50036736, "prompt_kind": "short", "t_relative_s": 3052.2189392922446}, {"decode_latency_s": 0.0696925716183614, "kv_live_bytes": 462274560, "latency_s": 2.2336307079531252, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70352896, "prompt_kind": "long", "t_relative_s": 3057.2290083752014}, {"decode_latency_s": 0.06062698567984626, "kv_live_bytes": 462274560, "latency_s": 1.940862957853824, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70664192, "prompt_kind": "short", "t_relative_s": 3062.2390959169716}, {"decode_latency_s": 0.06814589061832521, "kv_live_bytes": 462274560, "latency_s": 2.1832705000415444, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70729728, "prompt_kind": "long", "t_relative_s": 3067.2491645421833}, {"decode_latency_s": 0.060357361973728985, "kv_live_bytes": 462274560, "latency_s": 1.9323735409416258, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69255168, "prompt_kind": "short", "t_relative_s": 3072.2592371250503}, {"decode_latency_s": 0.060567260428797454, "kv_live_bytes": 462274560, "latency_s": 1.9391226670704782, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69926912, "prompt_kind": "long", "t_relative_s": 3077.269312541932}, {"decode_latency_s": 0.06037736982398201, "kv_live_bytes": 462274560, "latency_s": 1.9331481247209013, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70664192, "prompt_kind": "short", "t_relative_s": 3082.279367875308}, {"decode_latency_s": 0.06770763412350789, "kv_live_bytes": 462274560, "latency_s": 2.169224707875401, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70877184, "prompt_kind": "long", "t_relative_s": 3087.289434667211}, {"decode_latency_s": 0.060451239594840445, "kv_live_bytes": 462274560, "latency_s": 1.9351246668957174, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71434240, "prompt_kind": "short", "t_relative_s": 3092.2995168329217}, {"decode_latency_s": 0.0609192877454916, "kv_live_bytes": 462274560, "latency_s": 1.950750334188342, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71565312, "prompt_kind": "long", "t_relative_s": 3097.303608875256}, {"decode_latency_s": 0.060582282560062595, "kv_live_bytes": 462274560, "latency_s": 1.9395204591564834, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71532544, "prompt_kind": "short", "t_relative_s": 3102.3136829999276}, {"decode_latency_s": 0.06827467060065828, "kv_live_bytes": 462274560, "latency_s": 2.1868406673893332, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71024640, "prompt_kind": "long", "t_relative_s": 3107.323753166944}, {"decode_latency_s": 0.06628883462690283, "kv_live_bytes": 462274560, "latency_s": 2.122891040984541, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71761920, "prompt_kind": "short", "t_relative_s": 3112.3338038329966}, {"decode_latency_s": 0.06812807684764266, "kv_live_bytes": 462274560, "latency_s": 2.182680499739945, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71434240, "prompt_kind": "long", "t_relative_s": 3117.3439042079262}, {"decode_latency_s": 0.06050203647464514, "kv_live_bytes": 462274560, "latency_s": 1.936884207651019, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72155136, "prompt_kind": "short", "t_relative_s": 3122.3540093330666}, {"decode_latency_s": 0.06791881250683218, "kv_live_bytes": 462274560, "latency_s": 2.175685332622379, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72351744, "prompt_kind": "long", "t_relative_s": 3127.364048167132}, {"decode_latency_s": 0.06030649090826046, "kv_live_bytes": 462274560, "latency_s": 1.930684084072709, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71876608, "prompt_kind": "short", "t_relative_s": 3132.373785292264}, {"decode_latency_s": 0.06839869140821975, "kv_live_bytes": 462274560, "latency_s": 2.1906459159217775, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71106560, "prompt_kind": "long", "t_relative_s": 3137.383871458005}, {"decode_latency_s": 0.0604186158743687, "kv_live_bytes": 462274560, "latency_s": 1.934256125241518, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71172096, "prompt_kind": "short", "t_relative_s": 3142.393915625289}, {"decode_latency_s": 0.06765714843641035, "kv_live_bytes": 462274560, "latency_s": 2.1666911663487554, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71614464, "prompt_kind": "long", "t_relative_s": 3147.403985292185}, {"decode_latency_s": 0.06053536328545306, "kv_live_bytes": 462274560, "latency_s": 1.937912874855101, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72204288, "prompt_kind": "short", "t_relative_s": 3152.4140729173087}, {"decode_latency_s": 0.06801925259060226, "kv_live_bytes": 462274560, "latency_s": 2.17918345797807, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72253440, "prompt_kind": "long", "t_relative_s": 3157.4241189169697}, {"decode_latency_s": 0.06077515885408502, "kv_live_bytes": 462274560, "latency_s": 1.9455761667340994, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71974912, "prompt_kind": "short", "t_relative_s": 3162.43419237528}, {"decode_latency_s": 0.06863668491132557, "kv_live_bytes": 462274560, "latency_s": 2.1983133340254426, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71532544, "prompt_kind": "long", "t_relative_s": 3167.4442793750204}, {"decode_latency_s": 0.060258053403231315, "kv_live_bytes": 462274560, "latency_s": 1.9291384578682482, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72024064, "prompt_kind": "short", "t_relative_s": 3172.454339625314}, {"decode_latency_s": 0.06772228906629607, "kv_live_bytes": 462274560, "latency_s": 2.1690537496469915, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72138752, "prompt_kind": "long", "t_relative_s": 3177.464433291927}, {"decode_latency_s": 0.06061379816674162, "kv_live_bytes": 462274560, "latency_s": 1.9404585831798613, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72679424, "prompt_kind": "short", "t_relative_s": 3182.4745464171283}, {"decode_latency_s": 0.06768843490863219, "kv_live_bytes": 462274560, "latency_s": 2.168389833997935, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72777728, "prompt_kind": "long", "t_relative_s": 3187.484604583122}, {"decode_latency_s": 0.06038571354292799, "kv_live_bytes": 462274560, "latency_s": 1.9331845422275364, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72876032, "prompt_kind": "short", "t_relative_s": 3192.494683458004}, {"decode_latency_s": 0.0682988763001049, "kv_live_bytes": 462274560, "latency_s": 2.1876652501523495, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69877760, "prompt_kind": "long", "t_relative_s": 3197.5047596669756}, {"decode_latency_s": 0.06054678387590684, "kv_live_bytes": 462274560, "latency_s": 1.9384226673282683, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71958528, "prompt_kind": "short", "t_relative_s": 3202.514811167028}, {"decode_latency_s": 0.06796525912068319, "kv_live_bytes": 462274560, "latency_s": 2.177317707799375, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72089600, "prompt_kind": "long", "t_relative_s": 3207.524901667144}, {"decode_latency_s": 0.0668861301819561, "kv_live_bytes": 462274560, "latency_s": 2.1421050000935793, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72613888, "prompt_kind": "short", "t_relative_s": 3212.5349874999374}, {"decode_latency_s": 0.060912443994311616, "kv_live_bytes": 462274560, "latency_s": 1.950556042138487, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72663040, "prompt_kind": "long", "t_relative_s": 3217.5450401250273}, {"decode_latency_s": 0.060603165373322554, "kv_live_bytes": 462274560, "latency_s": 1.940154124982655, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72433664, "prompt_kind": "short", "t_relative_s": 3222.555128583219}, {"decode_latency_s": 0.06794035024358891, "kv_live_bytes": 462274560, "latency_s": 2.1761380839161575, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71909376, "prompt_kind": "long", "t_relative_s": 3227.565205417108}, {"decode_latency_s": 0.06033135674078949, "kv_live_bytes": 462274560, "latency_s": 1.93148074997589, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72581120, "prompt_kind": "short", "t_relative_s": 3232.575274417177}, {"decode_latency_s": 0.06764338022912852, "kv_live_bytes": 462274560, "latency_s": 2.167024917434901, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "long", "t_relative_s": 3237.5853598332033}, {"decode_latency_s": 0.060581700527109206, "kv_live_bytes": 462274560, "latency_s": 1.939382459037006, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73236480, "prompt_kind": "short", "t_relative_s": 3242.595449458342}, {"decode_latency_s": 0.06751182030711789, "kv_live_bytes": 462274560, "latency_s": 2.1629204587079585, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70729728, "prompt_kind": "long", "t_relative_s": 3247.605502374936}, {"decode_latency_s": 0.06068085419246927, "kv_live_bytes": 462274560, "latency_s": 1.9427691660821438, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70942720, "prompt_kind": "short", "t_relative_s": 3252.615550958086}, {"decode_latency_s": 0.17723929688509088, "kv_live_bytes": 462274560, "latency_s": 5.672626083251089, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66027520, "prompt_kind": "long", "t_relative_s": 3257.6256534582935}, {"decode_latency_s": 0.06047476953244768, "kv_live_bytes": 462274560, "latency_s": 1.9360612910240889, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66715648, "prompt_kind": "short", "t_relative_s": 3266.359179667197}, {"decode_latency_s": 0.06798669531417545, "kv_live_bytes": 462274560, "latency_s": 2.1779510830529034, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67764224, "prompt_kind": "long", "t_relative_s": 3271.3692274582572}, {"decode_latency_s": 0.06065109897463117, "kv_live_bytes": 462274560, "latency_s": 1.9416742497123778, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68354048, "prompt_kind": "short", "t_relative_s": 3276.379298250191}, {"decode_latency_s": 0.06763904297258705, "kv_live_bytes": 462274560, "latency_s": 2.1660304167307913, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66977792, "prompt_kind": "long", "t_relative_s": 3281.3894331669435}, {"decode_latency_s": 0.06535520311444998, "kv_live_bytes": 462274560, "latency_s": 2.0929697919636965, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66994176, "prompt_kind": "short", "t_relative_s": 3286.3995043751784}, {"decode_latency_s": 0.06761109894432593, "kv_live_bytes": 462274560, "latency_s": 2.1659241248853505, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67731456, "prompt_kind": "long", "t_relative_s": 3291.4095551669598}, {"decode_latency_s": 0.06041770185402129, "kv_live_bytes": 462274560, "latency_s": 1.9342665411531925, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68288512, "prompt_kind": "short", "t_relative_s": 3296.4196276250295}, {"decode_latency_s": 0.06096824737323914, "kv_live_bytes": 462274560, "latency_s": 1.9522171667777002, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68354048, "prompt_kind": "long", "t_relative_s": 3301.429701332934}, {"decode_latency_s": 0.06706080469302833, "kv_live_bytes": 462274560, "latency_s": 2.1478887908160686, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68943872, "prompt_kind": "short", "t_relative_s": 3306.436193624977}, {"decode_latency_s": 0.06820044141204562, "kv_live_bytes": 462274560, "latency_s": 2.1850739582441747, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69025792, "prompt_kind": "long", "t_relative_s": 3311.44627375016}, {"decode_latency_s": 0.06977463541261386, "kv_live_bytes": 462274560, "latency_s": 2.2345474995672703, "n_emitted": 32, "ok": true, "process_footprint_bytes": 64897024, "prompt_kind": "short", "t_relative_s": 3316.456369082909}, {"decode_latency_s": 0.06834447657456622, "kv_live_bytes": 462274560, "latency_s": 2.1901638749986887, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66306048, "prompt_kind": "long", "t_relative_s": 3321.466474249959}, {"decode_latency_s": 0.06075278256321326, "kv_live_bytes": 462274560, "latency_s": 1.9448617077432573, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68780032, "prompt_kind": "short", "t_relative_s": 3326.4765462921932}, {"decode_latency_s": 0.06781717446574476, "kv_live_bytes": 462274560, "latency_s": 2.172998458147049, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68780032, "prompt_kind": "long", "t_relative_s": 3331.486119000241}, {"decode_latency_s": 0.06092502735555172, "kv_live_bytes": 462274560, "latency_s": 1.9503504168242216, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69320704, "prompt_kind": "short", "t_relative_s": 3336.4962070421316}, {"decode_latency_s": 0.06842740492720623, "kv_live_bytes": 462274560, "latency_s": 2.191692791879177, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68239360, "prompt_kind": "long", "t_relative_s": 3341.5062622502446}, {"decode_latency_s": 0.06072361327824183, "kv_live_bytes": 462274560, "latency_s": 1.9439384168945253, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68239360, "prompt_kind": "short", "t_relative_s": 3346.5163347921334}, {"decode_latency_s": 0.06668194531812333, "kv_live_bytes": 462274560, "latency_s": 2.1359765003435314, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68763648, "prompt_kind": "long", "t_relative_s": 3351.526381792035}, {"decode_latency_s": 0.0667291926802136, "kv_live_bytes": 462274560, "latency_s": 2.1364208748564124, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67207168, "prompt_kind": "short", "t_relative_s": 3356.5364681249484}, {"decode_latency_s": 0.06804698046471458, "kv_live_bytes": 462274560, "latency_s": 2.179820166900754, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68681728, "prompt_kind": "long", "t_relative_s": 3361.5465412922204}, {"decode_latency_s": 0.06111341147334315, "kv_live_bytes": 462274560, "latency_s": 1.9564124578610063, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69124096, "prompt_kind": "short", "t_relative_s": 3366.5566140003502}, {"decode_latency_s": 0.06982604946824722, "kv_live_bytes": 462274560, "latency_s": 2.2353633330203593, "n_emitted": 32, "ok": true, "process_footprint_bytes": 65421312, "prompt_kind": "long", "t_relative_s": 3371.5666654580273}, {"decode_latency_s": 0.06804448696493637, "kv_live_bytes": 462274560, "latency_s": 2.178731416352093, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67567616, "prompt_kind": "short", "t_relative_s": 3376.5767061673105}, {"decode_latency_s": 0.06843766536621843, "kv_live_bytes": 462274560, "latency_s": 2.1925551663152874, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68534272, "prompt_kind": "long", "t_relative_s": 3381.5867907083593}, {"decode_latency_s": 0.06073246484447736, "kv_live_bytes": 462274560, "latency_s": 1.9442622913047671, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69124096, "prompt_kind": "short", "t_relative_s": 3386.596863999963}, {"decode_latency_s": 0.06791000519297086, "kv_live_bytes": 462274560, "latency_s": 2.1757732089608908, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69173248, "prompt_kind": "long", "t_relative_s": 3391.606916333083}, {"decode_latency_s": 0.06071188149508089, "kv_live_bytes": 462274560, "latency_s": 1.9435547916218638, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69697536, "prompt_kind": "short", "t_relative_s": 3396.6169941672124}, {"decode_latency_s": 0.06800223045866005, "kv_live_bytes": 462274560, "latency_s": 2.1778652500361204, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68583424, "prompt_kind": "long", "t_relative_s": 3401.6270448751748}, {"decode_latency_s": 0.06035160677856766, "kv_live_bytes": 462274560, "latency_s": 1.9322249577380717, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68468736, "prompt_kind": "short", "t_relative_s": 3406.6371070002206}, {"decode_latency_s": 0.06773081119172275, "kv_live_bytes": 462274560, "latency_s": 2.169413583818823, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69189632, "prompt_kind": "long", "t_relative_s": 3411.6471885000356}, {"decode_latency_s": 0.060540260397829115, "kv_live_bytes": 462274560, "latency_s": 1.9380573751404881, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69779456, "prompt_kind": "short", "t_relative_s": 3416.6572207920253}, {"decode_latency_s": 0.06817205468541943, "kv_live_bytes": 462274560, "latency_s": 2.183778249658644, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69828608, "prompt_kind": "long", "t_relative_s": 3421.6672722920775}, {"decode_latency_s": 0.060787756490753964, "kv_live_bytes": 462274560, "latency_s": 1.9459887919947505, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70254592, "prompt_kind": "short", "t_relative_s": 3426.6773742502555}, {"decode_latency_s": 0.06865222656051628, "kv_live_bytes": 462274560, "latency_s": 2.198921542149037, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69877760, "prompt_kind": "long", "t_relative_s": 3431.687432500068}, {"decode_latency_s": 0.060864614555612206, "kv_live_bytes": 462274560, "latency_s": 1.948520042002201, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67584000, "prompt_kind": "short", "t_relative_s": 3436.697488625068}, {"decode_latency_s": 0.06802398699801415, "kv_live_bytes": 462274560, "latency_s": 2.1792366253212094, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69763072, "prompt_kind": "long", "t_relative_s": 3441.707549124956}, {"decode_latency_s": 0.06024608726147562, "kv_live_bytes": 462274560, "latency_s": 1.9288158332929015, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70336512, "prompt_kind": "short", "t_relative_s": 3446.717651333194}, {"decode_latency_s": 0.060280285149929114, "kv_live_bytes": 462274560, "latency_s": 1.9302429580129683, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70172672, "prompt_kind": "long", "t_relative_s": 3451.7277281251736}, {"decode_latency_s": 0.060490070303785615, "kv_live_bytes": 462274560, "latency_s": 1.9365737908519804, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70811648, "prompt_kind": "short", "t_relative_s": 3456.737778916955}, {"decode_latency_s": 0.06816206118674017, "kv_live_bytes": 462274560, "latency_s": 2.1834916658699512, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70909952, "prompt_kind": "long", "t_relative_s": 3461.7478682082146}, {"decode_latency_s": 0.06069432421645615, "kv_live_bytes": 462274560, "latency_s": 1.9430585000663996, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70909952, "prompt_kind": "short", "t_relative_s": 3466.757965875324}, {"decode_latency_s": 0.06798688018170651, "kv_live_bytes": 462274560, "latency_s": 2.17803491698578, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71139328, "prompt_kind": "long", "t_relative_s": 3471.768020375166}, {"decode_latency_s": 0.06045774478116073, "kv_live_bytes": 462274560, "latency_s": 1.9354783752933145, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71761920, "prompt_kind": "short", "t_relative_s": 3476.7780952919275}, {"decode_latency_s": 0.06078859505942091, "kv_live_bytes": 462274560, "latency_s": 1.9464072501286864, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71057408, "prompt_kind": "long", "t_relative_s": 3481.7881761672907}, {"decode_latency_s": 0.060821378909167834, "kv_live_bytes": 462274560, "latency_s": 1.947163457982242, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71680000, "prompt_kind": "short", "t_relative_s": 3486.7982310829684}, {"decode_latency_s": 0.06799735806998797, "kv_live_bytes": 462274560, "latency_s": 2.178432165645063, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71761920, "prompt_kind": "long", "t_relative_s": 3491.808298625052}, {"decode_latency_s": 0.06259197393956129, "kv_live_bytes": 462274560, "latency_s": 2.003856707829982, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70385664, "prompt_kind": "short", "t_relative_s": 3496.8183632083237}, {"decode_latency_s": 0.06842638412490487, "kv_live_bytes": 462274560, "latency_s": 2.1913169999606907, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71221248, "prompt_kind": "long", "t_relative_s": 3501.8284314172342}, {"decode_latency_s": 0.0605008046986768, "kv_live_bytes": 462274560, "latency_s": 1.9368097083643079, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71778304, "prompt_kind": "short", "t_relative_s": 3506.8384897499345}, {"decode_latency_s": 0.06795277862693183, "kv_live_bytes": 462274560, "latency_s": 2.176926165819168, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71827456, "prompt_kind": "long", "t_relative_s": 3511.84857004229}, {"decode_latency_s": 0.06072245571704116, "kv_live_bytes": 462274560, "latency_s": 1.9439425421878695, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72384512, "prompt_kind": "short", "t_relative_s": 3516.854624541942}, {"decode_latency_s": 0.0680469804792665, "kv_live_bytes": 462274560, "latency_s": 2.179417208302766, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72237056, "prompt_kind": "long", "t_relative_s": 3521.8646810832433}, {"decode_latency_s": 0.0605891731538577, "kv_live_bytes": 462274560, "latency_s": 1.9397849580273032, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72204288, "prompt_kind": "short", "t_relative_s": 3526.8747533331625}, {"decode_latency_s": 0.06811518878384959, "kv_live_bytes": 462274560, "latency_s": 2.181374041829258, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72384512, "prompt_kind": "long", "t_relative_s": 3531.884851917159}, {"decode_latency_s": 0.06046997266821563, "kv_live_bytes": 462274560, "latency_s": 1.935849417001009, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72908800, "prompt_kind": "short", "t_relative_s": 3536.894922333304}, {"decode_latency_s": 0.060424683601013385, "kv_live_bytes": 462274560, "latency_s": 1.9346056669019163, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72957952, "prompt_kind": "long", "t_relative_s": 3541.9030587500893}, {"decode_latency_s": 0.06076185806887224, "kv_live_bytes": 462274560, "latency_s": 1.9451512922532856, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72974336, "prompt_kind": "short", "t_relative_s": 3546.9131443751976}, {"decode_latency_s": 0.060644763012533076, "kv_live_bytes": 462274560, "latency_s": 1.9419668749906123, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72990720, "prompt_kind": "long", "t_relative_s": 3551.9232037919573}, {"decode_latency_s": 0.06804445052694064, "kv_live_bytes": 462274560, "latency_s": 2.1790770827792585, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69894144, "prompt_kind": "short", "t_relative_s": 3556.93327533314}, {"decode_latency_s": 0.06806966145813931, "kv_live_bytes": 462274560, "latency_s": 2.1809090841561556, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70680576, "prompt_kind": "long", "t_relative_s": 3561.9433302921243}, {"decode_latency_s": 0.06030609374283813, "kv_live_bytes": 462274560, "latency_s": 1.9306367500685155, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72941568, "prompt_kind": "short", "t_relative_s": 3566.9534219172783}, {"decode_latency_s": 0.06827787762449589, "kv_live_bytes": 462274560, "latency_s": 2.1865613753907382, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72712192, "prompt_kind": "long", "t_relative_s": 3571.96352066705}, {"decode_latency_s": 0.06043506249261554, "kv_live_bytes": 462274560, "latency_s": 1.9348064167425036, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73547776, "prompt_kind": "short", "t_relative_s": 3576.9735732502304}, {"decode_latency_s": 0.06739484115678351, "kv_live_bytes": 462274560, "latency_s": 2.1588588752783835, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73728000, "prompt_kind": "long", "t_relative_s": 3581.9836655831896}, {"decode_latency_s": 0.06203888019081205, "kv_live_bytes": 462274560, "latency_s": 1.986158833373338, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72073216, "prompt_kind": "short", "t_relative_s": 3586.993757833261}, {"decode_latency_s": 0.06794765756058041, "kv_live_bytes": 462274560, "latency_s": 2.176030958071351, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73187328, "prompt_kind": "long", "t_relative_s": 3592.0038375421427}, {"decode_latency_s": 0.060526167959324084, "kv_live_bytes": 462274560, "latency_s": 1.9376824996434152, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73285632, "prompt_kind": "short", "t_relative_s": 3597.013902292121}, {"decode_latency_s": 0.06792727214633487, "kv_live_bytes": 462274560, "latency_s": 2.175884374883026, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73367552, "prompt_kind": "long", "t_relative_s": 3602.023975333199}, {"decode_latency_s": 0.060962309915339574, "kv_live_bytes": 462274560, "latency_s": 1.9515700000338256, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73924608, "prompt_kind": "short", "t_relative_s": 3607.0340533331037}, {"decode_latency_s": 0.06817057423177175, "kv_live_bytes": 462274560, "latency_s": 2.183360833209008, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73498624, "prompt_kind": "long", "t_relative_s": 3612.044105750043}, {"decode_latency_s": 0.06738488932023756, "kv_live_bytes": 462274560, "latency_s": 2.1587488329969347, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69058560, "prompt_kind": "short", "t_relative_s": 3617.0541633330286}, {"decode_latency_s": 0.0615160794404801, "kv_live_bytes": 462274560, "latency_s": 1.9695493751205504, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68845568, "prompt_kind": "long", "t_relative_s": 3622.06423874991}, {"decode_latency_s": 0.06044585678318981, "kv_live_bytes": 462274560, "latency_s": 1.935148708987981, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69271552, "prompt_kind": "short", "t_relative_s": 3627.074303708039}, {"decode_latency_s": 0.0682875859492924, "kv_live_bytes": 462274560, "latency_s": 2.1868207091465592, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71827456, "prompt_kind": "long", "t_relative_s": 3632.084367958363}, {"decode_latency_s": 0.06073119920620229, "kv_live_bytes": 462274560, "latency_s": 1.9442323329858482, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72384512, "prompt_kind": "short", "t_relative_s": 3637.094442333095}, {"decode_latency_s": 0.06796595444029663, "kv_live_bytes": 462274560, "latency_s": 2.1773860421963036, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72433664, "prompt_kind": "long", "t_relative_s": 3642.1045163753442}, {"decode_latency_s": 0.06223586459236685, "kv_live_bytes": 462274560, "latency_s": 1.9925566669553518, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70828032, "prompt_kind": "short", "t_relative_s": 3647.114585333038}, {"decode_latency_s": 0.06675421092950273, "kv_live_bytes": 462274560, "latency_s": 2.138224707916379, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72089600, "prompt_kind": "long", "t_relative_s": 3652.12465825025}, {"decode_latency_s": 0.060570091125555336, "kv_live_bytes": 462274560, "latency_s": 1.9390572500415146, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "short", "t_relative_s": 3657.1347363330424}, {"decode_latency_s": 0.06848615493800025, "kv_live_bytes": 462274560, "latency_s": 2.194362792186439, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "long", "t_relative_s": 3662.1447880421765}, {"decode_latency_s": 0.06063169660046697, "kv_live_bytes": 462274560, "latency_s": 1.9409891669638455, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73203712, "prompt_kind": "short", "t_relative_s": 3667.154867417179}, {"decode_latency_s": 0.06821241146826651, "kv_live_bytes": 462274560, "latency_s": 2.184967666864395, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72073216, "prompt_kind": "long", "t_relative_s": 3672.1649191672914}, {"decode_latency_s": 0.06140590492577758, "kv_live_bytes": 462274560, "latency_s": 1.965851916000247, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71122944, "prompt_kind": "short", "t_relative_s": 3677.174998542294}, {"decode_latency_s": 0.06789069659134839, "kv_live_bytes": 462274560, "latency_s": 2.174946791958064, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72105984, "prompt_kind": "long", "t_relative_s": 3682.185054417234}, {"decode_latency_s": 0.06047713803127408, "kv_live_bytes": 462274560, "latency_s": 1.9361756248399615, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72187904, "prompt_kind": "short", "t_relative_s": 3687.195142417215}, {"decode_latency_s": 0.0679823372483952, "kv_live_bytes": 462274560, "latency_s": 2.1770186657086015, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72515584, "prompt_kind": "long", "t_relative_s": 3692.205224458128}, {"decode_latency_s": 0.060612921879510395, "kv_live_bytes": 462274560, "latency_s": 1.9405318750068545, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73105408, "prompt_kind": "short", "t_relative_s": 3697.215269875247}, {"decode_latency_s": 0.06972469273023307, "kv_live_bytes": 462274560, "latency_s": 2.2333737080916762, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69730304, "prompt_kind": "long", "t_relative_s": 3702.2253502500243}, {"decode_latency_s": 0.060413071638322435, "kv_live_bytes": 462274560, "latency_s": 1.9340905416756868, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71794688, "prompt_kind": "short", "t_relative_s": 3707.2354282923043}, {"decode_latency_s": 0.06827319922740571, "kv_live_bytes": 462274560, "latency_s": 2.186426708009094, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72482816, "prompt_kind": "long", "t_relative_s": 3712.245495249983}, {"decode_latency_s": 0.06066615888266824, "kv_live_bytes": 462274560, "latency_s": 1.9421397079713643, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72712192, "prompt_kind": "short", "t_relative_s": 3717.2555470000952}, {"decode_latency_s": 0.06806967579177581, "kv_live_bytes": 462274560, "latency_s": 2.180955708026886, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72794112, "prompt_kind": "long", "t_relative_s": 3722.2656165831722}, {"decode_latency_s": 0.060597006493480876, "kv_live_bytes": 462274560, "latency_s": 1.9398338748142123, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73318400, "prompt_kind": "short", "t_relative_s": 3727.269062791951}, {"decode_latency_s": 0.06847890494100284, "kv_live_bytes": 462274560, "latency_s": 2.1932409163564444, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73416704, "prompt_kind": "long", "t_relative_s": 3732.2791095003486}, {"decode_latency_s": 0.060716795560438186, "kv_live_bytes": 462274560, "latency_s": 1.9439452500082552, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67911680, "prompt_kind": "short", "t_relative_s": 3737.2891844999976}, {"decode_latency_s": 0.06806530729227234, "kv_live_bytes": 462274560, "latency_s": 2.179955916944891, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72597504, "prompt_kind": "long", "t_relative_s": 3742.299250208307}, {"decode_latency_s": 0.06645655207103118, "kv_live_bytes": 462274560, "latency_s": 2.128273040987551, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72695808, "prompt_kind": "short", "t_relative_s": 3747.3092967919074}, {"decode_latency_s": 0.06799022915947717, "kv_live_bytes": 462274560, "latency_s": 2.1782250832766294, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72613888, "prompt_kind": "long", "t_relative_s": 3752.3193796249107}, {"decode_latency_s": 0.06070382290636189, "kv_live_bytes": 462274560, "latency_s": 1.9433303331024945, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73220096, "prompt_kind": "short", "t_relative_s": 3757.329448542092}, {"decode_latency_s": 0.06796785937331151, "kv_live_bytes": 462274560, "latency_s": 2.1770270843990147, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72794112, "prompt_kind": "long", "t_relative_s": 3762.339499874972}, {"decode_latency_s": 0.06033669011958409, "kv_live_bytes": 462274560, "latency_s": 1.9317061668261886, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72335360, "prompt_kind": "short", "t_relative_s": 3767.349570624996}, {"decode_latency_s": 0.06809606509341393, "kv_live_bytes": 462274560, "latency_s": 2.181189457885921, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72826880, "prompt_kind": "long", "t_relative_s": 3772.359646332916}, {"decode_latency_s": 0.06679064320633188, "kv_live_bytes": 462274560, "latency_s": 2.139764499850571, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72925184, "prompt_kind": "short", "t_relative_s": 3777.3696945421398}, {"decode_latency_s": 0.06821716015110724, "kv_live_bytes": 462274560, "latency_s": 2.185600040946156, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72974336, "prompt_kind": "long", "t_relative_s": 3782.379800416995}, {"decode_latency_s": 0.0667516497487668, "kv_live_bytes": 462274560, "latency_s": 2.137841542251408, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73433088, "prompt_kind": "short", "t_relative_s": 3787.38987383293}, {"decode_latency_s": 0.06070575912599452, "kv_live_bytes": 462274560, "latency_s": 1.9439373752102256, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73515008, "prompt_kind": "long", "t_relative_s": 3792.3999226251617}, {"decode_latency_s": 0.06248221483838279, "kv_live_bytes": 462274560, "latency_s": 2.000260249711573, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71286784, "prompt_kind": "short", "t_relative_s": 3797.4099874170497}, {"decode_latency_s": 0.06791104166768491, "kv_live_bytes": 462274560, "latency_s": 2.1755267907865345, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72794112, "prompt_kind": "long", "t_relative_s": 3802.4200422503054}, {"decode_latency_s": 0.06058208075410221, "kv_live_bytes": 462274560, "latency_s": 1.9394077500328422, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72794112, "prompt_kind": "short", "t_relative_s": 3807.430107833352}, {"decode_latency_s": 0.06810017446696293, "kv_live_bytes": 462274560, "latency_s": 2.1807704577222466, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72859648, "prompt_kind": "long", "t_relative_s": 3812.440196292009}, {"decode_latency_s": 0.06028467578289565, "kv_live_bytes": 462274560, "latency_s": 1.9299808330833912, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73416704, "prompt_kind": "short", "t_relative_s": 3817.4502532500774}, {"decode_latency_s": 0.06846458720974624, "kv_live_bytes": 462274560, "latency_s": 2.193135042209178, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73433088, "prompt_kind": "long", "t_relative_s": 3822.460341833066}, {"decode_latency_s": 0.06062678777379915, "kv_live_bytes": 462274560, "latency_s": 1.9408882092684507, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73269248, "prompt_kind": "short", "t_relative_s": 3827.470418542158}, {"decode_latency_s": 0.06779798959905747, "kv_live_bytes": 462274560, "latency_s": 2.1719862911850214, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73367552, "prompt_kind": "long", "t_relative_s": 3832.480469833128}, {"decode_latency_s": 0.06035917706321925, "kv_live_bytes": 462274560, "latency_s": 1.932402249891311, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73285632, "prompt_kind": "short", "t_relative_s": 3837.4905490833335}, {"decode_latency_s": 0.06762599086505361, "kv_live_bytes": 462274560, "latency_s": 2.1656346251256764, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73302016, "prompt_kind": "long", "t_relative_s": 3842.5006217500195}, {"decode_latency_s": 0.06053358984354418, "kv_live_bytes": 462274560, "latency_s": 1.9380069160833955, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "short", "t_relative_s": 3847.5106687080115}, {"decode_latency_s": 0.060912753906450234, "kv_live_bytes": 462274560, "latency_s": 1.9502072501927614, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73547776, "prompt_kind": "long", "t_relative_s": 3852.520746000111}, {"decode_latency_s": 0.060579660144867375, "kv_live_bytes": 462274560, "latency_s": 1.9393646251410246, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73007104, "prompt_kind": "short", "t_relative_s": 3857.5308380001225}, {"decode_latency_s": 0.06722516925947275, "kv_live_bytes": 462274560, "latency_s": 2.153846500441432, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73220096, "prompt_kind": "long", "t_relative_s": 3862.54089395795}, {"decode_latency_s": 0.060279345052549616, "kv_live_bytes": 462274560, "latency_s": 1.9298342079855502, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73285632, "prompt_kind": "short", "t_relative_s": 3867.550980000291}, {"decode_latency_s": 0.06804126431234181, "kv_live_bytes": 462274560, "latency_s": 2.179481790866703, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73203712, "prompt_kind": "long", "t_relative_s": 3872.561071000062}, {"decode_latency_s": 0.06040238149580546, "kv_live_bytes": 462274560, "latency_s": 1.9338055411353707, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73760768, "prompt_kind": "short", "t_relative_s": 3877.5614534579217}, {"decode_latency_s": 0.06875093359849416, "kv_live_bytes": 462274560, "latency_s": 2.2020011246204376, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66912256, "prompt_kind": "long", "t_relative_s": 3882.571531624999}, {"decode_latency_s": 0.060423347662435845, "kv_live_bytes": 462274560, "latency_s": 1.9345156671479344, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71172096, "prompt_kind": "short", "t_relative_s": 3887.5796765829436}, {"decode_latency_s": 0.06791532420902513, "kv_live_bytes": 462274560, "latency_s": 2.1754615837708116, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72351744, "prompt_kind": "long", "t_relative_s": 3892.5897477502003}, {"decode_latency_s": 0.060650621104286984, "kv_live_bytes": 462274560, "latency_s": 1.9418233749456704, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72499200, "prompt_kind": "short", "t_relative_s": 3897.59980416717}, {"decode_latency_s": 0.0682821314985631, "kv_live_bytes": 462274560, "latency_s": 2.18732316698879, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72564736, "prompt_kind": "long", "t_relative_s": 3902.606128375046}, {"decode_latency_s": 0.06051219010259956, "kv_live_bytes": 462274560, "latency_s": 1.9371555829420686, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73154560, "prompt_kind": "short", "t_relative_s": 3907.6162202083506}, {"decode_latency_s": 0.06811256772198249, "kv_live_bytes": 462274560, "latency_s": 2.1819567917846143, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72712192, "prompt_kind": "long", "t_relative_s": 3912.626251167152}, {"decode_latency_s": 0.06255858071381226, "kv_live_bytes": 462274560, "latency_s": 2.002702707890421, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72220672, "prompt_kind": "short", "t_relative_s": 3917.63632445829}, {"decode_latency_s": 0.06825240624311846, "kv_live_bytes": 462274560, "latency_s": 2.186641083098948, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72859648, "prompt_kind": "long", "t_relative_s": 3922.646401999984}, {"decode_latency_s": 0.06053844920825213, "kv_live_bytes": 462274560, "latency_s": 1.938037124928087, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73383936, "prompt_kind": "short", "t_relative_s": 3927.6564831673168}, {"decode_latency_s": 0.06794967578025535, "kv_live_bytes": 462274560, "latency_s": 2.176867707632482, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73302016, "prompt_kind": "long", "t_relative_s": 3932.6666627922095}, {"decode_latency_s": 0.060545639353222214, "kv_live_bytes": 462274560, "latency_s": 1.938222416676581, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73760768, "prompt_kind": "short", "t_relative_s": 3937.6767767919227}, {"decode_latency_s": 0.06342363153817132, "kv_live_bytes": 462274560, "latency_s": 2.0305602503940463, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68894720, "prompt_kind": "long", "t_relative_s": 3942.68682404235}, {"decode_latency_s": 0.060687490869895555, "kv_live_bytes": 462274560, "latency_s": 1.9433827078901231, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69517312, "prompt_kind": "short", "t_relative_s": 3947.696924542077}, {"decode_latency_s": 0.06316327211970929, "kv_live_bytes": 462274560, "latency_s": 2.0230176667682827, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70746112, "prompt_kind": "long", "t_relative_s": 3952.7069691251963}, {"decode_latency_s": 0.060492773438454606, "kv_live_bytes": 462274560, "latency_s": 1.936580709181726, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71237632, "prompt_kind": "short", "t_relative_s": 3957.717063332908}, {"decode_latency_s": 0.06345985157531686, "kv_live_bytes": 462274560, "latency_s": 2.032017458230257, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71319552, "prompt_kind": "long", "t_relative_s": 3962.720172292087}, {"decode_latency_s": 0.060472475248388946, "kv_live_bytes": 462274560, "latency_s": 1.9360606661066413, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71483392, "prompt_kind": "short", "t_relative_s": 3967.7302334173582}, {"decode_latency_s": 0.06836427471716888, "kv_live_bytes": 462274560, "latency_s": 2.1897036670707166, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72040448, "prompt_kind": "long", "t_relative_s": 3972.7403121669777}, {"decode_latency_s": 0.061220703122671694, "kv_live_bytes": 462274560, "latency_s": 1.9599759173579514, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68222976, "prompt_kind": "short", "t_relative_s": 3977.7490353332832}, {"decode_latency_s": 0.06794348437688313, "kv_live_bytes": 462274560, "latency_s": 2.176471082959324, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68796416, "prompt_kind": "long", "t_relative_s": 3982.7590874172747}, {"decode_latency_s": 0.060260688784183, "kv_live_bytes": 462274560, "latency_s": 1.9292601658962667, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69369856, "prompt_kind": "short", "t_relative_s": 3987.76916458318}, {"decode_latency_s": 0.0679630781232845, "kv_live_bytes": 462274560, "latency_s": 2.1764539587311447, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69500928, "prompt_kind": "long", "t_relative_s": 3992.7792428750545}, {"decode_latency_s": 0.06059671874390915, "kv_live_bytes": 462274560, "latency_s": 1.9399394998326898, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70074368, "prompt_kind": "short", "t_relative_s": 3997.789291750174}, {"decode_latency_s": 0.06762521353084594, "kv_live_bytes": 462274560, "latency_s": 2.1667886674404144, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70025216, "prompt_kind": "long", "t_relative_s": 4002.7993799173273}, {"decode_latency_s": 0.06056556379189715, "kv_live_bytes": 462274560, "latency_s": 1.93892833404243, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69566464, "prompt_kind": "short", "t_relative_s": 4007.8094511250965}, {"decode_latency_s": 0.06064263671578374, "kv_live_bytes": 462274560, "latency_s": 1.941553124692291, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69025792, "prompt_kind": "long", "t_relative_s": 4012.819499500096}, {"decode_latency_s": 0.06034044270927552, "kv_live_bytes": 462274560, "latency_s": 1.931819749996066, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69058560, "prompt_kind": "short", "t_relative_s": 4017.8296142923646}, {"decode_latency_s": 0.06824861590575892, "kv_live_bytes": 462274560, "latency_s": 2.1855755411088467, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69697536, "prompt_kind": "long", "t_relative_s": 4022.8396937083453}, {"decode_latency_s": 0.06710165756521747, "kv_live_bytes": 462274560, "latency_s": 2.148978707846254, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70270976, "prompt_kind": "short", "t_relative_s": 4027.8497550832108}, {"decode_latency_s": 0.06758616147271823, "kv_live_bytes": 462274560, "latency_s": 2.1650755829177797, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70402048, "prompt_kind": "long", "t_relative_s": 4032.859842750244}, {"decode_latency_s": 0.06117408331192564, "kv_live_bytes": 462274560, "latency_s": 1.958698958158493, "n_emitted": 32, "ok": true, "process_footprint_bytes": 64913408, "prompt_kind": "short", "t_relative_s": 4037.869948167354}, {"decode_latency_s": 0.06792352214688435, "kv_live_bytes": 462274560, "latency_s": 2.1752957082353532, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67502080, "prompt_kind": "long", "t_relative_s": 4042.8800487923436}, {"decode_latency_s": 0.06698737241094932, "kv_live_bytes": 462274560, "latency_s": 2.145467750262469, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69844992, "prompt_kind": "short", "t_relative_s": 4047.8900945419446}, {"decode_latency_s": 0.067749535155599, "kv_live_bytes": 462274560, "latency_s": 2.1702865827828646, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69926912, "prompt_kind": "long", "t_relative_s": 4052.9001832082868}, {"decode_latency_s": 0.06054736065561883, "kv_live_bytes": 462274560, "latency_s": 1.938283291645348, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70467584, "prompt_kind": "short", "t_relative_s": 4057.91027387511}, {"decode_latency_s": 0.06775684766762424, "kv_live_bytes": 462274560, "latency_s": 2.1704386668279767, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69238784, "prompt_kind": "long", "t_relative_s": 4062.92030183319}, {"decode_latency_s": 0.060428302094805986, "kv_live_bytes": 462274560, "latency_s": 1.9346277923323214, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69140480, "prompt_kind": "short", "t_relative_s": 4067.930375041906}, {"decode_latency_s": 0.06780017448181752, "kv_live_bytes": 462274560, "latency_s": 2.1714514158666134, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69861376, "prompt_kind": "long", "t_relative_s": 4072.940449083224}, {"decode_latency_s": 0.06052116534556262, "kv_live_bytes": 462274560, "latency_s": 1.9375864998437464, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70549504, "prompt_kind": "short", "t_relative_s": 4077.950146582909}, {"decode_latency_s": 0.06822515366366133, "kv_live_bytes": 462274560, "latency_s": 2.1853308747522533, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70582272, "prompt_kind": "long", "t_relative_s": 4082.9602424171753}, {"decode_latency_s": 0.060783934910432436, "kv_live_bytes": 462274560, "latency_s": 1.9459531656466424, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71172096, "prompt_kind": "short", "t_relative_s": 4087.9703230420128}, {"decode_latency_s": 0.06834057680680417, "kv_live_bytes": 462274560, "latency_s": 2.189270915929228, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70713344, "prompt_kind": "long", "t_relative_s": 4092.980375250336}, {"decode_latency_s": 0.0608038593782112, "kv_live_bytes": 462274560, "latency_s": 1.9465764998458326, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70270976, "prompt_kind": "short", "t_relative_s": 4097.990453124978}, {"decode_latency_s": 0.06875907942594495, "kv_live_bytes": 462274560, "latency_s": 2.201918624807149, "n_emitted": 32, "ok": true, "process_footprint_bytes": 64159744, "prompt_kind": "long", "t_relative_s": 4103.000535542145}, {"decode_latency_s": 0.0605662903690245, "kv_live_bytes": 462274560, "latency_s": 1.9390037921257317, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69828608, "prompt_kind": "short", "t_relative_s": 4108.010608917102}, {"decode_latency_s": 0.06045770311902743, "kv_live_bytes": 462274560, "latency_s": 1.9356391252949834, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70565888, "prompt_kind": "long", "t_relative_s": 4113.020661542192}, {"decode_latency_s": 0.06016007161815651, "kv_live_bytes": 462274560, "latency_s": 1.9260366247035563, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70877184, "prompt_kind": "short", "t_relative_s": 4118.030733792111}, {"decode_latency_s": 0.06754937890218571, "kv_live_bytes": 462274560, "latency_s": 2.1633440000005066, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71041024, "prompt_kind": "long", "t_relative_s": 4123.040815582965}, {"decode_latency_s": 0.06669795575726312, "kv_live_bytes": 462274560, "latency_s": 2.136203750036657, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71172096, "prompt_kind": "short", "t_relative_s": 4128.050886500161}, {"decode_latency_s": 0.06776955469103996, "kv_live_bytes": 462274560, "latency_s": 2.1708814576268196, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71303168, "prompt_kind": "long", "t_relative_s": 4133.060937125236}, {"decode_latency_s": 0.06049950000306126, "kv_live_bytes": 462274560, "latency_s": 1.9367447919212282, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71860224, "prompt_kind": "short", "t_relative_s": 4138.07103254227}, {"decode_latency_s": 0.06832872916129418, "kv_live_bytes": 462274560, "latency_s": 2.188623874913901, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71794688, "prompt_kind": "long", "t_relative_s": 4143.081087667029}, {"decode_latency_s": 0.06038501561852172, "kv_live_bytes": 462274560, "latency_s": 1.933244540821761, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71254016, "prompt_kind": "short", "t_relative_s": 4148.091154499911}, {"decode_latency_s": 0.0683900286530843, "kv_live_bytes": 462274560, "latency_s": 2.190195791888982, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71942144, "prompt_kind": "long", "t_relative_s": 4153.1012443332}, {"decode_latency_s": 0.060733173188054934, "kv_live_bytes": 462274560, "latency_s": 1.9443350001238286, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72007680, "prompt_kind": "short", "t_relative_s": 4158.111324542202}, {"decode_latency_s": 0.06050502083962783, "kv_live_bytes": 462274560, "latency_s": 1.9374538753181696, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72089600, "prompt_kind": "long", "t_relative_s": 4163.121386917308}, {"decode_latency_s": 0.06056126953626517, "kv_live_bytes": 462274560, "latency_s": 1.9387357500381768, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72630272, "prompt_kind": "short", "t_relative_s": 4168.131463458296}, {"decode_latency_s": 0.06790064585220534, "kv_live_bytes": 462274560, "latency_s": 2.175157249905169, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72548352, "prompt_kind": "long", "t_relative_s": 4173.141519207973}, {"decode_latency_s": 0.06024289193737786, "kv_live_bytes": 462274560, "latency_s": 1.9286755411885679, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72630272, "prompt_kind": "short", "t_relative_s": 4178.151583833154}, {"decode_latency_s": 0.06838235545728821, "kv_live_bytes": 462274560, "latency_s": 2.1895529995672405, "n_emitted": 32, "ok": true, "process_footprint_bytes": 63750144, "prompt_kind": "long", "t_relative_s": 4183.161653958261}, {"decode_latency_s": 0.061479433585191146, "kv_live_bytes": 462274560, "latency_s": 1.9691693750210106, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66912256, "prompt_kind": "short", "t_relative_s": 4188.171719292179}, {"decode_latency_s": 0.06813603515911382, "kv_live_bytes": 462274560, "latency_s": 2.182399165816605, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70746112, "prompt_kind": "long", "t_relative_s": 4193.181770833209}, {"decode_latency_s": 0.06038380469544791, "kv_live_bytes": 462274560, "latency_s": 1.9332510828971863, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70533120, "prompt_kind": "short", "t_relative_s": 4198.19183575036}, {"decode_latency_s": 0.060657394540612586, "kv_live_bytes": 462274560, "latency_s": 1.9419332090765238, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71188480, "prompt_kind": "long", "t_relative_s": 4203.201912167016}, {"decode_latency_s": 0.06035475652606692, "kv_live_bytes": 462274560, "latency_s": 1.932243041228503, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71729152, "prompt_kind": "short", "t_relative_s": 4208.211959083099}, {"decode_latency_s": 0.06096994399558753, "kv_live_bytes": 462274560, "latency_s": 1.952211958821863, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71778304, "prompt_kind": "long", "t_relative_s": 4213.222044917289}, {"decode_latency_s": 0.06358077343611512, "kv_live_bytes": 462274560, "latency_s": 2.035445415880531, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67092480, "prompt_kind": "short", "t_relative_s": 4218.232098875102}, {"decode_latency_s": 0.06796358594147023, "kv_live_bytes": 462274560, "latency_s": 2.176516542211175, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68091904, "prompt_kind": "long", "t_relative_s": 4224.506952208001}, {"decode_latency_s": 0.06078688151319511, "kv_live_bytes": 462274560, "latency_s": 1.946005207952112, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68747264, "prompt_kind": "short", "t_relative_s": 4229.5170046249405}, {"decode_latency_s": 0.06938032811740413, "kv_live_bytes": 462274560, "latency_s": 2.222579791676253, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68878336, "prompt_kind": "long", "t_relative_s": 4234.527086667251}, {"decode_latency_s": 0.06062459503300488, "kv_live_bytes": 462274560, "latency_s": 1.9408892076462507, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69402624, "prompt_kind": "short", "t_relative_s": 4239.537164083216}, {"decode_latency_s": 0.0680410664062947, "kv_live_bytes": 462274560, "latency_s": 2.1796668749302626, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66273280, "prompt_kind": "long", "t_relative_s": 4244.547228333075}, {"decode_latency_s": 0.06753829296212643, "kv_live_bytes": 462274560, "latency_s": 2.162589999847114, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67403776, "prompt_kind": "short", "t_relative_s": 4249.5573100419715}, {"decode_latency_s": 0.06591605990251992, "kv_live_bytes": 462274560, "latency_s": 2.111082249786705, "n_emitted": 32, "ok": true, "process_footprint_bytes": 55984128, "prompt_kind": "long", "t_relative_s": 4254.567390167154}, {"decode_latency_s": 0.06052566929429304, "kv_live_bytes": 462274560, "latency_s": 1.9377087918110192, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68501504, "prompt_kind": "short", "t_relative_s": 4259.577456166968}, {"decode_latency_s": 0.06852608331246302, "kv_live_bytes": 462274560, "latency_s": 2.1948546660132706, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69107712, "prompt_kind": "long", "t_relative_s": 4264.587508042343}, {"decode_latency_s": 0.060349276041961275, "kv_live_bytes": 462274560, "latency_s": 1.9321151250042021, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69107712, "prompt_kind": "short", "t_relative_s": 4269.59757958306}, {"decode_latency_s": 0.06897288412437774, "kv_live_bytes": 462274560, "latency_s": 2.2087374590337276, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67207168, "prompt_kind": "long", "t_relative_s": 4274.607655792031}, {"decode_latency_s": 0.06053803252871148, "kv_live_bytes": 462274560, "latency_s": 1.938073417171836, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68681728, "prompt_kind": "short", "t_relative_s": 4279.617725375108}, {"decode_latency_s": 0.06829715624917299, "kv_live_bytes": 462274560, "latency_s": 2.187232125084847, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68485120, "prompt_kind": "long", "t_relative_s": 4284.627799583133}, {"decode_latency_s": 0.06063508331135381, "kv_live_bytes": 462274560, "latency_s": 1.9411152079701424, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69173248, "prompt_kind": "short", "t_relative_s": 4289.637881624978}, {"decode_latency_s": 0.06824565235001501, "kv_live_bytes": 462274560, "latency_s": 2.1859375843778253, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69713920, "prompt_kind": "long", "t_relative_s": 4294.647916375194}, {"decode_latency_s": 0.0604976875038119, "kv_live_bytes": 462274560, "latency_s": 1.9368806667625904, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69812224, "prompt_kind": "short", "t_relative_s": 4299.657994417008}, {"decode_latency_s": 0.06823803514998872, "kv_live_bytes": 462274560, "latency_s": 2.185472166631371, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66355200, "prompt_kind": "long", "t_relative_s": 4304.6680698753335}, {"decode_latency_s": 0.06117926952720154, "kv_live_bytes": 462274560, "latency_s": 1.9586551669053733, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66224128, "prompt_kind": "short", "t_relative_s": 4309.67814195808}, {"decode_latency_s": 0.06846331640554126, "kv_live_bytes": 462274560, "latency_s": 2.1931947502307594, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69320704, "prompt_kind": "long", "t_relative_s": 4314.688212583307}, {"decode_latency_s": 0.060309967448120005, "kv_live_bytes": 462274560, "latency_s": 1.930869959294796, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68632576, "prompt_kind": "short", "t_relative_s": 4319.698306125123}, {"decode_latency_s": 0.06852156380773522, "kv_live_bytes": 462274560, "latency_s": 2.1952043338678777, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69566464, "prompt_kind": "long", "t_relative_s": 4324.708349042106}, {"decode_latency_s": 0.060624772115261294, "kv_live_bytes": 462274560, "latency_s": 1.940837583038956, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70238208, "prompt_kind": "short", "t_relative_s": 4329.7184326671995}, {"decode_latency_s": 0.06820776821405161, "kv_live_bytes": 462274560, "latency_s": 2.184940207749605, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67682304, "prompt_kind": "long", "t_relative_s": 4334.7192451669835}, {"decode_latency_s": 0.060326619786792435, "kv_live_bytes": 462274560, "latency_s": 1.9314792081713676, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68960256, "prompt_kind": "short", "t_relative_s": 4339.729330083355}, {"decode_latency_s": 0.06788668621447869, "kv_live_bytes": 462274560, "latency_s": 2.1739632501266897, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69746688, "prompt_kind": "long", "t_relative_s": 4344.739397374913}, {"decode_latency_s": 0.06091179166105576, "kv_live_bytes": 462274560, "latency_s": 1.9502930003218353, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69894144, "prompt_kind": "short", "t_relative_s": 4349.748921917286}, {"decode_latency_s": 0.0684197317896178, "kv_live_bytes": 462274560, "latency_s": 2.191493249963969, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70500352, "prompt_kind": "long", "t_relative_s": 4354.759009582922}, {"decode_latency_s": 0.06341457812231965, "kv_live_bytes": 462274560, "latency_s": 2.030197416432202, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69550080, "prompt_kind": "short", "t_relative_s": 4359.769092292059}, {"decode_latency_s": 0.09239275781146716, "kv_live_bytes": 462274560, "latency_s": 2.958213457837701, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67600384, "prompt_kind": "long", "t_relative_s": 4364.779165875167}, {"decode_latency_s": 0.0604607239656616, "kv_live_bytes": 462274560, "latency_s": 1.935711707919836, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67289088, "prompt_kind": "short", "t_relative_s": 4373.359112458304}, {"decode_latency_s": 0.24298020702553913, "kv_live_bytes": 462274560, "latency_s": 7.776230625342578, "n_emitted": 32, "ok": true, "process_footprint_bytes": 65847296, "prompt_kind": "long", "t_relative_s": 4378.3691513752565}, {"decode_latency_s": 0.06284499086905271, "kv_live_bytes": 462274560, "latency_s": 2.01195729104802, "n_emitted": 32, "ok": true, "process_footprint_bytes": 63455232, "prompt_kind": "short", "t_relative_s": 4392.363406875171}, {"decode_latency_s": 0.07005913149623666, "kv_live_bytes": 462274560, "latency_s": 2.2440097499638796, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66813952, "prompt_kind": "long", "t_relative_s": 4397.373488333076}, {"decode_latency_s": 0.061126451852032915, "kv_live_bytes": 462274560, "latency_s": 1.9569881660863757, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67633152, "prompt_kind": "short", "t_relative_s": 4402.3835432920605}, {"decode_latency_s": 0.064603037739289, "kv_live_bytes": 462274560, "latency_s": 2.068798207677901, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67469312, "prompt_kind": "long", "t_relative_s": 4407.393615167122}, {"decode_latency_s": 0.06115463409514632, "kv_live_bytes": 462274560, "latency_s": 1.9578917496837676, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67616768, "prompt_kind": "short", "t_relative_s": 4413.109373582993}, {"decode_latency_s": 0.06963876821100712, "kv_live_bytes": 462274560, "latency_s": 2.2305773328989744, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67616768, "prompt_kind": "long", "t_relative_s": 4418.119447917212}, {"decode_latency_s": 0.17310977083980106, "kv_live_bytes": 462274560, "latency_s": 5.540438541676849, "n_emitted": 32, "ok": true, "process_footprint_bytes": 63045632, "prompt_kind": "short", "t_relative_s": 4423.123319041915}, {"decode_latency_s": 0.061055473968735896, "kv_live_bytes": 462274560, "latency_s": 1.9548014998435974, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66338816, "prompt_kind": "long", "t_relative_s": 4434.236272375099}, {"decode_latency_s": 0.060609307285631076, "kv_live_bytes": 462274560, "latency_s": 1.940318957902491, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66994176, "prompt_kind": "short", "t_relative_s": 4439.246329166926}, {"decode_latency_s": 0.06795966146455612, "kv_live_bytes": 462274560, "latency_s": 2.177657708991319, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67649536, "prompt_kind": "long", "t_relative_s": 4444.256408541929}, {"decode_latency_s": 0.06060444012109656, "kv_live_bytes": 462274560, "latency_s": 1.9401070838794112, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68222976, "prompt_kind": "short", "t_relative_s": 4449.266499958001}, {"decode_latency_s": 0.06706480600405484, "kv_live_bytes": 462274560, "latency_s": 2.147910540923476, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68321280, "prompt_kind": "long", "t_relative_s": 4454.276554542128}, {"decode_latency_s": 0.060436285159084946, "kv_live_bytes": 462274560, "latency_s": 1.9348565000109375, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68943872, "prompt_kind": "short", "t_relative_s": 4459.2866402082145}, {"decode_latency_s": 0.06740111201361287, "kv_live_bytes": 462274560, "latency_s": 2.1590919997543097, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69042176, "prompt_kind": "long", "t_relative_s": 4464.296722417232}, {"decode_latency_s": 0.06049453516607173, "kv_live_bytes": 462274560, "latency_s": 1.9367648749612272, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69091328, "prompt_kind": "short", "t_relative_s": 4469.306793082971}, {"decode_latency_s": 0.06773041017004289, "kv_live_bytes": 462274560, "latency_s": 2.170031958259642, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69173248, "prompt_kind": "long", "t_relative_s": 4474.316861167084}, {"decode_latency_s": 0.0606467304751277, "kv_live_bytes": 462274560, "latency_s": 1.9414418330416083, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69222400, "prompt_kind": "short", "t_relative_s": 4479.326941583306}, {"decode_latency_s": 0.06514809114742093, "kv_live_bytes": 462274560, "latency_s": 2.0875124158337712, "n_emitted": 32, "ok": true, "process_footprint_bytes": 62717952, "prompt_kind": "long", "t_relative_s": 4484.336986250244}, {"decode_latency_s": 0.06044305078103207, "kv_live_bytes": 462274560, "latency_s": 1.9351804167963564, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66797568, "prompt_kind": "short", "t_relative_s": 4489.347065417096}, {"decode_latency_s": 0.06778677602414973, "kv_live_bytes": 462274560, "latency_s": 2.171178916003555, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68567040, "prompt_kind": "long", "t_relative_s": 4494.357146000024}, {"decode_latency_s": 0.060279522120254114, "kv_live_bytes": 462274560, "latency_s": 1.9298926671035588, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69238784, "prompt_kind": "short", "t_relative_s": 4499.367192458361}, {"decode_latency_s": 0.06042029298259877, "kv_live_bytes": 462274560, "latency_s": 1.9344202089123428, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69173248, "prompt_kind": "long", "t_relative_s": 4504.377269000281}, {"decode_latency_s": 0.06060829556372482, "kv_live_bytes": 462274560, "latency_s": 1.940320833120495, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69304320, "prompt_kind": "short", "t_relative_s": 4509.387341542169}, {"decode_latency_s": 0.06797465366253164, "kv_live_bytes": 462274560, "latency_s": 2.17733516683802, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68960256, "prompt_kind": "long", "t_relative_s": 4514.397388082929}, {"decode_latency_s": 0.0603776054631453, "kv_live_bytes": 462274560, "latency_s": 1.9329579998739064, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69533696, "prompt_kind": "short", "t_relative_s": 4519.407463292126}, {"decode_latency_s": 0.06805113413429353, "kv_live_bytes": 462274560, "latency_s": 2.1797797922044992, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69500928, "prompt_kind": "long", "t_relative_s": 4524.417542667128}, {"decode_latency_s": 0.06031271615938749, "kv_live_bytes": 462274560, "latency_s": 1.9309281660243869, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70107136, "prompt_kind": "short", "t_relative_s": 4529.427584375255}, {"decode_latency_s": 0.06871302083891351, "kv_live_bytes": 462274560, "latency_s": 2.201157667208463, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70156288, "prompt_kind": "long", "t_relative_s": 4534.437665417325}, {"decode_latency_s": 0.0671171354042599, "kv_live_bytes": 462274560, "latency_s": 2.1496181660331786, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70254592, "prompt_kind": "short", "t_relative_s": 4539.447776500136}, {"decode_latency_s": 0.13361867055937182, "kv_live_bytes": 462274560, "latency_s": 4.276869124732912, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67649536, "prompt_kind": "long", "t_relative_s": 4544.457805417012}, {"decode_latency_s": 0.061105222659534775, "kv_live_bytes": 462274560, "latency_s": 1.9567968752235174, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68370432, "prompt_kind": "short", "t_relative_s": 4551.509409375023}, {"decode_latency_s": 0.06805314715893473, "kv_live_bytes": 462274560, "latency_s": 2.1798327080905437, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68173824, "prompt_kind": "long", "t_relative_s": 4556.519503167365}, {"decode_latency_s": 0.060454320308053866, "kv_live_bytes": 462274560, "latency_s": 1.9355002921074629, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68173824, "prompt_kind": "short", "t_relative_s": 4561.52955300035}, {"decode_latency_s": 0.0682732239656616, "kv_live_bytes": 462274560, "latency_s": 2.186857624910772, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68927488, "prompt_kind": "long", "t_relative_s": 4566.539630792104}, {"decode_latency_s": 0.06057993878494017, "kv_live_bytes": 462274560, "latency_s": 1.9393163747154176, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68993024, "prompt_kind": "short", "t_relative_s": 4571.54969691718}, {"decode_latency_s": 0.0682582695299061, "kv_live_bytes": 462274560, "latency_s": 2.1867134999483824, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68435968, "prompt_kind": "long", "t_relative_s": 4576.5597512503155}, {"decode_latency_s": 0.06051299086539075, "kv_live_bytes": 462274560, "latency_s": 1.9371957089751959, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68993024, "prompt_kind": "short", "t_relative_s": 4581.569822667167}, {"decode_latency_s": 0.060844457024359144, "kv_live_bytes": 462274560, "latency_s": 1.9482400002889335, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68960256, "prompt_kind": "long", "t_relative_s": 4586.579881292302}, {"decode_latency_s": 0.06045249218004756, "kv_live_bytes": 462274560, "latency_s": 1.9354270417243242, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67993600, "prompt_kind": "short", "t_relative_s": 4591.589910292067}, {"decode_latency_s": 0.06835595965094399, "kv_live_bytes": 462274560, "latency_s": 2.189274291973561, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68829184, "prompt_kind": "long", "t_relative_s": 4596.599993250333}, {"decode_latency_s": 0.06059261977497954, "kv_live_bytes": 462274560, "latency_s": 1.939782917033881, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68861952, "prompt_kind": "short", "t_relative_s": 4601.610046749935}, {"decode_latency_s": 0.06824986197170801, "kv_live_bytes": 462274560, "latency_s": 2.1866881670430303, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68681728, "prompt_kind": "long", "t_relative_s": 4606.620137500111}, {"decode_latency_s": 0.0666193060023943, "kv_live_bytes": 462274560, "latency_s": 2.133578124921769, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69222400, "prompt_kind": "short", "t_relative_s": 4611.630212417338}, {"decode_latency_s": 0.06081228515540715, "kv_live_bytes": 462274560, "latency_s": 1.9474863749928772, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69287936, "prompt_kind": "long", "t_relative_s": 4616.640253541991}, {"decode_latency_s": 0.06041073828237131, "kv_live_bytes": 462274560, "latency_s": 1.9340131669305265, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69189632, "prompt_kind": "short", "t_relative_s": 4621.650332124904}, {"decode_latency_s": 0.06817592187144328, "kv_live_bytes": 462274560, "latency_s": 2.183285417035222, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69189632, "prompt_kind": "long", "t_relative_s": 4626.6603944581}, {"decode_latency_s": 0.06058508464775514, "kv_live_bytes": 462274560, "latency_s": 1.9396117078140378, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69287936, "prompt_kind": "short", "t_relative_s": 4631.670443207957}, {"decode_latency_s": 0.06769412763242144, "kv_live_bytes": 462274560, "latency_s": 2.168645082972944, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69255168, "prompt_kind": "long", "t_relative_s": 4636.680529708043}, {"decode_latency_s": 0.06076895572186913, "kv_live_bytes": 462274560, "latency_s": 1.945475374814123, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69844992, "prompt_kind": "short", "t_relative_s": 4641.6906420420855}, {"decode_latency_s": 0.06827145049464889, "kv_live_bytes": 462274560, "latency_s": 2.1871984591707587, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69910528, "prompt_kind": "long", "t_relative_s": 4646.700703167357}, {"decode_latency_s": 0.06037577733513899, "kv_live_bytes": 462274560, "latency_s": 1.9329420421272516, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70451200, "prompt_kind": "short", "t_relative_s": 4651.710790333338}, {"decode_latency_s": 0.06815442060178611, "kv_live_bytes": 462274560, "latency_s": 2.1830195407383144, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70369280, "prompt_kind": "long", "t_relative_s": 4656.720875957981}, {"decode_latency_s": 0.06114270049147308, "kv_live_bytes": 462274560, "latency_s": 1.9574724999256432, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70516736, "prompt_kind": "short", "t_relative_s": 4661.730963333044}, {"decode_latency_s": 0.06862086978799198, "kv_live_bytes": 462274560, "latency_s": 2.19819433381781, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70516736, "prompt_kind": "long", "t_relative_s": 4666.741033167113}, {"decode_latency_s": 0.06071917319786735, "kv_live_bytes": 462274560, "latency_s": 1.9438328328542411, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70057984, "prompt_kind": "short", "t_relative_s": 4671.751147292089}, {"decode_latency_s": 0.0681587630242575, "kv_live_bytes": 462274560, "latency_s": 2.18379120901227, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70598656, "prompt_kind": "long", "t_relative_s": 4676.761200750247}, {"decode_latency_s": 0.06038513020030223, "kv_live_bytes": 462274560, "latency_s": 1.9332144162617624, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70991872, "prompt_kind": "short", "t_relative_s": 4681.771285833325}, {"decode_latency_s": 0.06791740492917597, "kv_live_bytes": 462274560, "latency_s": 2.1755690001882613, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71090176, "prompt_kind": "long", "t_relative_s": 4686.781365333125}, {"decode_latency_s": 0.06687057031376753, "kv_live_bytes": 462274560, "latency_s": 2.1415487499907613, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71188480, "prompt_kind": "short", "t_relative_s": 4691.791419917252}, {"decode_latency_s": 0.06787540625373367, "kv_live_bytes": 462274560, "latency_s": 2.1743784160353243, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70664192, "prompt_kind": "long", "t_relative_s": 4696.801509708166}, {"decode_latency_s": 0.06059817709319759, "kv_live_bytes": 462274560, "latency_s": 1.9399295831099153, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71237632, "prompt_kind": "short", "t_relative_s": 4701.811595333274}, {"decode_latency_s": 0.06858668618951924, "kv_live_bytes": 462274560, "latency_s": 2.19701504195109, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71254016, "prompt_kind": "long", "t_relative_s": 4706.821649417281}, {"decode_latency_s": 0.06030108334380202, "kv_live_bytes": 462274560, "latency_s": 1.9305298337712884, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71778304, "prompt_kind": "short", "t_relative_s": 4711.83172116708}, {"decode_latency_s": 0.060404626303352416, "kv_live_bytes": 462274560, "latency_s": 1.9338352913036942, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71139328, "prompt_kind": "long", "t_relative_s": 4716.83742854232}, {"decode_latency_s": 0.06039355204848107, "kv_live_bytes": 462274560, "latency_s": 1.9335197499021888, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70565888, "prompt_kind": "short", "t_relative_s": 4721.847481958102}, {"decode_latency_s": 0.06056170574447606, "kv_live_bytes": 462274560, "latency_s": 1.9392929170280695, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70909952, "prompt_kind": "long", "t_relative_s": 4726.857548958156}, {"decode_latency_s": 0.06675348959106486, "kv_live_bytes": 462274560, "latency_s": 2.137522667180747, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71565312, "prompt_kind": "short", "t_relative_s": 4731.867609791923}, {"decode_latency_s": 0.06744516143226065, "kv_live_bytes": 462274560, "latency_s": 2.160501040983945, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71680000, "prompt_kind": "long", "t_relative_s": 4736.877668375149}, {"decode_latency_s": 0.0604398632858647, "kv_live_bytes": 462274560, "latency_s": 1.9348346251063049, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72171520, "prompt_kind": "short", "t_relative_s": 4741.881665583234}, {"decode_latency_s": 0.068340677084052, "kv_live_bytes": 462274560, "latency_s": 2.1887551662512124, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70615040, "prompt_kind": "long", "t_relative_s": 4746.884756166954}, {"decode_latency_s": 0.0602996875095414, "kv_live_bytes": 462274560, "latency_s": 1.9306397079490125, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70238208, "prompt_kind": "short", "t_relative_s": 4751.8948022080585}, {"decode_latency_s": 0.0685246224020375, "kv_live_bytes": 462274560, "latency_s": 2.1952765407040715, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71237632, "prompt_kind": "long", "t_relative_s": 4756.904886208009}, {"decode_latency_s": 0.06672046224412043, "kv_live_bytes": 462274560, "latency_s": 2.1367073338478804, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72220672, "prompt_kind": "short", "t_relative_s": 4761.914943417069}, {"decode_latency_s": 0.06781036978645716, "kv_live_bytes": 462274560, "latency_s": 2.1723663341253996, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72187904, "prompt_kind": "long", "t_relative_s": 4766.92501100013}, {"decode_latency_s": 0.06056901431293227, "kv_live_bytes": 462274560, "latency_s": 1.9390004999004304, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "short", "t_relative_s": 4771.935098000336}, {"decode_latency_s": 0.0684082903462695, "kv_live_bytes": 462274560, "latency_s": 2.190709416754544, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72400896, "prompt_kind": "long", "t_relative_s": 4776.94516670797}, {"decode_latency_s": 0.06057929297094233, "kv_live_bytes": 462274560, "latency_s": 1.939426500350237, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72482816, "prompt_kind": "short", "t_relative_s": 4781.951034083031}, {"decode_latency_s": 0.06340114974591415, "kv_live_bytes": 462274560, "latency_s": 2.0298934588208795, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69844992, "prompt_kind": "long", "t_relative_s": 4786.961101583205}, {"decode_latency_s": 0.06038183852797374, "kv_live_bytes": 462274560, "latency_s": 1.93313662474975, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69926912, "prompt_kind": "short", "t_relative_s": 4793.065678250045}, {"decode_latency_s": 0.06804364843992516, "kv_live_bytes": 462274560, "latency_s": 2.1793563752435148, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69910528, "prompt_kind": "long", "t_relative_s": 4798.075747667346}, {"decode_latency_s": 0.0606545377522707, "kv_live_bytes": 462274560, "latency_s": 1.9418526249937713, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70483968, "prompt_kind": "short", "t_relative_s": 4803.0858117919415}, {"decode_latency_s": 0.06800217709678691, "kv_live_bytes": 462274560, "latency_s": 2.1781224170699716, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70516736, "prompt_kind": "long", "t_relative_s": 4808.095872667152}, {"decode_latency_s": 0.060612475252128206, "kv_live_bytes": 462274560, "latency_s": 1.9403772503137589, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70565888, "prompt_kind": "short", "t_relative_s": 4813.105968875345}, {"decode_latency_s": 0.06825047655729577, "kv_live_bytes": 462274560, "latency_s": 2.186089209280908, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70303744, "prompt_kind": "long", "t_relative_s": 4818.1160246669315}, {"decode_latency_s": 0.06032723959651776, "kv_live_bytes": 462274560, "latency_s": 1.9313323749229312, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70909952, "prompt_kind": "short", "t_relative_s": 4823.126103499904}, {"decode_latency_s": 0.06829401825962123, "kv_live_bytes": 462274560, "latency_s": 2.1871190830133855, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71024640, "prompt_kind": "long", "t_relative_s": 4828.136186833028}, {"decode_latency_s": 0.0606272135337349, "kv_live_bytes": 462274560, "latency_s": 1.9414509576745331, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71090176, "prompt_kind": "short", "t_relative_s": 4833.146240375005}, {"decode_latency_s": 0.06841318230726756, "kv_live_bytes": 462274560, "latency_s": 2.191872874740511, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71172096, "prompt_kind": "long", "t_relative_s": 4838.156334000174}, {"decode_latency_s": 0.0612777890637517, "kv_live_bytes": 462274560, "latency_s": 1.9617392080835998, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70303744, "prompt_kind": "short", "t_relative_s": 4843.166411207989}, {"decode_latency_s": 0.06822548827040009, "kv_live_bytes": 462274560, "latency_s": 2.185636874753982, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70172672, "prompt_kind": "long", "t_relative_s": 4848.176464875229}, {"decode_latency_s": 0.060646917976555414, "kv_live_bytes": 462274560, "latency_s": 1.9416298330761492, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70057984, "prompt_kind": "short", "t_relative_s": 4853.18653549999}, {"decode_latency_s": 0.06773027864983305, "kv_live_bytes": 462274560, "latency_s": 2.169249958358705, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70467584, "prompt_kind": "long", "t_relative_s": 4858.196608541999}, {"decode_latency_s": 0.06042998826887924, "kv_live_bytes": 462274560, "latency_s": 1.9354559159837663, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71139328, "prompt_kind": "short", "t_relative_s": 4863.206659000367}, {"decode_latency_s": 0.06749762628169265, "kv_live_bytes": 462274560, "latency_s": 2.1619292497634888, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71172096, "prompt_kind": "long", "t_relative_s": 4868.216749708168}, {"decode_latency_s": 0.06074832293961663, "kv_live_bytes": 462274560, "latency_s": 1.9447468328289688, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71303168, "prompt_kind": "short", "t_relative_s": 4873.226846167352}, {"decode_latency_s": 0.06836746483168099, "kv_live_bytes": 462274560, "latency_s": 2.19005358312279, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71270400, "prompt_kind": "long", "t_relative_s": 4878.2368795829825}, {"decode_latency_s": 0.06041995050327387, "kv_live_bytes": 462274560, "latency_s": 1.9343196251429617, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71876608, "prompt_kind": "short", "t_relative_s": 4883.246962750331}, {"decode_latency_s": 0.06062530599592719, "kv_live_bytes": 462274560, "latency_s": 1.9412900409661233, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71401472, "prompt_kind": "long", "t_relative_s": 4888.257039583288}, {"decode_latency_s": 0.06038472396903671, "kv_live_bytes": 462274560, "latency_s": 1.9331955411471426, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71974912, "prompt_kind": "short", "t_relative_s": 4893.267093792092}, {"decode_latency_s": 0.06827966016135179, "kv_live_bytes": 462274560, "latency_s": 2.186707832850516, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72089600, "prompt_kind": "long", "t_relative_s": 4898.277218125295}, {"decode_latency_s": 0.06080744792416226, "kv_live_bytes": 462274560, "latency_s": 1.9466319167986512, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72138752, "prompt_kind": "short", "t_relative_s": 4903.28731620824}, {"decode_latency_s": 0.06780675781192258, "kv_live_bytes": 462274560, "latency_s": 2.1724208327941597, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71712768, "prompt_kind": "long", "t_relative_s": 4908.297371292021}, {"decode_latency_s": 0.06046383985085413, "kv_live_bytes": 462274560, "latency_s": 1.935690165963024, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71827456, "prompt_kind": "short", "t_relative_s": 4913.307451167144}, {"decode_latency_s": 0.06764483853476122, "kv_live_bytes": 462274560, "latency_s": 2.1663628332316875, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71581696, "prompt_kind": "long", "t_relative_s": 4918.31753608305}, {"decode_latency_s": 0.06027386590722017, "kv_live_bytes": 462274560, "latency_s": 1.9296706253662705, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71614464, "prompt_kind": "short", "t_relative_s": 4923.3275832082145}, {"decode_latency_s": 0.06794851433369331, "kv_live_bytes": 462274560, "latency_s": 2.1761529999785125, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72237056, "prompt_kind": "long", "t_relative_s": 4928.337666708045}, {"decode_latency_s": 0.06042721224366687, "kv_live_bytes": 462274560, "latency_s": 1.9344794591888785, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72318976, "prompt_kind": "short", "t_relative_s": 4933.347723999992}, {"decode_latency_s": 0.06790122266102117, "kv_live_bytes": 462274560, "latency_s": 2.175185250118375, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71712768, "prompt_kind": "long", "t_relative_s": 4938.3577764579095}, {"decode_latency_s": 0.06041820443351753, "kv_live_bytes": 462274560, "latency_s": 1.9341715001501143, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71745536, "prompt_kind": "short", "t_relative_s": 4943.367850417271}, {"decode_latency_s": 0.0681297590926988, "kv_live_bytes": 462274560, "latency_s": 2.1817848328500986, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71401472, "prompt_kind": "long", "t_relative_s": 4948.3754688752815}, {"decode_latency_s": 0.06027451693080366, "kv_live_bytes": 462274560, "latency_s": 1.9297515000216663, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71876608, "prompt_kind": "short", "t_relative_s": 4953.385514582973}, {"decode_latency_s": 0.06818188281613402, "kv_live_bytes": 462274560, "latency_s": 2.184057124890387, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71991296, "prompt_kind": "long", "t_relative_s": 4958.395587624982}, {"decode_latency_s": 0.06785534633672796, "kv_live_bytes": 462274560, "latency_s": 2.173623625189066, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72056832, "prompt_kind": "short", "t_relative_s": 4963.405660542194}, {"decode_latency_s": 0.06767837499501184, "kv_live_bytes": 462274560, "latency_s": 2.1678038337267935, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72056832, "prompt_kind": "long", "t_relative_s": 4968.415730375331}, {"decode_latency_s": 0.06047050128108822, "kv_live_bytes": 462274560, "latency_s": 1.9358538752421737, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72007680, "prompt_kind": "short", "t_relative_s": 4973.425819000229}, {"decode_latency_s": 0.06789363543794025, "kv_live_bytes": 462274560, "latency_s": 2.174544000066817, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71909376, "prompt_kind": "long", "t_relative_s": 4978.43544233311}, {"decode_latency_s": 0.06162783724721521, "kv_live_bytes": 462274560, "latency_s": 1.9731065407395363, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68485120, "prompt_kind": "short", "t_relative_s": 4983.445497833192}, {"decode_latency_s": 0.06768397397536319, "kv_live_bytes": 462274560, "latency_s": 2.1679603750817478, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70582272, "prompt_kind": "long", "t_relative_s": 4988.4555446673185}, {"decode_latency_s": 0.06048193227616139, "kv_live_bytes": 462274560, "latency_s": 1.9361697086133063, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70729728, "prompt_kind": "short", "t_relative_s": 4993.4656403330155}, {"decode_latency_s": 0.06792391797353048, "kv_live_bytes": 462274560, "latency_s": 2.1753797912970185, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70074368, "prompt_kind": "long", "t_relative_s": 4998.4756803330965}, {"decode_latency_s": 0.0603189140529139, "kv_live_bytes": 462274560, "latency_s": 1.9311778331175447, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70385664, "prompt_kind": "short", "t_relative_s": 5003.485750208143}, {"decode_latency_s": 0.06819344009272754, "kv_live_bytes": 462274560, "latency_s": 2.1844648751430213, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71057408, "prompt_kind": "long", "t_relative_s": 5008.4958337503485}, {"decode_latency_s": 0.06647875520866364, "kv_live_bytes": 462274560, "latency_s": 2.129160459153354, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71630848, "prompt_kind": "short", "t_relative_s": 5013.505903083365}, {"decode_latency_s": 0.0680990026157815, "kv_live_bytes": 462274560, "latency_s": 2.1816036659292877, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71729152, "prompt_kind": "long", "t_relative_s": 5018.5159669169225}, {"decode_latency_s": 0.0633603671885794, "kv_live_bytes": 462274560, "latency_s": 2.0284154578112066, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69910528, "prompt_kind": "short", "t_relative_s": 5023.526051583234}, {"decode_latency_s": 0.06824391537520569, "kv_live_bytes": 462274560, "latency_s": 2.186329582706094, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70811648, "prompt_kind": "long", "t_relative_s": 5028.536135625094}, {"decode_latency_s": 0.060702839837176725, "kv_live_bytes": 462274560, "latency_s": 1.9432618748396635, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71352320, "prompt_kind": "short", "t_relative_s": 5033.546220000368}, {"decode_latency_s": 0.06064721875009127, "kv_live_bytes": 462274560, "latency_s": 1.941572249867022, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71401472, "prompt_kind": "long", "t_relative_s": 5038.55625441717}, {"decode_latency_s": 0.06045028124935925, "kv_live_bytes": 462274560, "latency_s": 1.9351682080887258, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71892992, "prompt_kind": "short", "t_relative_s": 5043.566330542322}, {"decode_latency_s": 0.068170985658071, "kv_live_bytes": 462274560, "latency_s": 2.183476416859776, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71942144, "prompt_kind": "long", "t_relative_s": 5048.576410542242}, {"decode_latency_s": 0.06040767187369056, "kv_live_bytes": 462274560, "latency_s": 1.9339162497781217, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72073216, "prompt_kind": "short", "t_relative_s": 5053.586466582958}, {"decode_latency_s": 0.06796346225019079, "kv_live_bytes": 462274560, "latency_s": 2.176752499770373, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72171520, "prompt_kind": "long", "t_relative_s": 5058.596545167267}, {"decode_latency_s": 0.06057231512386352, "kv_live_bytes": 462274560, "latency_s": 1.9391308748163283, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72269824, "prompt_kind": "short", "t_relative_s": 5063.606648458168}, {"decode_latency_s": 0.06789435936661903, "kv_live_bytes": 462274560, "latency_s": 2.175045750103891, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71729152, "prompt_kind": "long", "t_relative_s": 5068.616705375258}, {"decode_latency_s": 0.06037813150032889, "kv_live_bytes": 462274560, "latency_s": 1.9330028328113258, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72269824, "prompt_kind": "short", "t_relative_s": 5073.626786167268}, {"decode_latency_s": 0.06833160415408202, "kv_live_bytes": 462274560, "latency_s": 2.1881961659528315, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72368128, "prompt_kind": "long", "t_relative_s": 5078.6369097922}, {"decode_latency_s": 0.060807358066085726, "kv_live_bytes": 462274560, "latency_s": 1.946648959070444, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71745536, "prompt_kind": "short", "t_relative_s": 5083.646952250041}, {"decode_latency_s": 0.06064116668130737, "kv_live_bytes": 462274560, "latency_s": 1.942085084039718, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71925760, "prompt_kind": "long", "t_relative_s": 5088.657005625311}, {"decode_latency_s": 0.06038334374898113, "kv_live_bytes": 462274560, "latency_s": 1.9331630831584334, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72302592, "prompt_kind": "short", "t_relative_s": 5093.667072083335}, {"decode_latency_s": 0.0678113307512831, "kv_live_bytes": 462274560, "latency_s": 2.1716542500071228, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72351744, "prompt_kind": "long", "t_relative_s": 5098.677161999978}, {"decode_latency_s": 0.06059409637236968, "kv_live_bytes": 462274560, "latency_s": 1.9399039172567427, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72892416, "prompt_kind": "short", "t_relative_s": 5103.6872396250255}, {"decode_latency_s": 0.0675557044451125, "kv_live_bytes": 462274560, "latency_s": 2.1642337907105684, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72925184, "prompt_kind": "long", "t_relative_s": 5108.697334625293}, {"decode_latency_s": 0.06055227605975233, "kv_live_bytes": 462274560, "latency_s": 1.9384324587881565, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73007104, "prompt_kind": "short", "t_relative_s": 5113.707436542027}, {"decode_latency_s": 0.06811691797338426, "kv_live_bytes": 462274560, "latency_s": 2.181955875363201, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73007104, "prompt_kind": "long", "t_relative_s": 5118.717488416936}, {"decode_latency_s": 0.06038824869028758, "kv_live_bytes": 462274560, "latency_s": 1.9333183751441538, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72351744, "prompt_kind": "short", "t_relative_s": 5123.727576750331}, {"decode_latency_s": 0.06036850131931715, "kv_live_bytes": 462274560, "latency_s": 1.9327156245708466, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72351744, "prompt_kind": "long", "t_relative_s": 5128.737657958176}, {"decode_latency_s": 0.06082091406278778, "kv_live_bytes": 462274560, "latency_s": 1.9471612498164177, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72892416, "prompt_kind": "short", "t_relative_s": 5133.747712083161}, {"decode_latency_s": 0.06758523438475095, "kv_live_bytes": 462274560, "latency_s": 2.1648479159921408, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72564736, "prompt_kind": "long", "t_relative_s": 5138.757809875067}, {"decode_latency_s": 0.061882764304755256, "kv_live_bytes": 462274560, "latency_s": 1.9814810841344297, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71090176, "prompt_kind": "short", "t_relative_s": 5143.767896042205}, {"decode_latency_s": 0.0686498124996433, "kv_live_bytes": 462274560, "latency_s": 2.1987967076711357, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72138752, "prompt_kind": "long", "t_relative_s": 5148.7779948753305}, {"decode_latency_s": 0.0607852877874393, "kv_live_bytes": 462274560, "latency_s": 1.945929542183876, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72630272, "prompt_kind": "short", "t_relative_s": 5153.788041042164}, {"decode_latency_s": 0.06776922264543828, "kv_live_bytes": 462274560, "latency_s": 2.1710374588146806, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72728576, "prompt_kind": "long", "t_relative_s": 5158.7981252921745}, {"decode_latency_s": 0.06084985155030154, "kv_live_bytes": 462274560, "latency_s": 1.9480158751830459, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73236480, "prompt_kind": "short", "t_relative_s": 5163.808200458065}, {"decode_latency_s": 0.06736497003294062, "kv_live_bytes": 462274560, "latency_s": 2.1576606249436736, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72761344, "prompt_kind": "long", "t_relative_s": 5168.8182581672445}, {"decode_latency_s": 0.06049643881851807, "kv_live_bytes": 462274560, "latency_s": 1.9368297909386456, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72024064, "prompt_kind": "short", "t_relative_s": 5173.82831449993}, {"decode_latency_s": 0.06792703906830866, "kv_live_bytes": 462274560, "latency_s": 2.1753812078386545, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "long", "t_relative_s": 5178.838405082934}, {"decode_latency_s": 0.06047511064389255, "kv_live_bytes": 462274560, "latency_s": 1.9360533328726888, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72810496, "prompt_kind": "short", "t_relative_s": 5183.848455999978}, {"decode_latency_s": 0.06779176952841226, "kv_live_bytes": 462274560, "latency_s": 2.1716402913443744, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72859648, "prompt_kind": "long", "t_relative_s": 5188.858518667053}, {"decode_latency_s": 0.060833471376099624, "kv_live_bytes": 462274560, "latency_s": 1.9474725839681923, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73498624, "prompt_kind": "short", "t_relative_s": 5193.86858775001}, {"decode_latency_s": 0.06817613540624734, "kv_live_bytes": 462274560, "latency_s": 2.184313707984984, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73596928, "prompt_kind": "long", "t_relative_s": 5198.878648458049}, {"decode_latency_s": 0.06037428906711284, "kv_live_bytes": 462274560, "latency_s": 1.9329004581086338, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72318976, "prompt_kind": "short", "t_relative_s": 5203.888722958043}, {"decode_latency_s": 0.06806583462457638, "kv_live_bytes": 462274560, "latency_s": 2.180670500267297, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72974336, "prompt_kind": "long", "t_relative_s": 5208.898758500349}, {"decode_latency_s": 0.060314485686831176, "kv_live_bytes": 462274560, "latency_s": 1.93087258329615, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73023488, "prompt_kind": "short", "t_relative_s": 5213.908831333276}, {"decode_latency_s": 0.06805656899814494, "kv_live_bytes": 462274560, "latency_s": 2.179716874845326, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72810496, "prompt_kind": "long", "t_relative_s": 5218.918903542217}, {"decode_latency_s": 0.06131038410239853, "kv_live_bytes": 462274560, "latency_s": 1.9627695833332837, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70025216, "prompt_kind": "short", "t_relative_s": 5223.928959167097}, {"decode_latency_s": 0.06689815752906725, "kv_live_bytes": 462274560, "latency_s": 2.143056540749967, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72876032, "prompt_kind": "long", "t_relative_s": 5228.939011916984}, {"decode_latency_s": 0.0602659687428968, "kv_live_bytes": 462274560, "latency_s": 1.929359540808946, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72761344, "prompt_kind": "short", "t_relative_s": 5233.949092749972}, {"decode_latency_s": 0.06789299218507949, "kv_live_bytes": 462274560, "latency_s": 2.17489912500605, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72450048, "prompt_kind": "long", "t_relative_s": 5238.959147625137}, {"decode_latency_s": 0.06037170052877627, "kv_live_bytes": 462274560, "latency_s": 1.9326664158143103, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73056256, "prompt_kind": "short", "t_relative_s": 5243.969221916981}, {"decode_latency_s": 0.06763759764726274, "kv_live_bytes": 462274560, "latency_s": 2.166039042174816, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73089024, "prompt_kind": "long", "t_relative_s": 5248.979266083334}, {"decode_latency_s": 0.06054453906835988, "kv_live_bytes": 462274560, "latency_s": 1.9382123341783881, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73154560, "prompt_kind": "short", "t_relative_s": 5253.989337208215}, {"decode_latency_s": 0.06804172655392904, "kv_live_bytes": 462274560, "latency_s": 2.1794967497698963, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73744384, "prompt_kind": "long", "t_relative_s": 5258.9993938333355}, {"decode_latency_s": 0.06086014321772382, "kv_live_bytes": 462274560, "latency_s": 1.948417709209025, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73809920, "prompt_kind": "short", "t_relative_s": 5264.00741745811}, {"decode_latency_s": 0.067548365885159, "kv_live_bytes": 462274560, "latency_s": 2.163176625035703, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73334784, "prompt_kind": "long", "t_relative_s": 5269.017503583338}, {"decode_latency_s": 0.060765792964957654, "kv_live_bytes": 462274560, "latency_s": 1.9453703751787543, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72957952, "prompt_kind": "short", "t_relative_s": 5274.0275592501275}, {"decode_latency_s": 0.06794223828183021, "kv_live_bytes": 462274560, "latency_s": 2.1759425830096006, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73580544, "prompt_kind": "long", "t_relative_s": 5279.037636208348}, {"decode_latency_s": 0.06081358203664422, "kv_live_bytes": 462274560, "latency_s": 1.9468350000679493, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74006528, "prompt_kind": "short", "t_relative_s": 5284.047742542345}, {"decode_latency_s": 0.06745507684536278, "kv_live_bytes": 462274560, "latency_s": 2.1607816251926124, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74186752, "prompt_kind": "long", "t_relative_s": 5289.057797833346}, {"decode_latency_s": 0.060605492180911824, "kv_live_bytes": 462274560, "latency_s": 1.9401220004074275, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73678848, "prompt_kind": "short", "t_relative_s": 5294.067877416965}, {"decode_latency_s": 0.06522247528482694, "kv_live_bytes": 462274560, "latency_s": 2.088503667153418, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69894144, "prompt_kind": "long", "t_relative_s": 5299.205730125308}, {"decode_latency_s": 0.06037222524173558, "kv_live_bytes": 462274560, "latency_s": 1.9328091670759022, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70025216, "prompt_kind": "short", "t_relative_s": 5305.65885675}, {"decode_latency_s": 0.06806552734633442, "kv_live_bytes": 462274560, "latency_s": 2.179945208132267, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70139904, "prompt_kind": "long", "t_relative_s": 5310.668921750039}, {"decode_latency_s": 0.06083634766400792, "kv_live_bytes": 462274560, "latency_s": 1.9476372092030942, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70516736, "prompt_kind": "short", "t_relative_s": 5315.678985208273}, {"decode_latency_s": 0.06797776820894796, "kv_live_bytes": 462274560, "latency_s": 2.177840624935925, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70615040, "prompt_kind": "long", "t_relative_s": 5320.689063082915}, {"decode_latency_s": 0.06076434765418526, "kv_live_bytes": 462274560, "latency_s": 1.9452499998733401, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70680576, "prompt_kind": "short", "t_relative_s": 5325.699155000038}, {"decode_latency_s": 0.06748307812085841, "kv_live_bytes": 462274560, "latency_s": 2.1613934580236673, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70172672, "prompt_kind": "long", "t_relative_s": 5330.709202458151}, {"decode_latency_s": 0.06048290233593434, "kv_live_bytes": 462274560, "latency_s": 1.9363846252672374, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69828608, "prompt_kind": "short", "t_relative_s": 5335.719256542157}, {"decode_latency_s": 0.06748781508940738, "kv_live_bytes": 462274560, "latency_s": 2.162140333093703, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70418432, "prompt_kind": "long", "t_relative_s": 5340.729300958104}, {"decode_latency_s": 0.06038838150561787, "kv_live_bytes": 462274560, "latency_s": 1.9332730420865119, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70959104, "prompt_kind": "short", "t_relative_s": 5345.73938725004}, {"decode_latency_s": 0.06798706122208387, "kv_live_bytes": 462274560, "latency_s": 2.177361374720931, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71008256, "prompt_kind": "long", "t_relative_s": 5350.749483500142}, {"decode_latency_s": 0.060692601575283334, "kv_live_bytes": 462274560, "latency_s": 1.9431758751161397, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70647808, "prompt_kind": "short", "t_relative_s": 5355.759537125006}, {"decode_latency_s": 0.06760408593981992, "kv_live_bytes": 462274560, "latency_s": 2.1654435000382364, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70746112, "prompt_kind": "long", "t_relative_s": 5360.769598292187}, {"decode_latency_s": 0.060482018219772726, "kv_live_bytes": 462274560, "latency_s": 1.9361868752166629, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71467008, "prompt_kind": "short", "t_relative_s": 5365.77967308322}, {"decode_latency_s": 0.0680831002391642, "kv_live_bytes": 462274560, "latency_s": 2.1812083330005407, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71434240, "prompt_kind": "long", "t_relative_s": 5370.789729792159}, {"decode_latency_s": 0.06050017577945255, "kv_live_bytes": 462274560, "latency_s": 1.9369919998571277, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71499776, "prompt_kind": "short", "t_relative_s": 5375.799800542183}, {"decode_latency_s": 0.06810191666590981, "kv_live_bytes": 462274560, "latency_s": 2.1809036671184003, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70303744, "prompt_kind": "long", "t_relative_s": 5380.8098741672}, {"decode_latency_s": 0.06343649477639701, "kv_live_bytes": 462274560, "latency_s": 2.030799083877355, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66125824, "prompt_kind": "short", "t_relative_s": 5385.8123172502965}, {"decode_latency_s": 0.0681252916547237, "kv_live_bytes": 462274560, "latency_s": 2.1820747079327703, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70795264, "prompt_kind": "long", "t_relative_s": 5390.8224103753455}, {"decode_latency_s": 0.06048014711996075, "kv_live_bytes": 462274560, "latency_s": 1.9361817087046802, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71417856, "prompt_kind": "short", "t_relative_s": 5395.832482667174}, {"decode_latency_s": 0.06782659243617672, "kv_live_bytes": 462274560, "latency_s": 2.172808540984988, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71467008, "prompt_kind": "long", "t_relative_s": 5400.842534500174}, {"decode_latency_s": 0.06082322915608529, "kv_live_bytes": 462274560, "latency_s": 1.9471597908996046, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71876608, "prompt_kind": "short", "t_relative_s": 5405.852609250229}, {"decode_latency_s": 0.06795397003588732, "kv_live_bytes": 462274560, "latency_s": 2.1762945423834026, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71401472, "prompt_kind": "long", "t_relative_s": 5410.862658792175}, {"decode_latency_s": 0.06097272525948938, "kv_live_bytes": 462274560, "latency_s": 1.952005541883409, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70746112, "prompt_kind": "short", "t_relative_s": 5415.872744458262}, {"decode_latency_s": 0.06782374999602325, "kv_live_bytes": 462274560, "latency_s": 2.1724480828270316, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71368704, "prompt_kind": "long", "t_relative_s": 5420.882795417216}, {"decode_latency_s": 0.06046809506369755, "kv_live_bytes": 462274560, "latency_s": 1.9358747922815382, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71401472, "prompt_kind": "short", "t_relative_s": 5425.892891542055}, {"decode_latency_s": 0.06887476301926654, "kv_live_bytes": 462274560, "latency_s": 2.2057210421189666, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69484544, "prompt_kind": "long", "t_relative_s": 5430.902987250127}, {"decode_latency_s": 0.0605760963371722, "kv_live_bytes": 462274560, "latency_s": 1.9393745423294604, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71073792, "prompt_kind": "short", "t_relative_s": 5435.913033707999}, {"decode_latency_s": 0.06869764193834271, "kv_live_bytes": 462274560, "latency_s": 2.2005172092467546, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71811072, "prompt_kind": "long", "t_relative_s": 5440.92310133297}, {"decode_latency_s": 0.06068565105670132, "kv_live_bytes": 462274560, "latency_s": 1.9427439579740167, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71892992, "prompt_kind": "short", "t_relative_s": 5445.933157166932}, {"decode_latency_s": 0.0626436510210624, "kv_live_bytes": 462274560, "latency_s": 2.006334625184536, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71385088, "prompt_kind": "long", "t_relative_s": 5450.943206042051}, {"decode_latency_s": 0.06504875000973698, "kv_live_bytes": 462274560, "latency_s": 2.083002416882664, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69337088, "prompt_kind": "short", "t_relative_s": 5455.95155287534}, {"decode_latency_s": 0.06754390624701045, "kv_live_bytes": 462274560, "latency_s": 2.163765999954194, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70320128, "prompt_kind": "long", "t_relative_s": 5460.961672374979}, {"decode_latency_s": 0.06778468750417233, "kv_live_bytes": 462274560, "latency_s": 2.170501542277634, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69386240, "prompt_kind": "short", "t_relative_s": 5465.971719833091}, {"decode_latency_s": 0.06809895571495872, "kv_live_bytes": 462274560, "latency_s": 2.1811808752827346, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70746112, "prompt_kind": "long", "t_relative_s": 5470.981811167207}, {"decode_latency_s": 0.06687271874397993, "kv_live_bytes": 462274560, "latency_s": 2.1416138340719044, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70926336, "prompt_kind": "short", "t_relative_s": 5475.991866874974}, {"decode_latency_s": 0.0678113789181225, "kv_live_bytes": 462274560, "latency_s": 2.1723600002005696, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70991872, "prompt_kind": "long", "t_relative_s": 5481.001979667228}, {"decode_latency_s": 0.060588087246287614, "kv_live_bytes": 462274560, "latency_s": 1.9395846673287451, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71630848, "prompt_kind": "short", "t_relative_s": 5486.012055667117}, {"decode_latency_s": 0.06814298697281629, "kv_live_bytes": 462274560, "latency_s": 2.1829429999925196, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71712768, "prompt_kind": "long", "t_relative_s": 5491.022108542267}, {"decode_latency_s": 0.06058556250354741, "kv_live_bytes": 462274560, "latency_s": 1.939576624892652, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72187904, "prompt_kind": "short", "t_relative_s": 5496.0321736251935}, {"decode_latency_s": 0.06857630077865906, "kv_live_bytes": 462274560, "latency_s": 2.19611837528646, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71221248, "prompt_kind": "long", "t_relative_s": 5501.042254874948}, {"decode_latency_s": 0.060632440116023645, "kv_live_bytes": 462274560, "latency_s": 1.9411457078531384, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71155712, "prompt_kind": "short", "t_relative_s": 5506.052305832971}, {"decode_latency_s": 0.06824071222217754, "kv_live_bytes": 462274560, "latency_s": 2.185823292005807, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71843840, "prompt_kind": "long", "t_relative_s": 5511.062390374951}, {"decode_latency_s": 0.06069001040305011, "kv_live_bytes": 462274560, "latency_s": 1.9428435829468071, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72384512, "prompt_kind": "short", "t_relative_s": 5516.072472792119}, {"decode_latency_s": 0.06525982031598687, "kv_live_bytes": 462274560, "latency_s": 2.090739833191037, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72302592, "prompt_kind": "long", "t_relative_s": 5521.082525667269}, {"decode_latency_s": 0.06117649868247099, "kv_live_bytes": 462274560, "latency_s": 1.958419790957123, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72318976, "prompt_kind": "short", "t_relative_s": 5526.0926021672785}, {"decode_latency_s": 0.06794289973913692, "kv_live_bytes": 462274560, "latency_s": 2.1759312078356743, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71696384, "prompt_kind": "long", "t_relative_s": 5531.099274000153}, {"decode_latency_s": 0.06047992059029639, "kv_live_bytes": 462274560, "latency_s": 1.936479666735977, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70254592, "prompt_kind": "short", "t_relative_s": 5536.109302250203}, {"decode_latency_s": 0.06812442706723232, "kv_live_bytes": 462274560, "latency_s": 2.1821375000290573, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72155136, "prompt_kind": "long", "t_relative_s": 5541.119377458002}, {"decode_latency_s": 0.06661527471442241, "kv_live_bytes": 462274560, "latency_s": 2.133393708150834, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72892416, "prompt_kind": "short", "t_relative_s": 5546.129453833215}, {"decode_latency_s": 0.06039210027665831, "kv_live_bytes": 462274560, "latency_s": 1.9335072920657694, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72908800, "prompt_kind": "long", "t_relative_s": 5551.139526083134}, {"decode_latency_s": 0.06081705207179766, "kv_live_bytes": 462274560, "latency_s": 1.9469068329781294, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73531392, "prompt_kind": "short", "t_relative_s": 5556.149596458301}, {"decode_latency_s": 0.0681757994752843, "kv_live_bytes": 462274560, "latency_s": 2.1836214996874332, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73351168, "prompt_kind": "long", "t_relative_s": 5561.159647458233}, {"decode_latency_s": 0.06053571743541397, "kv_live_bytes": 462274560, "latency_s": 1.9379719588905573, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73023488, "prompt_kind": "short", "t_relative_s": 5566.169730375055}, {"decode_latency_s": 0.06773890493786894, "kv_live_bytes": 462274560, "latency_s": 2.1701154173351824, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72810496, "prompt_kind": "long", "t_relative_s": 5571.179782667197}, {"decode_latency_s": 0.060279744779109024, "kv_live_bytes": 462274560, "latency_s": 1.9299211660400033, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72859648, "prompt_kind": "short", "t_relative_s": 5576.1898743333295}, {"decode_latency_s": 0.06758727997657843, "kv_live_bytes": 462274560, "latency_s": 2.164323542267084, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72843264, "prompt_kind": "long", "t_relative_s": 5581.1999528333545}, {"decode_latency_s": 0.06697456250549294, "kv_live_bytes": 462274560, "latency_s": 2.145015375223011, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73400320, "prompt_kind": "short", "t_relative_s": 5586.210003792308}, {"decode_latency_s": 0.06792639844934456, "kv_live_bytes": 462274560, "latency_s": 2.1761746662668884, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73498624, "prompt_kind": "long", "t_relative_s": 5591.220110583119}, {"decode_latency_s": 0.060787549504311755, "kv_live_bytes": 462274560, "latency_s": 1.9460160001181066, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72957952, "prompt_kind": "short", "t_relative_s": 5596.230195542332}, {"decode_latency_s": 0.06791008592699654, "kv_live_bytes": 462274560, "latency_s": 2.1754917092621326, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72974336, "prompt_kind": "long", "t_relative_s": 5601.240249708295}, {"decode_latency_s": 0.06040788804239128, "kv_live_bytes": 462274560, "latency_s": 1.9339602920226753, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "short", "t_relative_s": 5606.2423578752205}, {"decode_latency_s": 0.06844254296447616, "kv_live_bytes": 462274560, "latency_s": 2.1917563751339912, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73465856, "prompt_kind": "long", "t_relative_s": 5611.252437457908}, {"decode_latency_s": 0.06698280728596728, "kv_live_bytes": 462274560, "latency_s": 2.145127291791141, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74137600, "prompt_kind": "short", "t_relative_s": 5616.262495333329}, {"decode_latency_s": 0.06802464193606284, "kv_live_bytes": 462274560, "latency_s": 2.1792359170503914, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74088448, "prompt_kind": "long", "t_relative_s": 5621.272609750275}, {"decode_latency_s": 0.0606346562562976, "kv_live_bytes": 462274560, "latency_s": 1.9410959999077022, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74039296, "prompt_kind": "short", "t_relative_s": 5626.2826888333075}, {"decode_latency_s": 0.06792924087494612, "kv_live_bytes": 462274560, "latency_s": 2.175788374617696, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73973760, "prompt_kind": "long", "t_relative_s": 5631.292742125224}, {"decode_latency_s": 0.06025821615185123, "kv_live_bytes": 462274560, "latency_s": 1.9291592920199037, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72744960, "prompt_kind": "short", "t_relative_s": 5636.302814458031}, {"decode_latency_s": 0.06806530077301431, "kv_live_bytes": 462274560, "latency_s": 2.1798598342575133, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73433088, "prompt_kind": "long", "t_relative_s": 5641.312896457966}, {"decode_latency_s": 0.06076919271436054, "kv_live_bytes": 462274560, "latency_s": 1.9454945409670472, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73957376, "prompt_kind": "short", "t_relative_s": 5646.313850583043}, {"decode_latency_s": 0.06799564715765882, "kv_live_bytes": 462274560, "latency_s": 2.178237958345562, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74104832, "prompt_kind": "long", "t_relative_s": 5651.323931792285}, {"decode_latency_s": 0.2484007604070939, "kv_live_bytes": 462274560, "latency_s": 7.949615667108446, "n_emitted": 32, "ok": true, "process_footprint_bytes": 197476352, "prompt_kind": "short", "t_relative_s": 5656.3340245829895}, {"decode_latency_s": 0.06817980730556883, "kv_live_bytes": 462274560, "latency_s": 2.184153167065233, "n_emitted": 32, "ok": true, "process_footprint_bytes": 198148096, "prompt_kind": "long", "t_relative_s": 5665.166657625232}, {"decode_latency_s": 0.06037077865039464, "kv_live_bytes": 462274560, "latency_s": 1.9328107498586178, "n_emitted": 32, "ok": true, "process_footprint_bytes": 197689344, "prompt_kind": "short", "t_relative_s": 5670.176732750144}, {"decode_latency_s": 0.06067856641311664, "kv_live_bytes": 462274560, "latency_s": 1.9425846668891609, "n_emitted": 32, "ok": true, "process_footprint_bytes": 197820416, "prompt_kind": "long", "t_relative_s": 5675.186814625282}, {"decode_latency_s": 0.06068708070961293, "kv_live_bytes": 462274560, "latency_s": 1.9427944589406252, "n_emitted": 32, "ok": true, "process_footprint_bytes": 197754880, "prompt_kind": "short", "t_relative_s": 5680.196896083187}, {"decode_latency_s": 0.0688575182139175, "kv_live_bytes": 462274560, "latency_s": 2.206577999982983, "n_emitted": 32, "ok": true, "process_footprint_bytes": 103628800, "prompt_kind": "long", "t_relative_s": 5685.206950458232}, {"decode_latency_s": 0.06079288934415672, "kv_live_bytes": 462274560, "latency_s": 1.9462702088057995, "n_emitted": 32, "ok": true, "process_footprint_bytes": 97419264, "prompt_kind": "short", "t_relative_s": 5690.217053417116}, {"decode_latency_s": 0.06762899218301754, "kv_live_bytes": 462274560, "latency_s": 2.166463583242148, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71614464, "prompt_kind": "long", "t_relative_s": 5695.227102207951}, {"decode_latency_s": 0.06065555990790017, "kv_live_bytes": 462274560, "latency_s": 1.941780417226255, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72335360, "prompt_kind": "short", "t_relative_s": 5700.237167542335}, {"decode_latency_s": 0.061032920581055805, "kv_live_bytes": 462274560, "latency_s": 1.954207916278392, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72466432, "prompt_kind": "long", "t_relative_s": 5705.24720324995}, {"decode_latency_s": 0.06044326821574941, "kv_live_bytes": 462274560, "latency_s": 1.9351019589230418, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73089024, "prompt_kind": "short", "t_relative_s": 5710.251912042033}, {"decode_latency_s": 0.06809791925479658, "kv_live_bytes": 462274560, "latency_s": 2.1809624172747135, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73170944, "prompt_kind": "long", "t_relative_s": 5715.261976667214}, {"decode_latency_s": 0.06050046875316184, "kv_live_bytes": 462274560, "latency_s": 1.9368598330765963, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73416704, "prompt_kind": "short", "t_relative_s": 5720.272041375283}, {"decode_latency_s": 0.06851379034924321, "kv_live_bytes": 462274560, "latency_s": 2.19506649998948, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72777728, "prompt_kind": "long", "t_relative_s": 5725.282108208165}, {"decode_latency_s": 0.06628958853252698, "kv_live_bytes": 462274560, "latency_s": 2.1234282911755145, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "short", "t_relative_s": 5730.292178750038}, {"decode_latency_s": 0.0684919101477135, "kv_live_bytes": 462274560, "latency_s": 2.1941080410033464, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73498624, "prompt_kind": "long", "t_relative_s": 5735.302236041985}, {"decode_latency_s": 0.060270656264037825, "kv_live_bytes": 462274560, "latency_s": 1.9295859588310122, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74055680, "prompt_kind": "short", "t_relative_s": 5740.312309875153}, {"decode_latency_s": 0.06804388150339946, "kv_live_bytes": 462274560, "latency_s": 2.1795366248115897, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73269248, "prompt_kind": "long", "t_relative_s": 5745.322390458081}, {"decode_latency_s": 0.06059109764464665, "kv_live_bytes": 462274560, "latency_s": 1.9397494997829199, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73515008, "prompt_kind": "short", "t_relative_s": 5750.332448666915}, {"decode_latency_s": 0.06674873827432748, "kv_live_bytes": 462274560, "latency_s": 2.1379295829683542, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73678848, "prompt_kind": "long", "t_relative_s": 5755.342515999917}, {"decode_latency_s": 0.06601769920962397, "kv_live_bytes": 462274560, "latency_s": 2.114217000082135, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "short", "t_relative_s": 5760.347380542196}, {"decode_latency_s": 0.06801001430721954, "kv_live_bytes": 462274560, "latency_s": 2.1791731668636203, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72515584, "prompt_kind": "long", "t_relative_s": 5765.357428708114}, {"decode_latency_s": 0.060416535154217854, "kv_live_bytes": 462274560, "latency_s": 1.934269833844155, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73072640, "prompt_kind": "short", "t_relative_s": 5770.367504124995}, {"decode_latency_s": 0.06762728774629068, "kv_live_bytes": 462274560, "latency_s": 2.1657128748483956, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71663616, "prompt_kind": "long", "t_relative_s": 5775.377575207967}, {"decode_latency_s": 0.060371364568709396, "kv_live_bytes": 462274560, "latency_s": 1.9328658329322934, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71827456, "prompt_kind": "short", "t_relative_s": 5780.387626416981}, {"decode_latency_s": 0.06809502863325179, "kv_live_bytes": 462274560, "latency_s": 2.1810887921601534, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "long", "t_relative_s": 5785.3977007921785}, {"decode_latency_s": 0.0672395481960848, "kv_live_bytes": 462274560, "latency_s": 2.1535207917913795, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73203712, "prompt_kind": "short", "t_relative_s": 5790.407749083359}, {"decode_latency_s": 0.06824851952842437, "kv_live_bytes": 462274560, "latency_s": 2.186586208175868, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73236480, "prompt_kind": "long", "t_relative_s": 5795.417805583216}, {"decode_latency_s": 0.06034682680910919, "kv_live_bytes": 462274560, "latency_s": 1.9318410828709602, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73891840, "prompt_kind": "short", "t_relative_s": 5800.427868375089}, {"decode_latency_s": 0.0682612330710981, "kv_live_bytes": 462274560, "latency_s": 2.186104209162295, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73728000, "prompt_kind": "long", "t_relative_s": 5805.437949708197}, {"decode_latency_s": 0.060392700528609566, "kv_live_bytes": 462274560, "latency_s": 1.9335027909837663, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73236480, "prompt_kind": "short", "t_relative_s": 5810.447997292038}, {"decode_latency_s": 0.06798155469004996, "kv_live_bytes": 462274560, "latency_s": 2.177728542126715, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73777152, "prompt_kind": "long", "t_relative_s": 5815.458098417148}, {"decode_latency_s": 0.06682088281377219, "kv_live_bytes": 462274560, "latency_s": 2.139820707961917, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73891840, "prompt_kind": "short", "t_relative_s": 5820.468149292283}, {"decode_latency_s": 0.06831002475519199, "kv_live_bytes": 462274560, "latency_s": 2.188266792334616, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73383936, "prompt_kind": "long", "t_relative_s": 5825.4781993329525}, {"decode_latency_s": 0.060364595061400905, "kv_live_bytes": 462274560, "latency_s": 1.9325463329441845, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73940992, "prompt_kind": "short", "t_relative_s": 5830.4882920002565}, {"decode_latency_s": 0.06796566405682825, "kv_live_bytes": 462274560, "latency_s": 2.176494542043656, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72712192, "prompt_kind": "long", "t_relative_s": 5835.498366250191}, {"decode_latency_s": 0.060275053401710466, "kv_live_bytes": 462274560, "latency_s": 1.9297597911208868, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72957952, "prompt_kind": "short", "t_relative_s": 5840.508407500107}, {"decode_latency_s": 0.06775046484835912, "kv_live_bytes": 462274560, "latency_s": 2.1702853329479694, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73531392, "prompt_kind": "long", "t_relative_s": 5845.518484292086}, {"decode_latency_s": 0.06059787760023028, "kv_live_bytes": 462274560, "latency_s": 1.9399456670507789, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74072064, "prompt_kind": "short", "t_relative_s": 5850.528582167346}, {"decode_latency_s": 0.060538467470905744, "kv_live_bytes": 462274560, "latency_s": 1.938066417351365, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74104832, "prompt_kind": "long", "t_relative_s": 5855.538634375203}, {"decode_latency_s": 0.060385523436707444, "kv_live_bytes": 462274560, "latency_s": 1.9331875410862267, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74727424, "prompt_kind": "short", "t_relative_s": 5860.548719750252}, {"decode_latency_s": 0.06076521356590092, "kv_live_bytes": 462274560, "latency_s": 1.9455079999752343, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70320128, "prompt_kind": "long", "t_relative_s": 5865.558846917003}, {"decode_latency_s": 0.060465204442152753, "kv_live_bytes": 462274560, "latency_s": 1.9358712090179324, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69746688, "prompt_kind": "short", "t_relative_s": 5870.568903333042}, {"decode_latency_s": 0.06787632293708157, "kv_live_bytes": 462274560, "latency_s": 2.1741129588335752, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73580544, "prompt_kind": "long", "t_relative_s": 5875.578954375349}, {"decode_latency_s": 0.060380695314961486, "kv_live_bytes": 462274560, "latency_s": 1.9331278330646455, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73957376, "prompt_kind": "short", "t_relative_s": 5880.589056707919}, {"decode_latency_s": 0.0679703646019334, "kv_live_bytes": 462274560, "latency_s": 2.1772683328017592, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73728000, "prompt_kind": "long", "t_relative_s": 5885.599141791929}, {"decode_latency_s": 0.060549934903974645, "kv_live_bytes": 462274560, "latency_s": 1.9385314169339836, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74350592, "prompt_kind": "short", "t_relative_s": 5890.60918550007}, {"decode_latency_s": 0.0688144817831926, "kv_live_bytes": 462274560, "latency_s": 2.20452345861122, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74383360, "prompt_kind": "long", "t_relative_s": 5895.619269375224}, {"decode_latency_s": 0.060603151068789884, "kv_live_bytes": 462274560, "latency_s": 1.9400901249609888, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74514432, "prompt_kind": "short", "t_relative_s": 5900.623227042146}, {"decode_latency_s": 0.06049291665840428, "kv_live_bytes": 462274560, "latency_s": 1.936766792088747, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74579968, "prompt_kind": "long", "t_relative_s": 5905.628472167067}, {"decode_latency_s": 0.06033000780735165, "kv_live_bytes": 462274560, "latency_s": 1.9314641249366105, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74612736, "prompt_kind": "short", "t_relative_s": 5910.638543667272}, {"decode_latency_s": 0.06827424609218724, "kv_live_bytes": 462274560, "latency_s": 2.186509625054896, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74661888, "prompt_kind": "long", "t_relative_s": 5915.64862750005}, {"decode_latency_s": 0.06074355596501846, "kv_live_bytes": 462274560, "latency_s": 1.9447232079692185, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75235328, "prompt_kind": "short", "t_relative_s": 5920.658676417079}, {"decode_latency_s": 0.06814580209902488, "kv_live_bytes": 462274560, "latency_s": 2.1831902088597417, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75104256, "prompt_kind": "long", "t_relative_s": 5925.668749208096}, {"decode_latency_s": 0.06069794272480067, "kv_live_bytes": 462274560, "latency_s": 1.9431522912345827, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75317248, "prompt_kind": "short", "t_relative_s": 5930.67884333292}, {"decode_latency_s": 0.06757323700003326, "kv_live_bytes": 462274560, "latency_s": 2.164455291815102, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74825728, "prompt_kind": "long", "t_relative_s": 5935.68890075013}, {"decode_latency_s": 0.06034611980430782, "kv_live_bytes": 462274560, "latency_s": 1.9319432084448636, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75497472, "prompt_kind": "short", "t_relative_s": 5940.698984167073}, {"decode_latency_s": 0.06804254556482192, "kv_live_bytes": 462274560, "latency_s": 2.1789476671256125, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75464704, "prompt_kind": "long", "t_relative_s": 5945.709054875188}, {"decode_latency_s": 0.06058230208873283, "kv_live_bytes": 462274560, "latency_s": 1.93951016664505, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76021760, "prompt_kind": "short", "t_relative_s": 5950.716877458151}, {"decode_latency_s": 0.06807538410066627, "kv_live_bytes": 462274560, "latency_s": 2.180792292114347, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75759616, "prompt_kind": "long", "t_relative_s": 5955.726962625049}, {"decode_latency_s": 0.060526542976731434, "kv_live_bytes": 462274560, "latency_s": 1.9376769163645804, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75841536, "prompt_kind": "short", "t_relative_s": 5960.737068958115}, {"decode_latency_s": 0.06760344789654482, "kv_live_bytes": 462274560, "latency_s": 2.165592874865979, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75890688, "prompt_kind": "long", "t_relative_s": 5965.739824749995}, {"decode_latency_s": 0.06034560938132927, "kv_live_bytes": 462274560, "latency_s": 1.931951165664941, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76496896, "prompt_kind": "short", "t_relative_s": 5970.749919292051}, {"decode_latency_s": 0.0683147942472715, "kv_live_bytes": 462274560, "latency_s": 2.188046208117157, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76447744, "prompt_kind": "long", "t_relative_s": 5975.759996917099}, {"decode_latency_s": 0.06341628514928743, "kv_live_bytes": 462274560, "latency_s": 2.0303834578953683, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67698688, "prompt_kind": "short", "t_relative_s": 5980.770063625183}, {"decode_latency_s": 0.08076485805213451, "kv_live_bytes": 462274560, "latency_s": 2.585367249790579, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66699264, "prompt_kind": "long", "t_relative_s": 5985.780115167145}, {"decode_latency_s": 0.06036216275242623, "kv_live_bytes": 462274560, "latency_s": 1.9324978329241276, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67485696, "prompt_kind": "short", "t_relative_s": 5992.892627499998}, {"decode_latency_s": 0.06811161850055214, "kv_live_bytes": 462274560, "latency_s": 2.1820076247677207, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69763072, "prompt_kind": "long", "t_relative_s": 5997.902697792277}, {"decode_latency_s": 0.06673798436531797, "kv_live_bytes": 462274560, "latency_s": 2.1374402083456516, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70402048, "prompt_kind": "short", "t_relative_s": 6002.912769292016}, {"decode_latency_s": 0.06792404559382703, "kv_live_bytes": 462274560, "latency_s": 2.1758832498453557, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70893568, "prompt_kind": "long", "t_relative_s": 6007.922806375194}, {"decode_latency_s": 0.060282118487521075, "kv_live_bytes": 462274560, "latency_s": 1.9299999168142676, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71450624, "prompt_kind": "short", "t_relative_s": 6012.932914667297}, {"decode_latency_s": 0.06838226821855642, "kv_live_bytes": 462274560, "latency_s": 2.190319959074259, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71335936, "prompt_kind": "long", "t_relative_s": 6017.943001749925}, {"decode_latency_s": 0.06106709374580532, "kv_live_bytes": 462274560, "latency_s": 1.9550232919864357, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70975488, "prompt_kind": "short", "t_relative_s": 6022.953069167212}, {"decode_latency_s": 0.06816423175041564, "kv_live_bytes": 462274560, "latency_s": 2.1832820419222116, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70959104, "prompt_kind": "long", "t_relative_s": 6027.9631459172815}, {"decode_latency_s": 0.06094574868620839, "kv_live_bytes": 462274560, "latency_s": 1.9510905831120908, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71565312, "prompt_kind": "short", "t_relative_s": 6032.973260750063}, {"decode_latency_s": 0.06035583330958616, "kv_live_bytes": 462274560, "latency_s": 1.9321801252663136, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71581696, "prompt_kind": "long", "t_relative_s": 6037.9833146249875}, {"decode_latency_s": 0.06076848309021443, "kv_live_bytes": 462274560, "latency_s": 1.9454345419071615, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72187904, "prompt_kind": "short", "t_relative_s": 6042.993411000352}, {"decode_latency_s": 0.06811595443286933, "kv_live_bytes": 462274560, "latency_s": 2.181398832704872, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72220672, "prompt_kind": "long", "t_relative_s": 6048.003496792167}, {"decode_latency_s": 0.060632532549789175, "kv_live_bytes": 462274560, "latency_s": 1.9415180408395827, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72335360, "prompt_kind": "short", "t_relative_s": 6053.0135516249575}, {"decode_latency_s": 0.06759824868640862, "kv_live_bytes": 462274560, "latency_s": 2.1656367918476462, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72368128, "prompt_kind": "long", "t_relative_s": 6058.0236294581555}, {"decode_latency_s": 0.060611509121372364, "kv_live_bytes": 462274560, "latency_s": 1.9403555416502059, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72908800, "prompt_kind": "short", "t_relative_s": 6063.033719625324}, {"decode_latency_s": 0.06792745574784931, "kv_live_bytes": 462274560, "latency_s": 2.1760287918150425, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72974336, "prompt_kind": "long", "t_relative_s": 6068.043777292129}, {"decode_latency_s": 0.06044243881478906, "kv_live_bytes": 462274560, "latency_s": 1.9350382080301642, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72925184, "prompt_kind": "short", "t_relative_s": 6073.053849332966}, {"decode_latency_s": 0.06831476301886141, "kv_live_bytes": 462274560, "latency_s": 2.1877139159478247, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72220672, "prompt_kind": "long", "t_relative_s": 6078.06391841732}, {"decode_latency_s": 0.06052056509361137, "kv_live_bytes": 462274560, "latency_s": 1.9374940837733448, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72335360, "prompt_kind": "short", "t_relative_s": 6083.073969041929}, {"decode_latency_s": 0.06790236981760245, "kv_live_bytes": 462274560, "latency_s": 2.1754929996095598, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72450048, "prompt_kind": "long", "t_relative_s": 6088.084046707954}, {"decode_latency_s": 0.06055896746693179, "kv_live_bytes": 462274560, "latency_s": 1.9387224996462464, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72990720, "prompt_kind": "short", "t_relative_s": 6093.0941585423425}, {"decode_latency_s": 0.0678413801942952, "kv_live_bytes": 462274560, "latency_s": 2.1732647912576795, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73072640, "prompt_kind": "long", "t_relative_s": 6098.104213042185}, {"decode_latency_s": 0.06066399090923369, "kv_live_bytes": 462274560, "latency_s": 1.9421747080050409, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73646080, "prompt_kind": "short", "t_relative_s": 6103.114293208346}, {"decode_latency_s": 0.06784712240914814, "kv_live_bytes": 462274560, "latency_s": 2.1726531670428813, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72728576, "prompt_kind": "long", "t_relative_s": 6108.124360083137}, {"decode_latency_s": 0.06692025260417722, "kv_live_bytes": 462274560, "latency_s": 2.1431248327717185, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72957952, "prompt_kind": "short", "t_relative_s": 6113.134405583143}, {"decode_latency_s": 0.06778684115852229, "kv_live_bytes": 462274560, "latency_s": 2.1720040407963097, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73089024, "prompt_kind": "long", "t_relative_s": 6118.14450333314}, {"decode_latency_s": 0.06048155990720261, "kv_live_bytes": 462274560, "latency_s": 1.936155208852142, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73646080, "prompt_kind": "short", "t_relative_s": 6123.15460345801}, {"decode_latency_s": 0.06795627866813447, "kv_live_bytes": 462274560, "latency_s": 2.17698224959895, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73728000, "prompt_kind": "long", "t_relative_s": 6128.164652958047}, {"decode_latency_s": 0.06027522656950168, "kv_live_bytes": 462274560, "latency_s": 1.9296477087773383, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74268672, "prompt_kind": "short", "t_relative_s": 6133.1747208749875}, {"decode_latency_s": 0.06053515493113082, "kv_live_bytes": 462274560, "latency_s": 1.9380102083086967, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73072640, "prompt_kind": "long", "t_relative_s": 6138.184795042034}, {"decode_latency_s": 0.060267312510404736, "kv_live_bytes": 462274560, "latency_s": 1.9295053747482598, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73170944, "prompt_kind": "short", "t_relative_s": 6143.1948533752}, {"decode_latency_s": 0.06771073827985674, "kv_live_bytes": 462274560, "latency_s": 2.1690715420991182, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73711616, "prompt_kind": "long", "t_relative_s": 6148.204938125331}, {"decode_latency_s": 0.0604764231829904, "kv_live_bytes": 462274560, "latency_s": 1.9360253745689988, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73711616, "prompt_kind": "short", "t_relative_s": 6153.2150179580785}, {"decode_latency_s": 0.06802722266002093, "kv_live_bytes": 462274560, "latency_s": 2.1793033331632614, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73826304, "prompt_kind": "long", "t_relative_s": 6158.225099042058}, {"decode_latency_s": 0.060613296867813915, "kv_live_bytes": 462274560, "latency_s": 1.940498541109264, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74366976, "prompt_kind": "short", "t_relative_s": 6163.235182708129}, {"decode_latency_s": 0.06777385156601667, "kv_live_bytes": 462274560, "latency_s": 2.170665374957025, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74137600, "prompt_kind": "long", "t_relative_s": 6168.245259958319}, {"decode_latency_s": 0.06036099868651945, "kv_live_bytes": 462274560, "latency_s": 1.9325249157845974, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74203136, "prompt_kind": "short", "t_relative_s": 6173.255309500266}, {"decode_latency_s": 0.06025839973881375, "kv_live_bytes": 462274560, "latency_s": 1.9291702499613166, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74268672, "prompt_kind": "long", "t_relative_s": 6178.265390167013}, {"decode_latency_s": 0.06051263150584418, "kv_live_bytes": 462274560, "latency_s": 1.9371860423125327, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74842112, "prompt_kind": "short", "t_relative_s": 6183.2754810419865}, {"decode_latency_s": 0.06754006378469057, "kv_live_bytes": 462274560, "latency_s": 2.163632749579847, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72876032, "prompt_kind": "long", "t_relative_s": 6188.285534667317}, {"decode_latency_s": 0.06060721352696419, "kv_live_bytes": 462274560, "latency_s": 1.9403336672112346, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73515008, "prompt_kind": "short", "t_relative_s": 6193.295580583159}, {"decode_latency_s": 0.06815589976031333, "kv_live_bytes": 462274560, "latency_s": 2.183439166750759, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73695232, "prompt_kind": "long", "t_relative_s": 6198.30564900022}, {"decode_latency_s": 0.060493255223263986, "kv_live_bytes": 462274560, "latency_s": 1.9365654583089054, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73105408, "prompt_kind": "short", "t_relative_s": 6203.315735083073}, {"decode_latency_s": 0.06782836066849995, "kv_live_bytes": 462274560, "latency_s": 2.1725026671774685, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73089024, "prompt_kind": "long", "t_relative_s": 6208.325790833216}, {"decode_latency_s": 0.060291213550954126, "kv_live_bytes": 462274560, "latency_s": 1.9302142914384604, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73089024, "prompt_kind": "short", "t_relative_s": 6213.335861042142}, {"decode_latency_s": 0.06748441405943595, "kv_live_bytes": 462274560, "latency_s": 2.1618750002235174, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73547776, "prompt_kind": "long", "t_relative_s": 6218.3459210833535}, {"decode_latency_s": 0.06087006772577297, "kv_live_bytes": 462274560, "latency_s": 1.9485877500846982, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74121216, "prompt_kind": "short", "t_relative_s": 6223.356025542133}, {"decode_latency_s": 0.06813049738411792, "kv_live_bytes": 462274560, "latency_s": 2.181814916897565, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73629696, "prompt_kind": "long", "t_relative_s": 6228.36370645836}, {"decode_latency_s": 0.06024862889898941, "kv_live_bytes": 462274560, "latency_s": 1.9288696660660207, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73515008, "prompt_kind": "short", "t_relative_s": 6233.373754417058}, {"decode_latency_s": 0.06803939453675412, "kv_live_bytes": 462274560, "latency_s": 2.1796649578027427, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73728000, "prompt_kind": "long", "t_relative_s": 6238.383829625323}, {"decode_latency_s": 0.060686111959512345, "kv_live_bytes": 462274560, "latency_s": 1.9427461251616478, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74350592, "prompt_kind": "short", "t_relative_s": 6243.3938747919165}, {"decode_latency_s": 0.06733484765572939, "kv_live_bytes": 462274560, "latency_s": 2.157046042382717, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74465280, "prompt_kind": "long", "t_relative_s": 6248.403943000361}, {"decode_latency_s": 0.06065216015849728, "kv_live_bytes": 462274560, "latency_s": 1.9416377088055015, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75005952, "prompt_kind": "short", "t_relative_s": 6253.414016417228}, {"decode_latency_s": 0.06800465626292862, "kv_live_bytes": 462274560, "latency_s": 2.1779562919400632, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74989568, "prompt_kind": "long", "t_relative_s": 6258.414721542038}, {"decode_latency_s": 0.060390023441868834, "kv_live_bytes": 462274560, "latency_s": 1.9333529160358012, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74465280, "prompt_kind": "short", "t_relative_s": 6263.42479391722}, {"decode_latency_s": 0.06853612631675787, "kv_live_bytes": 462274560, "latency_s": 2.19510304229334, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74498048, "prompt_kind": "long", "t_relative_s": 6268.434871792328}, {"decode_latency_s": 0.06057882292952854, "kv_live_bytes": 462274560, "latency_s": 1.9394039581529796, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75022336, "prompt_kind": "short", "t_relative_s": 6273.444937208202}, {"decode_latency_s": 0.06068117187533062, "kv_live_bytes": 462274560, "latency_s": 1.942764916922897, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75104256, "prompt_kind": "long", "t_relative_s": 6278.455013333354}, {"decode_latency_s": 0.06255679296737071, "kv_live_bytes": 462274560, "latency_s": 2.0035589998587966, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75743232, "prompt_kind": "short", "t_relative_s": 6283.465098999906}, {"decode_latency_s": 0.06812428125704173, "kv_live_bytes": 462274560, "latency_s": 2.182148292195052, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75710464, "prompt_kind": "long", "t_relative_s": 6288.475154333282}, {"decode_latency_s": 0.060262059909291565, "kv_live_bytes": 462274560, "latency_s": 1.9292968749068677, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75137024, "prompt_kind": "short", "t_relative_s": 6293.485227708239}, {"decode_latency_s": 0.06825944922456983, "kv_live_bytes": 462274560, "latency_s": 2.18602008279413, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75284480, "prompt_kind": "long", "t_relative_s": 6298.495320708025}, {"decode_latency_s": 0.06048230468877591, "kv_live_bytes": 462274560, "latency_s": 1.9363155830651522, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75333632, "prompt_kind": "short", "t_relative_s": 6303.504505250137}, {"decode_latency_s": 0.06780761849950068, "kv_live_bytes": 462274560, "latency_s": 2.1720944168046117, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78397440, "prompt_kind": "long", "t_relative_s": 6308.514597374946}, {"decode_latency_s": 0.06054704816779122, "kv_live_bytes": 462274560, "latency_s": 1.9382908749394119, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78970880, "prompt_kind": "short", "t_relative_s": 6313.524680292234}, {"decode_latency_s": 0.06788160806172527, "kv_live_bytes": 462274560, "latency_s": 2.1731307078152895, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72056832, "prompt_kind": "long", "t_relative_s": 6318.534729250241}, {"decode_latency_s": 0.06046995572978631, "kv_live_bytes": 462274560, "latency_s": 1.9358299998566508, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73547776, "prompt_kind": "short", "t_relative_s": 6323.699963625055}, {"decode_latency_s": 0.06798473828530405, "kv_live_bytes": 462274560, "latency_s": 2.1781647498719394, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73564160, "prompt_kind": "long", "t_relative_s": 6328.71001070831}, {"decode_latency_s": 0.06036605077679269, "kv_live_bytes": 462274560, "latency_s": 1.9325609160587192, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73990144, "prompt_kind": "short", "t_relative_s": 6333.720090833027}, {"decode_latency_s": 0.06793391796236392, "kv_live_bytes": 462274560, "latency_s": 2.176781083922833, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74170368, "prompt_kind": "long", "t_relative_s": 6338.730148083065}, {"decode_latency_s": 0.06086500649689697, "kv_live_bytes": 462274560, "latency_s": 1.9485772079788148, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74547200, "prompt_kind": "short", "t_relative_s": 6343.740207375027}, {"decode_latency_s": 0.06808588672720362, "kv_live_bytes": 462274560, "latency_s": 2.1806753748096526, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73498624, "prompt_kind": "long", "t_relative_s": 6348.750249583274}, {"decode_latency_s": 0.06067173175688367, "kv_live_bytes": 462274560, "latency_s": 1.9424148327670991, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73351168, "prompt_kind": "short", "t_relative_s": 6353.760318083223}, {"decode_latency_s": 0.06817729557224084, "kv_live_bytes": 462274560, "latency_s": 2.1842388748191297, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73678848, "prompt_kind": "long", "t_relative_s": 6358.77039545821}, {"decode_latency_s": 0.06070319400168955, "kv_live_bytes": 462274560, "latency_s": 1.9433535831049085, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74317824, "prompt_kind": "short", "t_relative_s": 6363.780446417164}, {"decode_latency_s": 0.06715558202995453, "kv_live_bytes": 462274560, "latency_s": 2.1513521657325327, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74383360, "prompt_kind": "long", "t_relative_s": 6368.788684458006}, {"decode_latency_s": 0.060818348938482814, "kv_live_bytes": 462274560, "latency_s": 1.946981250308454, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74891264, "prompt_kind": "short", "t_relative_s": 6373.798754208256}, {"decode_latency_s": 0.06768316144007258, "kv_live_bytes": 462274560, "latency_s": 2.167874749749899, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74907648, "prompt_kind": "long", "t_relative_s": 6378.808810375165}, {"decode_latency_s": 0.06018417968880385, "kv_live_bytes": 462274560, "latency_s": 1.9269143748097122, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73809920, "prompt_kind": "short", "t_relative_s": 6383.818900083192}, {"decode_latency_s": 0.06798205077939201, "kv_live_bytes": 462274560, "latency_s": 2.177006166893989, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72728576, "prompt_kind": "long", "t_relative_s": 6388.8289670832455}, {"decode_latency_s": 0.06036805600160733, "kv_live_bytes": 462274560, "latency_s": 1.9327046247199178, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73302016, "prompt_kind": "short", "t_relative_s": 6393.836654208135}, {"decode_latency_s": 0.06788227996730711, "kv_live_bytes": 462274560, "latency_s": 2.1744661671109498, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73334784, "prompt_kind": "long", "t_relative_s": 6398.846722875256}, {"decode_latency_s": 0.06057142840290908, "kv_live_bytes": 462274560, "latency_s": 1.939124541822821, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73891840, "prompt_kind": "short", "t_relative_s": 6403.856791708153}, {"decode_latency_s": 0.06801296224875841, "kv_live_bytes": 462274560, "latency_s": 2.1787322079762816, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73695232, "prompt_kind": "long", "t_relative_s": 6408.866839167196}, {"decode_latency_s": 0.060300062512396835, "kv_live_bytes": 462274560, "latency_s": 1.930606042034924, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72499200, "prompt_kind": "short", "t_relative_s": 6413.8769162921235}, {"decode_latency_s": 0.06821711717930157, "kv_live_bytes": 462274560, "latency_s": 2.184833041857928, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73121792, "prompt_kind": "long", "t_relative_s": 6418.886999083217}, {"decode_latency_s": 0.08154847002879251, "kv_live_bytes": 462274560, "latency_s": 2.610342166852206, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68763648, "prompt_kind": "short", "t_relative_s": 6423.897054166999}, {"decode_latency_s": 0.06793871225090697, "kv_live_bytes": 462274560, "latency_s": 2.1765197906643152, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69419008, "prompt_kind": "long", "t_relative_s": 6431.324171083048}, {"decode_latency_s": 0.061036606784909964, "kv_live_bytes": 462274560, "latency_s": 1.9540103748440742, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69910528, "prompt_kind": "short", "t_relative_s": 6436.334249458276}, {"decode_latency_s": 0.06821274608955719, "kv_live_bytes": 462274560, "latency_s": 2.1849205000326037, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69304320, "prompt_kind": "long", "t_relative_s": 6441.344295124989}, {"decode_latency_s": 0.0602965325670084, "kv_live_bytes": 462274560, "latency_s": 1.9303709589876235, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69304320, "prompt_kind": "short", "t_relative_s": 6446.3543520001695}, {"decode_latency_s": 0.06766182029969059, "kv_live_bytes": 462274560, "latency_s": 2.1672371667809784, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69517312, "prompt_kind": "long", "t_relative_s": 6451.364424250089}, {"decode_latency_s": 0.06095165364968125, "kv_live_bytes": 462274560, "latency_s": 1.9512707497924566, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70156288, "prompt_kind": "short", "t_relative_s": 6456.374465500005}, {"decode_latency_s": 0.06883889452728909, "kv_live_bytes": 462274560, "latency_s": 2.2052392079494894, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70172672, "prompt_kind": "long", "t_relative_s": 6461.3845397080295}, {"decode_latency_s": 0.0624699830514146, "kv_live_bytes": 462274560, "latency_s": 2.000049250200391, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69009408, "prompt_kind": "short", "t_relative_s": 6466.3946235422045}, {"decode_latency_s": 0.06749764452979434, "kv_live_bytes": 462274560, "latency_s": 2.162504583131522, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70041600, "prompt_kind": "long", "t_relative_s": 6471.404692667071}, {"decode_latency_s": 0.06701876825536601, "kv_live_bytes": 462274560, "latency_s": 2.1462977919727564, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70123520, "prompt_kind": "short", "t_relative_s": 6476.407824625261}, {"decode_latency_s": 0.06775108985311817, "kv_live_bytes": 462274560, "latency_s": 2.1703587910160422, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70090752, "prompt_kind": "long", "t_relative_s": 6481.417894625105}, {"decode_latency_s": 0.06061944141401909, "kv_live_bytes": 462274560, "latency_s": 1.9405785826966166, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70729728, "prompt_kind": "short", "t_relative_s": 6486.427964916918}, {"decode_latency_s": 0.06836201433907263, "kv_live_bytes": 462274560, "latency_s": 2.18916904181242, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70762496, "prompt_kind": "long", "t_relative_s": 6491.438044750132}, {"decode_latency_s": 0.060534480478963815, "kv_live_bytes": 462274560, "latency_s": 1.9379963329993188, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71368704, "prompt_kind": "short", "t_relative_s": 6496.448094916996}, {"decode_latency_s": 0.06819571483356413, "kv_live_bytes": 462274560, "latency_s": 2.184683416970074, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71532544, "prompt_kind": "long", "t_relative_s": 6501.458182542119}, {"decode_latency_s": 0.06080438803473953, "kv_live_bytes": 462274560, "latency_s": 1.9465087498538196, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71680000, "prompt_kind": "short", "t_relative_s": 6506.468278875109}, {"decode_latency_s": 0.06787971094308887, "kv_live_bytes": 462274560, "latency_s": 2.1744786659255624, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71712768, "prompt_kind": "long", "t_relative_s": 6511.478328708094}, {"decode_latency_s": 0.060536717472132295, "kv_live_bytes": 462274560, "latency_s": 1.938100166618824, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72286208, "prompt_kind": "short", "t_relative_s": 6516.488406707998}, {"decode_latency_s": 0.06811385415494442, "kv_live_bytes": 462274560, "latency_s": 2.1815312090329826, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72237056, "prompt_kind": "long", "t_relative_s": 6521.49850779213}, {"decode_latency_s": 0.06703488022321835, "kv_live_bytes": 462274560, "latency_s": 2.146816208027303, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72302592, "prompt_kind": "short", "t_relative_s": 6526.508558917325}, {"decode_latency_s": 0.0677392968791537, "kv_live_bytes": 462274560, "latency_s": 2.170055333059281, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72335360, "prompt_kind": "long", "t_relative_s": 6531.518643625081}, {"decode_latency_s": 0.060572498725377955, "kv_live_bytes": 462274560, "latency_s": 1.9391660001128912, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72417280, "prompt_kind": "short", "t_relative_s": 6536.528701208066}, {"decode_latency_s": 0.06731744010176044, "kv_live_bytes": 462274560, "latency_s": 2.1565941656008363, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72450048, "prompt_kind": "long", "t_relative_s": 6541.538754791953}, {"decode_latency_s": 0.06042759766569361, "kv_live_bytes": 462274560, "latency_s": 1.9345664996653795, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72957952, "prompt_kind": "short", "t_relative_s": 6546.548836458009}, {"decode_latency_s": 0.06040541795664467, "kv_live_bytes": 462274560, "latency_s": 1.9342653332278132, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73023488, "prompt_kind": "long", "t_relative_s": 6551.558926917147}, {"decode_latency_s": 0.06703950518567581, "kv_live_bytes": 462274560, "latency_s": 2.1467580827884376, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73039872, "prompt_kind": "short", "t_relative_s": 6556.568980542012}, {"decode_latency_s": 0.06812583463033661, "kv_live_bytes": 462274560, "latency_s": 2.1823657918721437, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73138176, "prompt_kind": "long", "t_relative_s": 6561.579065208323}, {"decode_latency_s": 0.060623891913564876, "kv_live_bytes": 462274560, "latency_s": 1.9407445839606225, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73203712, "prompt_kind": "short", "t_relative_s": 6566.589173083194}, {"decode_latency_s": 0.06810564712213818, "kv_live_bytes": 462274560, "latency_s": 2.1814750423654914, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73236480, "prompt_kind": "long", "t_relative_s": 6571.599230791908}, {"decode_latency_s": 0.06048735024523921, "kv_live_bytes": 462274560, "latency_s": 1.93651287490502, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73695232, "prompt_kind": "short", "t_relative_s": 6576.609318125062}, {"decode_latency_s": 0.06798610938130878, "kv_live_bytes": 462274560, "latency_s": 2.177558584138751, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73777152, "prompt_kind": "long", "t_relative_s": 6581.619408333208}, {"decode_latency_s": 0.06129327733651735, "kv_live_bytes": 462274560, "latency_s": 1.9623019164428115, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73908224, "prompt_kind": "short", "t_relative_s": 6586.629457667004}, {"decode_latency_s": 0.06940429037786089, "kv_live_bytes": 462274560, "latency_s": 2.2234986247494817, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73908224, "prompt_kind": "long", "t_relative_s": 6591.6395622082055}, {"decode_latency_s": 0.06692745834880043, "kv_live_bytes": 462274560, "latency_s": 2.14324662508443, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72859648, "prompt_kind": "short", "t_relative_s": 6596.646474624984}, {"decode_latency_s": 0.06802198306831997, "kv_live_bytes": 462274560, "latency_s": 2.1794455838389695, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73367552, "prompt_kind": "long", "t_relative_s": 6601.656512792222}, {"decode_latency_s": 0.06040773828863166, "kv_live_bytes": 462274560, "latency_s": 1.9339251657947898, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73908224, "prompt_kind": "short", "t_relative_s": 6606.666601125151}, {"decode_latency_s": 0.06813787890132517, "kv_live_bytes": 462274560, "latency_s": 2.182209166698158, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73695232, "prompt_kind": "long", "t_relative_s": 6611.676691625267}, {"decode_latency_s": 0.06148922396823764, "kv_live_bytes": 462274560, "latency_s": 1.96876479126513, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73465856, "prompt_kind": "short", "t_relative_s": 6616.686742166989}, {"decode_latency_s": 0.0675954283651663, "kv_live_bytes": 462274560, "latency_s": 2.1656034174375236, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74137600, "prompt_kind": "long", "t_relative_s": 6621.696828042157}, {"decode_latency_s": 0.06073211328475736, "kv_live_bytes": 462274560, "latency_s": 1.9442346249707043, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74268672, "prompt_kind": "short", "t_relative_s": 6626.706761542242}, {"decode_latency_s": 0.060474348938441835, "kv_live_bytes": 462274560, "latency_s": 1.9359965417534113, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73629696, "prompt_kind": "long", "t_relative_s": 6631.716810292099}, {"decode_latency_s": 0.06064325390616432, "kv_live_bytes": 462274560, "latency_s": 1.9414317500777543, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74186752, "prompt_kind": "short", "t_relative_s": 6636.726899583358}, {"decode_latency_s": 0.06903509503172245, "kv_live_bytes": 462274560, "latency_s": 2.2117362916469574, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72531968, "prompt_kind": "long", "t_relative_s": 6641.736986082979}, {"decode_latency_s": 0.06737803386931773, "kv_live_bytes": 462274560, "latency_s": 2.1578103750944138, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72810496, "prompt_kind": "short", "t_relative_s": 6646.747040250339}, {"decode_latency_s": 0.06713513018621597, "kv_live_bytes": 462274560, "latency_s": 2.1502767908386886, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73105408, "prompt_kind": "long", "t_relative_s": 6651.749470625073}, {"decode_latency_s": 0.06220977606426459, "kv_live_bytes": 462274560, "latency_s": 1.9915132499299943, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69861376, "prompt_kind": "short", "t_relative_s": 6656.754676583223}, {"decode_latency_s": 0.06788123046862893, "kv_live_bytes": 462274560, "latency_s": 2.1749926251359284, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70320128, "prompt_kind": "long", "t_relative_s": 6663.129346167203}, {"decode_latency_s": 0.060928015620447695, "kv_live_bytes": 462274560, "latency_s": 1.950544792227447, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70270976, "prompt_kind": "short", "t_relative_s": 6668.139440916944}, {"decode_latency_s": 0.06824966143176425, "kv_live_bytes": 462274560, "latency_s": 2.1860095830634236, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70860800, "prompt_kind": "long", "t_relative_s": 6673.149533083197}, {"decode_latency_s": 0.060839850251795724, "kv_live_bytes": 462274560, "latency_s": 1.947775417007506, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71434240, "prompt_kind": "short", "t_relative_s": 6678.1595918331295}, {"decode_latency_s": 0.06779638802981935, "kv_live_bytes": 462274560, "latency_s": 2.1719587077386677, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71483392, "prompt_kind": "long", "t_relative_s": 6683.169666000176}, {"decode_latency_s": 0.06054126824892592, "kv_live_bytes": 462274560, "latency_s": 1.9385538334026933, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71565312, "prompt_kind": "short", "t_relative_s": 6688.179741625208}, {"decode_latency_s": 0.06754533464845736, "kv_live_bytes": 462274560, "latency_s": 2.163694041315466, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70909952, "prompt_kind": "long", "t_relative_s": 6693.189790083095}, {"decode_latency_s": 0.06029063022288028, "kv_live_bytes": 462274560, "latency_s": 1.9301736671477556, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70877184, "prompt_kind": "short", "t_relative_s": 6698.199855083134}, {"decode_latency_s": 0.06811471484252252, "kv_live_bytes": 462274560, "latency_s": 2.1819858751259744, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71417856, "prompt_kind": "long", "t_relative_s": 6703.209932292346}, {"decode_latency_s": 0.06067425649962388, "kv_live_bytes": 462274560, "latency_s": 1.9424776253290474, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72040448, "prompt_kind": "short", "t_relative_s": 6708.219996583182}, {"decode_latency_s": 0.06831242446787655, "kv_live_bytes": 462274560, "latency_s": 2.1885259998962283, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71942144, "prompt_kind": "long", "t_relative_s": 6713.230069833342}, {"decode_latency_s": 0.06063889975484926, "kv_live_bytes": 462274560, "latency_s": 1.9412361672148108, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72105984, "prompt_kind": "short", "t_relative_s": 6718.240160082933}, {"decode_latency_s": 0.06782830206793733, "kv_live_bytes": 462274560, "latency_s": 2.1726888748817146, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71614464, "prompt_kind": "long", "t_relative_s": 6723.24338775035}, {"decode_latency_s": 0.06024668618920259, "kv_live_bytes": 462274560, "latency_s": 1.9288125843741, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72253440, "prompt_kind": "short", "t_relative_s": 6728.253468375187}, {"decode_latency_s": 0.06790773960528895, "kv_live_bytes": 462274560, "latency_s": 2.174685582984239, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72318976, "prompt_kind": "long", "t_relative_s": 6733.26354425028}, {"decode_latency_s": 0.060611463559325784, "kv_live_bytes": 462274560, "latency_s": 1.9404754168353975, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72826880, "prompt_kind": "short", "t_relative_s": 6738.2736004581675}, {"decode_latency_s": 0.06825083075091243, "kv_live_bytes": 462274560, "latency_s": 2.1864239582791924, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72908800, "prompt_kind": "long", "t_relative_s": 6743.283681124914}, {"decode_latency_s": 0.06054655209300108, "kv_live_bytes": 462274560, "latency_s": 1.938289541285485, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72974336, "prompt_kind": "short", "t_relative_s": 6748.293751250021}, {"decode_latency_s": 0.06054599872732069, "kv_live_bytes": 462274560, "latency_s": 1.9387172916904092, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73089024, "prompt_kind": "long", "t_relative_s": 6753.303796750028}, {"decode_latency_s": 0.060258484387304634, "kv_live_bytes": 462274560, "latency_s": 1.9291852079331875, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73039872, "prompt_kind": "short", "t_relative_s": 6758.313880708069}, {"decode_latency_s": 0.06862893099605571, "kv_live_bytes": 462274560, "latency_s": 2.198017292190343, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73089024, "prompt_kind": "long", "t_relative_s": 6763.323949958198}, {"decode_latency_s": 0.0604484935029177, "kv_live_bytes": 462274560, "latency_s": 1.9353173752315342, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73154560, "prompt_kind": "short", "t_relative_s": 6768.3340231669135}, {"decode_latency_s": 0.06782393230241723, "kv_live_bytes": 462274560, "latency_s": 2.1728618750348687, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73203712, "prompt_kind": "long", "t_relative_s": 6773.34410912497}, {"decode_latency_s": 0.060575592462555505, "kv_live_bytes": 462274560, "latency_s": 1.939256208948791, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73203712, "prompt_kind": "short", "t_relative_s": 6778.354189957958}, {"decode_latency_s": 0.06071783594961744, "kv_live_bytes": 462274560, "latency_s": 1.9445243342779577, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73285632, "prompt_kind": "long", "t_relative_s": 6783.364238083363}, {"decode_latency_s": 0.06023632813594304, "kv_live_bytes": 462274560, "latency_s": 1.9285018327645957, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73154560, "prompt_kind": "short", "t_relative_s": 6788.371253417339}, {"decode_latency_s": 0.06855343749339227, "kv_live_bytes": 462274560, "latency_s": 2.1957504581660032, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73285632, "prompt_kind": "long", "t_relative_s": 6793.381337624975}, {"decode_latency_s": 0.06042363931192085, "kv_live_bytes": 462274560, "latency_s": 1.9344112081453204, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73285632, "prompt_kind": "short", "t_relative_s": 6798.391404417343}, {"decode_latency_s": 0.06811639711668249, "kv_live_bytes": 462274560, "latency_s": 2.1821763748303056, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73416704, "prompt_kind": "long", "t_relative_s": 6803.401466458105}, {"decode_latency_s": 0.0604990768188145, "kv_live_bytes": 462274560, "latency_s": 1.9367217076942325, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "short", "t_relative_s": 6808.411543125287}, {"decode_latency_s": 0.06803874087927397, "kv_live_bytes": 462274560, "latency_s": 2.1797041250392795, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73269248, "prompt_kind": "long", "t_relative_s": 6813.421601792332}, {"decode_latency_s": 0.06034891927265562, "kv_live_bytes": 462274560, "latency_s": 1.9324356662109494, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "short", "t_relative_s": 6818.431636957917}, {"decode_latency_s": 0.0686035781254759, "kv_live_bytes": 462274560, "latency_s": 2.197694499976933, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73482240, "prompt_kind": "long", "t_relative_s": 6823.441719749942}, {"decode_latency_s": 0.0666248072811868, "kv_live_bytes": 462274560, "latency_s": 2.1336878747679293, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74039296, "prompt_kind": "short", "t_relative_s": 6828.451781624928}, {"decode_latency_s": 0.06784063021768816, "kv_live_bytes": 462274560, "latency_s": 2.173622709233314, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74039296, "prompt_kind": "long", "t_relative_s": 6833.46185025014}, {"decode_latency_s": 0.06051137109170668, "kv_live_bytes": 462274560, "latency_s": 1.9371552909724414, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74072064, "prompt_kind": "short", "t_relative_s": 6838.471927457955}, {"decode_latency_s": 0.06790853777783923, "kv_live_bytes": 462274560, "latency_s": 2.1747387498617172, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73957376, "prompt_kind": "long", "t_relative_s": 6843.481952458154}, {"decode_latency_s": 0.06035283334495034, "kv_live_bytes": 462274560, "latency_s": 1.9323892495594919, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73383936, "prompt_kind": "short", "t_relative_s": 6848.490829417016}, {"decode_latency_s": 0.06807419141114224, "kv_live_bytes": 462274560, "latency_s": 2.180523042101413, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74072064, "prompt_kind": "long", "t_relative_s": 6853.500909542199}, {"decode_latency_s": 0.06059661981998943, "kv_live_bytes": 462274560, "latency_s": 1.939896292053163, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74645504, "prompt_kind": "short", "t_relative_s": 6858.510989542119}, {"decode_latency_s": 0.06783013543463312, "kv_live_bytes": 462274560, "latency_s": 2.1729437075555325, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74727424, "prompt_kind": "long", "t_relative_s": 6863.521061208099}, {"decode_latency_s": 0.060814214855781756, "kv_live_bytes": 462274560, "latency_s": 1.9468059167265892, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74809344, "prompt_kind": "short", "t_relative_s": 6868.531142792199}, {"decode_latency_s": 0.06752500653965399, "kv_live_bytes": 462274560, "latency_s": 2.163085165899247, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74694656, "prompt_kind": "long", "t_relative_s": 6873.541190833319}, {"decode_latency_s": 0.0603371875040466, "kv_live_bytes": 462274560, "latency_s": 1.9316887920722365, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75317248, "prompt_kind": "short", "t_relative_s": 6878.5512460833415}, {"decode_latency_s": 0.06514560937648639, "kv_live_bytes": 462274560, "latency_s": 2.086497541051358, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75399168, "prompt_kind": "long", "t_relative_s": 6883.561333416961}, {"decode_latency_s": 0.060483555993414484, "kv_live_bytes": 462274560, "latency_s": 1.9362960420548916, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75300864, "prompt_kind": "short", "t_relative_s": 6888.571395708248}, {"decode_latency_s": 0.06402310419071, "kv_live_bytes": 462274560, "latency_s": 2.050088125281036, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71254016, "prompt_kind": "long", "t_relative_s": 6893.581503957976}, {"decode_latency_s": 0.06044693231524434, "kv_live_bytes": 462274560, "latency_s": 1.9351775832474232, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71729152, "prompt_kind": "short", "t_relative_s": 6899.448747625109}, {"decode_latency_s": 0.06803389193373732, "kv_live_bytes": 462274560, "latency_s": 2.179498916026205, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71909376, "prompt_kind": "long", "t_relative_s": 6904.458802875131}, {"decode_latency_s": 0.06094789321650751, "kv_live_bytes": 462274560, "latency_s": 1.9512512078508735, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72433664, "prompt_kind": "short", "t_relative_s": 6909.468900416978}, {"decode_latency_s": 0.06874694272119086, "kv_live_bytes": 462274560, "latency_s": 2.2020337502472103, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71204864, "prompt_kind": "long", "t_relative_s": 6914.478955667}, {"decode_latency_s": 0.060417270855396055, "kv_live_bytes": 462274560, "latency_s": 1.9342564581893384, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69681152, "prompt_kind": "short", "t_relative_s": 6919.489029542077}, {"decode_latency_s": 0.06935764584341086, "kv_live_bytes": 462274560, "latency_s": 2.221222249791026, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70877184, "prompt_kind": "long", "t_relative_s": 6924.49910487514}, {"decode_latency_s": 0.060618161427555606, "kv_live_bytes": 462274560, "latency_s": 1.9405664158985019, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71008256, "prompt_kind": "short", "t_relative_s": 6929.509203458205}, {"decode_latency_s": 0.06815244010067545, "kv_live_bytes": 462274560, "latency_s": 2.183843707665801, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71434240, "prompt_kind": "long", "t_relative_s": 6934.519253833219}, {"decode_latency_s": 0.06029174737341236, "kv_live_bytes": 462274560, "latency_s": 1.930234500207007, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72105984, "prompt_kind": "short", "t_relative_s": 6939.529284083284}, {"decode_latency_s": 0.0682766302197706, "kv_live_bytes": 462274560, "latency_s": 2.1874222080223262, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72286208, "prompt_kind": "long", "t_relative_s": 6944.539351500105}, {"decode_latency_s": 0.06057329296891112, "kv_live_bytes": 462274560, "latency_s": 1.939168291632086, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72843264, "prompt_kind": "short", "t_relative_s": 6949.549409583211}, {"decode_latency_s": 0.06710867577930912, "kv_live_bytes": 462274560, "latency_s": 2.149984208866954, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76300288, "prompt_kind": "long", "t_relative_s": 6954.559464416932}, {"decode_latency_s": 0.06676679555675946, "kv_live_bytes": 462274560, "latency_s": 2.138285582885146, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76365824, "prompt_kind": "short", "t_relative_s": 6959.56953795813}, {"decode_latency_s": 0.06840181117877364, "kv_live_bytes": 462274560, "latency_s": 2.1913007921539247, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74334208, "prompt_kind": "long", "t_relative_s": 6964.579585167114}, {"decode_latency_s": 0.06022480990213808, "kv_live_bytes": 462274560, "latency_s": 1.9281031666323543, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74219520, "prompt_kind": "short", "t_relative_s": 6969.5896417503245}, {"decode_latency_s": 0.06844901044678409, "kv_live_bytes": 462274560, "latency_s": 2.192008042242378, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74711040, "prompt_kind": "long", "t_relative_s": 6974.599718875252}, {"decode_latency_s": 0.06681322134681977, "kv_live_bytes": 462274560, "latency_s": 2.1397442501038313, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75284480, "prompt_kind": "short", "t_relative_s": 6979.609759625047}, {"decode_latency_s": 0.067910436206148, "kv_live_bytes": 462274560, "latency_s": 2.1757638338021934, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75399168, "prompt_kind": "long", "t_relative_s": 6984.61983929202}, {"decode_latency_s": 0.060810083348769695, "kv_live_bytes": 462274560, "latency_s": 1.9467107919044793, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75202560, "prompt_kind": "short", "t_relative_s": 6989.62193437526}, {"decode_latency_s": 0.06825759897765238, "kv_live_bytes": 462274560, "latency_s": 2.1866439171135426, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75235328, "prompt_kind": "long", "t_relative_s": 6994.631985000335}, {"decode_latency_s": 0.06020677734341007, "kv_live_bytes": 462274560, "latency_s": 1.9275576672516763, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75055104, "prompt_kind": "short", "t_relative_s": 6999.642064624932}, {"decode_latency_s": 0.06822254559665453, "kv_live_bytes": 462274560, "latency_s": 2.184775499626994, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75169792, "prompt_kind": "long", "t_relative_s": 7004.652127292007}, {"decode_latency_s": 0.06065628906071652, "kv_live_bytes": 462274560, "latency_s": 1.941864333115518, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75710464, "prompt_kind": "short", "t_relative_s": 7009.662205750123}, {"decode_latency_s": 0.06769943359540775, "kv_live_bytes": 462274560, "latency_s": 2.1688099172897637, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75857920, "prompt_kind": "long", "t_relative_s": 7014.672271292191}, {"decode_latency_s": 0.06054500000027474, "kv_live_bytes": 462274560, "latency_s": 1.9382267920300364, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75874304, "prompt_kind": "short", "t_relative_s": 7019.682366208173}, {"decode_latency_s": 0.06840506900334731, "kv_live_bytes": 462274560, "latency_s": 2.191465000156313, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75317248, "prompt_kind": "long", "t_relative_s": 7024.69242304191}, {"decode_latency_s": 0.060321815093629993, "kv_live_bytes": 462274560, "latency_s": 1.931181417312473, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75202560, "prompt_kind": "short", "t_relative_s": 7029.694860457908}, {"decode_latency_s": 0.06788559375854675, "kv_live_bytes": 462274560, "latency_s": 2.1741069168783724, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75137024, "prompt_kind": "long", "t_relative_s": 7034.70493687503}, {"decode_latency_s": 0.060350222658598796, "kv_live_bytes": 462274560, "latency_s": 1.9319648332893848, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75857920, "prompt_kind": "short", "t_relative_s": 7039.714995208196}, {"decode_latency_s": 0.06792755337664858, "kv_live_bytes": 462274560, "latency_s": 2.176520749926567, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76316672, "prompt_kind": "long", "t_relative_s": 7044.7250861250795}, {"decode_latency_s": 0.060142270842334256, "kv_live_bytes": 462274560, "latency_s": 1.9258347498252988, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76464128, "prompt_kind": "short", "t_relative_s": 7049.735140417237}, {"decode_latency_s": 0.06812575132062193, "kv_live_bytes": 462274560, "latency_s": 2.182192667387426, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76513280, "prompt_kind": "long", "t_relative_s": 7054.745216458105}, {"decode_latency_s": 0.06045704166172072, "kv_live_bytes": 462274560, "latency_s": 1.93543391674757, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76529664, "prompt_kind": "short", "t_relative_s": 7059.755332082976}, {"decode_latency_s": 0.06781800130556803, "kv_live_bytes": 462274560, "latency_s": 2.172644999809563, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76627968, "prompt_kind": "long", "t_relative_s": 7064.76538245799}, {"decode_latency_s": 0.06033712369389832, "kv_live_bytes": 462274560, "latency_s": 1.9315768331289291, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76644352, "prompt_kind": "short", "t_relative_s": 7069.775458374992}, {"decode_latency_s": 0.06821641279384494, "kv_live_bytes": 462274560, "latency_s": 2.184670540969819, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76627968, "prompt_kind": "long", "t_relative_s": 7074.785539083183}, {"decode_latency_s": 0.06030397916038055, "kv_live_bytes": 462274560, "latency_s": 1.9306435002945364, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76693504, "prompt_kind": "short", "t_relative_s": 7079.795590917114}, {"decode_latency_s": 0.06778747524367645, "kv_live_bytes": 462274560, "latency_s": 2.1716428752988577, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76759040, "prompt_kind": "long", "t_relative_s": 7084.805678833276}, {"decode_latency_s": 0.06061357681755908, "kv_live_bytes": 462274560, "latency_s": 1.9404225829057395, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77332480, "prompt_kind": "short", "t_relative_s": 7089.815772417001}, {"decode_latency_s": 0.06762485025683418, "kv_live_bytes": 462274560, "latency_s": 2.166610708925873, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76808192, "prompt_kind": "long", "t_relative_s": 7094.825827583205}, {"decode_latency_s": 0.0603937018167926, "kv_live_bytes": 462274560, "latency_s": 1.9334585419856012, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77365248, "prompt_kind": "short", "t_relative_s": 7099.835907167289}, {"decode_latency_s": 0.06809217056434136, "kv_live_bytes": 462274560, "latency_s": 2.180301041342318, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77430784, "prompt_kind": "long", "t_relative_s": 7104.845986292232}, {"decode_latency_s": 0.0604689283791231, "kv_live_bytes": 462274560, "latency_s": 1.9358604582957923, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77479936, "prompt_kind": "short", "t_relative_s": 7109.856033083051}, {"decode_latency_s": 0.06834389193682, "kv_live_bytes": 462274560, "latency_s": 2.189312625210732, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77529088, "prompt_kind": "long", "t_relative_s": 7114.866124667227}, {"decode_latency_s": 0.06066801300039515, "kv_live_bytes": 462274560, "latency_s": 1.9422239172272384, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78069760, "prompt_kind": "short", "t_relative_s": 7119.876245541964}, {"decode_latency_s": 0.06779302474751603, "kv_live_bytes": 462274560, "latency_s": 2.1719817076809704, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78069760, "prompt_kind": "long", "t_relative_s": 7124.8849015422165}, {"decode_latency_s": 0.060217217440367676, "kv_live_bytes": 462274560, "latency_s": 1.9278416251763701, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78020608, "prompt_kind": "short", "t_relative_s": 7129.894979167264}, {"decode_latency_s": 0.06073010675027035, "kv_live_bytes": 462274560, "latency_s": 1.9449086668901145, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78118912, "prompt_kind": "long", "t_relative_s": 7134.905030833092}, {"decode_latency_s": 0.06042916275328025, "kv_live_bytes": 462274560, "latency_s": 1.9344858331605792, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78233600, "prompt_kind": "short", "t_relative_s": 7139.915117667057}, {"decode_latency_s": 0.06819904559233692, "kv_live_bytes": 462274560, "latency_s": 2.1842479580082, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77545472, "prompt_kind": "long", "t_relative_s": 7144.9252056670375}, {"decode_latency_s": 0.060347897160681896, "kv_live_bytes": 462274560, "latency_s": 1.9320002500899136, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77545472, "prompt_kind": "short", "t_relative_s": 7149.9352502920665}, {"decode_latency_s": 0.06833205859584268, "kv_live_bytes": 462274560, "latency_s": 2.1885101245716214, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77611008, "prompt_kind": "long", "t_relative_s": 7154.9453179999255}, {"decode_latency_s": 0.06668451562291011, "kv_live_bytes": 462274560, "latency_s": 2.1356006669811904, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78151680, "prompt_kind": "short", "t_relative_s": 7159.9554134579375}, {"decode_latency_s": 0.06796012369159143, "kv_live_bytes": 462274560, "latency_s": 2.1775870001874864, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78282752, "prompt_kind": "long", "t_relative_s": 7164.965476792306}, {"decode_latency_s": 0.060344161465764046, "kv_live_bytes": 462274560, "latency_s": 1.9319207081571221, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78348288, "prompt_kind": "short", "t_relative_s": 7169.975562416948}, {"decode_latency_s": 0.06809596093080472, "kv_live_bytes": 462274560, "latency_s": 2.1809667921625078, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76677120, "prompt_kind": "long", "t_relative_s": 7174.985653417185}, {"decode_latency_s": 0.06657560546591412, "kv_live_bytes": 462274560, "latency_s": 2.1319570830091834, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76709888, "prompt_kind": "short", "t_relative_s": 7179.995706958231}, {"decode_latency_s": 0.06459774999530055, "kv_live_bytes": 462274560, "latency_s": 2.0685719158500433, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75972608, "prompt_kind": "long", "t_relative_s": 7185.005796167068}, {"decode_latency_s": 0.06667969921545591, "kv_live_bytes": 462274560, "latency_s": 2.135548332706094, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76120064, "prompt_kind": "short", "t_relative_s": 7190.015862208325}, {"decode_latency_s": 0.06803118489915505, "kv_live_bytes": 462274560, "latency_s": 2.1794971246272326, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75890688, "prompt_kind": "long", "t_relative_s": 7195.025919750333}, {"decode_latency_s": 0.06049961849930696, "kv_live_bytes": 462274560, "latency_s": 1.936876333784312, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75415552, "prompt_kind": "short", "t_relative_s": 7200.035992917139}, {"decode_latency_s": 0.06881481381424237, "kv_live_bytes": 462274560, "latency_s": 2.204003875143826, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75710464, "prompt_kind": "long", "t_relative_s": 7205.04607662512}, {"decode_latency_s": 0.061233985688886605, "kv_live_bytes": 462274560, "latency_s": 1.9603989161550999, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75382784, "prompt_kind": "short", "t_relative_s": 7210.056131000165}, {"decode_latency_s": 0.06856543486355804, "kv_live_bytes": 462274560, "latency_s": 2.195921041071415, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75038720, "prompt_kind": "long", "t_relative_s": 7215.06621375028}, {"decode_latency_s": 0.06059744791127741, "kv_live_bytes": 462274560, "latency_s": 1.9402917078696191, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76382208, "prompt_kind": "short", "t_relative_s": 7220.071125958115}, {"decode_latency_s": 0.06097530078841373, "kv_live_bytes": 462274560, "latency_s": 1.95262208301574, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76496896, "prompt_kind": "long", "t_relative_s": 7225.081176124979}, {"decode_latency_s": 0.06033020179893356, "kv_live_bytes": 462274560, "latency_s": 1.9312975001521409, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76693504, "prompt_kind": "short", "t_relative_s": 7230.091256957967}, {"decode_latency_s": 0.060568524757400155, "kv_live_bytes": 462274560, "latency_s": 1.9390561250038445, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76021760, "prompt_kind": "long", "t_relative_s": 7235.101328582969}, {"decode_latency_s": 0.06045986847311724, "kv_live_bytes": 462274560, "latency_s": 1.9356704577803612, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "short", "t_relative_s": 7240.111398125067}, {"decode_latency_s": 0.06815410546550993, "kv_live_bytes": 462274560, "latency_s": 2.1832765829749405, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75169792, "prompt_kind": "long", "t_relative_s": 7245.121443333104}, {"decode_latency_s": 0.06037955210194923, "kv_live_bytes": 462274560, "latency_s": 1.9330731253139675, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75530240, "prompt_kind": "short", "t_relative_s": 7250.131534541957}, {"decode_latency_s": 0.06405449999147095, "kv_live_bytes": 462274560, "latency_s": 2.050872541964054, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70090752, "prompt_kind": "long", "t_relative_s": 7255.141787083354}, {"decode_latency_s": 0.06038871612690855, "kv_live_bytes": 462274560, "latency_s": 1.9334346670657396, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70336512, "prompt_kind": "short", "t_relative_s": 7261.5923203751445}, {"decode_latency_s": 0.06800741665938403, "kv_live_bytes": 462274560, "latency_s": 2.177788292057812, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70664192, "prompt_kind": "long", "t_relative_s": 7266.602387832943}, {"decode_latency_s": 0.06057979818433523, "kv_live_bytes": 462274560, "latency_s": 1.9393331673927605, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71254016, "prompt_kind": "short", "t_relative_s": 7271.612444542348}, {"decode_latency_s": 0.0680844140733825, "kv_live_bytes": 462274560, "latency_s": 2.1815390000119805, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71172096, "prompt_kind": "long", "t_relative_s": 7276.622523583006}, {"decode_latency_s": 0.06065312889404595, "kv_live_bytes": 462274560, "latency_s": 1.9416884589008987, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71745536, "prompt_kind": "short", "t_relative_s": 7281.632620250341}, {"decode_latency_s": 0.06817869530641474, "kv_live_bytes": 462274560, "latency_s": 2.183564541861415, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71827456, "prompt_kind": "long", "t_relative_s": 7286.642662125174}, {"decode_latency_s": 0.060331419241265394, "kv_live_bytes": 462274560, "latency_s": 1.9315054579637945, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71385088, "prompt_kind": "short", "t_relative_s": 7291.652714082971}, {"decode_latency_s": 0.06786925780761521, "kv_live_bytes": 462274560, "latency_s": 2.1737910001538694, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71516160, "prompt_kind": "long", "t_relative_s": 7296.662800667342}, {"decode_latency_s": 0.06067643100686837, "kv_live_bytes": 462274560, "latency_s": 1.9425064590759575, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72105984, "prompt_kind": "short", "t_relative_s": 7301.672875457909}, {"decode_latency_s": 0.06920446225558408, "kv_live_bytes": 462274560, "latency_s": 2.2169441250152886, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71909376, "prompt_kind": "long", "t_relative_s": 7306.682934999932}, {"decode_latency_s": 0.06663027212198358, "kv_live_bytes": 462274560, "latency_s": 2.133940333034843, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72499200, "prompt_kind": "short", "t_relative_s": 7311.692994917277}, {"decode_latency_s": 0.06818807160016149, "kv_live_bytes": 462274560, "latency_s": 2.1843085000291467, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72531968, "prompt_kind": "long", "t_relative_s": 7316.70305208303}, {"decode_latency_s": 0.0605999244726263, "kv_live_bytes": 462274560, "latency_s": 1.9400837919674814, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71942144, "prompt_kind": "short", "t_relative_s": 7321.7131439582445}, {"decode_latency_s": 0.06836798306903802, "kv_live_bytes": 462274560, "latency_s": 2.189271583221853, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72056832, "prompt_kind": "long", "t_relative_s": 7326.723225000314}, {"decode_latency_s": 0.060364704419043846, "kv_live_bytes": 462274560, "latency_s": 1.9324877923354506, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "short", "t_relative_s": 7331.733267208096}, {"decode_latency_s": 0.060567696593352593, "kv_live_bytes": 462274560, "latency_s": 1.9394711670465767, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72597504, "prompt_kind": "long", "t_relative_s": 7336.743318708148}, {"decode_latency_s": 0.06080362890497781, "kv_live_bytes": 462274560, "latency_s": 1.9465409996919334, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73187328, "prompt_kind": "short", "t_relative_s": 7341.753396542277}, {"decode_latency_s": 0.0677484583138721, "kv_live_bytes": 462274560, "latency_s": 2.1703937500715256, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73269248, "prompt_kind": "long", "t_relative_s": 7346.763445042074}, {"decode_latency_s": 0.060320829434203915, "kv_live_bytes": 462274560, "latency_s": 1.931175250094384, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73269248, "prompt_kind": "short", "t_relative_s": 7351.77350633312}, {"decode_latency_s": 0.06798711587907746, "kv_live_bytes": 462274560, "latency_s": 2.1772938752546906, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73334784, "prompt_kind": "long", "t_relative_s": 7356.783576000016}, {"decode_latency_s": 0.060703451847075485, "kv_live_bytes": 462274560, "latency_s": 1.9433642500080168, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73908224, "prompt_kind": "short", "t_relative_s": 7361.793634083122}, {"decode_latency_s": 0.06777945571229793, "kv_live_bytes": 462274560, "latency_s": 2.1712264167144895, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73940992, "prompt_kind": "long", "t_relative_s": 7366.803691625129}, {"decode_latency_s": 0.06058150000171736, "kv_live_bytes": 462274560, "latency_s": 1.939401624724269, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73973760, "prompt_kind": "short", "t_relative_s": 7371.8137663332745}, {"decode_latency_s": 0.06791937108209822, "kv_live_bytes": 462274560, "latency_s": 2.175516915973276, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73990144, "prompt_kind": "long", "t_relative_s": 7376.823821458034}, {"decode_latency_s": 0.06039214583870489, "kv_live_bytes": 462274560, "latency_s": 1.9334094170480967, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74022912, "prompt_kind": "short", "t_relative_s": 7381.833898042329}, {"decode_latency_s": 0.060280412741121836, "kv_live_bytes": 462274560, "latency_s": 1.929837082978338, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74072064, "prompt_kind": "long", "t_relative_s": 7386.843973125331}, {"decode_latency_s": 0.060482282569864765, "kv_live_bytes": 462274560, "latency_s": 1.9362763329409063, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74661888, "prompt_kind": "short", "t_relative_s": 7391.854016125202}, {"decode_latency_s": 0.06823273569170851, "kv_live_bytes": 462274560, "latency_s": 2.186238208319992, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74072064, "prompt_kind": "long", "t_relative_s": 7396.864078583196}, {"decode_latency_s": 0.06053659116150811, "kv_live_bytes": 462274560, "latency_s": 1.9380071656778455, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74645504, "prompt_kind": "short", "t_relative_s": 7401.874178833328}, {"decode_latency_s": 0.06782558462873567, "kv_live_bytes": 462274560, "latency_s": 2.172533041331917, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74711040, "prompt_kind": "long", "t_relative_s": 7406.884228667244}, {"decode_latency_s": 0.060414208346628584, "kv_live_bytes": 462274560, "latency_s": 1.934136999770999, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74825728, "prompt_kind": "short", "t_relative_s": 7411.894281167071}, {"decode_latency_s": 0.06830718491983134, "kv_live_bytes": 462274560, "latency_s": 2.1879371250979602, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74891264, "prompt_kind": "long", "t_relative_s": 7416.904368500225}, {"decode_latency_s": 0.06049879555939697, "kv_live_bytes": 462274560, "latency_s": 1.9367778752930462, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74907648, "prompt_kind": "short", "t_relative_s": 7421.914435875136}, {"decode_latency_s": 0.06068203515314963, "kv_live_bytes": 462274560, "latency_s": 1.94279395788908, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74924032, "prompt_kind": "long", "t_relative_s": 7426.924492083024}, {"decode_latency_s": 0.06690829688159283, "kv_live_bytes": 462274560, "latency_s": 2.1427332921884954, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74956800, "prompt_kind": "short", "t_relative_s": 7431.934571458027}, {"decode_latency_s": 0.06035788022563793, "kv_live_bytes": 462274560, "latency_s": 1.9325604159384966, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74956800, "prompt_kind": "long", "t_relative_s": 7436.944616000168}, {"decode_latency_s": 0.060384578129742295, "kv_live_bytes": 462274560, "latency_s": 1.933192207943648, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74809344, "prompt_kind": "short", "t_relative_s": 7441.9546777503565}, {"decode_latency_s": 0.06824153257184662, "kv_live_bytes": 462274560, "latency_s": 2.186123457737267, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74924032, "prompt_kind": "long", "t_relative_s": 7446.964758499991}, {"decode_latency_s": 0.06652402994222939, "kv_live_bytes": 462274560, "latency_s": 2.1304632909595966, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75481088, "prompt_kind": "short", "t_relative_s": 7451.974815333262}, {"decode_latency_s": 0.06768269008898642, "kv_live_bytes": 462274560, "latency_s": 2.168578375130892, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75513856, "prompt_kind": "long", "t_relative_s": 7456.98488041712}, {"decode_latency_s": 0.06052105731214397, "kv_live_bytes": 462274560, "latency_s": 1.9374624998308718, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75513856, "prompt_kind": "short", "t_relative_s": 7461.994970917236}, {"decode_latency_s": 0.06815102342807222, "kv_live_bytes": 462274560, "latency_s": 2.1828500418923795, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75399168, "prompt_kind": "long", "t_relative_s": 7467.005017917138}, {"decode_latency_s": 0.060446233095717616, "kv_live_bytes": 462274560, "latency_s": 1.9351708749309182, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75628544, "prompt_kind": "short", "t_relative_s": 7472.015065208077}, {"decode_latency_s": 0.06796266142919194, "kv_live_bytes": 462274560, "latency_s": 2.1768078338354826, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75694080, "prompt_kind": "long", "t_relative_s": 7477.025146208238}, {"decode_latency_s": 0.06088845184422098, "kv_live_bytes": 462274560, "latency_s": 1.9491943749599159, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76234752, "prompt_kind": "short", "t_relative_s": 7482.035235667136}, {"decode_latency_s": 0.06314969400409609, "kv_live_bytes": 462274560, "latency_s": 2.0219997498206794, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71401472, "prompt_kind": "long", "t_relative_s": 7487.035287207924}, {"decode_latency_s": 0.06032462891016621, "kv_live_bytes": 462274560, "latency_s": 1.9312961250543594, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71909376, "prompt_kind": "short", "t_relative_s": 7492.905891292263}, {"decode_latency_s": 0.06749759503873065, "kv_live_bytes": 462274560, "latency_s": 2.1621838328428566, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71991296, "prompt_kind": "long", "t_relative_s": 7497.915963458363}, {"decode_latency_s": 0.060747489595087245, "kv_live_bytes": 462274560, "latency_s": 1.9447706658393145, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72630272, "prompt_kind": "short", "t_relative_s": 7502.926042875275}, {"decode_latency_s": 0.06858877862396184, "kv_live_bytes": 462274560, "latency_s": 2.1972520002163947, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71368704, "prompt_kind": "long", "t_relative_s": 7507.9360987083055}, {"decode_latency_s": 0.06042635288031306, "kv_live_bytes": 462274560, "latency_s": 1.9345455411821604, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71090176, "prompt_kind": "short", "t_relative_s": 7512.946175666992}, {"decode_latency_s": 0.06835198700719047, "kv_live_bytes": 462274560, "latency_s": 2.1894545000977814, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71548928, "prompt_kind": "long", "t_relative_s": 7517.956252083182}, {"decode_latency_s": 0.06060058203001972, "kv_live_bytes": 462274560, "latency_s": 1.9401006251573563, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72089600, "prompt_kind": "short", "t_relative_s": 7522.966296874918}, {"decode_latency_s": 0.06805089712725021, "kv_live_bytes": 462274560, "latency_s": 2.180019082967192, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72155136, "prompt_kind": "long", "t_relative_s": 7527.976386208087}, {"decode_latency_s": 0.06699903385015205, "kv_live_bytes": 462274560, "latency_s": 2.145375957712531, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72253440, "prompt_kind": "short", "t_relative_s": 7532.986459041946}, {"decode_latency_s": 0.06833687370817643, "kv_live_bytes": 462274560, "latency_s": 2.1891484172083437, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72253440, "prompt_kind": "long", "t_relative_s": 7537.996516332962}, {"decode_latency_s": 0.060812951807747595, "kv_live_bytes": 462274560, "latency_s": 1.9469340411014855, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71991296, "prompt_kind": "short", "t_relative_s": 7542.999826667365}, {"decode_latency_s": 0.06793122657109052, "kv_live_bytes": 462274560, "latency_s": 2.1758836661465466, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71876608, "prompt_kind": "long", "t_relative_s": 7548.009909583256}, {"decode_latency_s": 0.0607969466218492, "kv_live_bytes": 462274560, "latency_s": 1.946375290863216, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72433664, "prompt_kind": "short", "t_relative_s": 7553.019959625322}, {"decode_latency_s": 0.06748826822149567, "kv_live_bytes": 462274560, "latency_s": 2.1619398752227426, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72499200, "prompt_kind": "long", "t_relative_s": 7558.029271374922}, {"decode_latency_s": 0.06109037759597413, "kv_live_bytes": 462274560, "latency_s": 1.9556813752278686, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72581120, "prompt_kind": "short", "t_relative_s": 7563.039374208078}, {"decode_latency_s": 0.06793339975411072, "kv_live_bytes": 462274560, "latency_s": 2.176313958130777, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72548352, "prompt_kind": "long", "t_relative_s": 7568.049426542129}, {"decode_latency_s": 0.060269205714575946, "kv_live_bytes": 462274560, "latency_s": 1.9295261250808835, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73138176, "prompt_kind": "short", "t_relative_s": 7573.059504833072}, {"decode_latency_s": 0.06729600390826818, "kv_live_bytes": 462274560, "latency_s": 2.1550620002672076, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72581120, "prompt_kind": "long", "t_relative_s": 7578.069583167322}, {"decode_latency_s": 0.06041502735752147, "kv_live_bytes": 462274560, "latency_s": 1.9342403332702816, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73154560, "prompt_kind": "short", "t_relative_s": 7583.079638000112}, {"decode_latency_s": 0.06800155078235548, "kv_live_bytes": 462274560, "latency_s": 2.1781423329375684, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73285632, "prompt_kind": "long", "t_relative_s": 7588.089749875013}, {"decode_latency_s": 0.06063200130301993, "kv_live_bytes": 462274560, "latency_s": 1.9410469997674227, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73367552, "prompt_kind": "short", "t_relative_s": 7593.099864624906}, {"decode_latency_s": 0.06785003647382837, "kv_live_bytes": 462274560, "latency_s": 2.1735819168388844, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "long", "t_relative_s": 7598.109926542267}, {"decode_latency_s": 0.06074229556543287, "kv_live_bytes": 462274560, "latency_s": 1.944717874750495, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73990144, "prompt_kind": "short", "t_relative_s": 7603.120008542202}, {"decode_latency_s": 0.06819481639831793, "kv_live_bytes": 462274560, "latency_s": 2.1837957077659667, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73957376, "prompt_kind": "long", "t_relative_s": 7608.130096042063}, {"decode_latency_s": 0.06717539712553844, "kv_live_bytes": 462274560, "latency_s": 2.1513825422152877, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73957376, "prompt_kind": "short", "t_relative_s": 7613.140145167243}, {"decode_latency_s": 0.067998328115209, "kv_live_bytes": 462274560, "latency_s": 2.1786036249250174, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74055680, "prompt_kind": "long", "t_relative_s": 7618.150275708176}, {"decode_latency_s": 0.06056154947145842, "kv_live_bytes": 462274560, "latency_s": 1.9387181662023067, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74137600, "prompt_kind": "short", "t_relative_s": 7623.160357542336}, {"decode_latency_s": 0.06800824349920731, "kv_live_bytes": 462274560, "latency_s": 2.178863083012402, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74039296, "prompt_kind": "long", "t_relative_s": 7628.170404375065}, {"decode_latency_s": 0.06029634765582159, "kv_live_bytes": 462274560, "latency_s": 1.9303993340581656, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73596928, "prompt_kind": "short", "t_relative_s": 7633.18049970828}, {"decode_latency_s": 0.06765080337936524, "kv_live_bytes": 462274560, "latency_s": 2.166704291012138, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73547776, "prompt_kind": "long", "t_relative_s": 7638.190571417101}, {"decode_latency_s": 0.06658908724784851, "kv_live_bytes": 462274560, "latency_s": 2.13264033338055, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74121216, "prompt_kind": "short", "t_relative_s": 7643.200626958162}, {"decode_latency_s": 0.06790290234494023, "kv_live_bytes": 462274560, "latency_s": 2.176214958075434, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74203136, "prompt_kind": "long", "t_relative_s": 7648.210718958173}, {"decode_latency_s": 0.06050004165444989, "kv_live_bytes": 462274560, "latency_s": 1.936825416982174, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74268672, "prompt_kind": "short", "t_relative_s": 7653.220832750201}, {"decode_latency_s": 0.0678200221300358, "kv_live_bytes": 462274560, "latency_s": 2.172898749820888, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74416128, "prompt_kind": "long", "t_relative_s": 7658.230877708178}, {"decode_latency_s": 0.06074672655086033, "kv_live_bytes": 462274560, "latency_s": 1.944766249973327, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74399744, "prompt_kind": "short", "t_relative_s": 7663.240951917134}, {"decode_latency_s": 0.06798610936675686, "kv_live_bytes": 462274560, "latency_s": 2.1776407919824123, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73875456, "prompt_kind": "long", "t_relative_s": 7668.2510328749195}, {"decode_latency_s": 0.060470147160231136, "kv_live_bytes": 462274560, "latency_s": 1.9359576245769858, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74383360, "prompt_kind": "short", "t_relative_s": 7673.26108945813}, {"decode_latency_s": 0.06799275781668257, "kv_live_bytes": 462274560, "latency_s": 2.1778074582107365, "n_emitted": 32, "ok": true, "process_footprint_bytes": 81903616, "prompt_kind": "long", "t_relative_s": 7678.271176208276}, {"decode_latency_s": 0.060512173164170235, "kv_live_bytes": 462274560, "latency_s": 1.9371324582025409, "n_emitted": 32, "ok": true, "process_footprint_bytes": 82001920, "prompt_kind": "short", "t_relative_s": 7683.2812906671315}, {"decode_latency_s": 0.06052619403635617, "kv_live_bytes": 462274560, "latency_s": 1.937973499763757, "n_emitted": 32, "ok": true, "process_footprint_bytes": 82018304, "prompt_kind": "long", "t_relative_s": 7688.291341250297}, {"decode_latency_s": 0.06041463931614999, "kv_live_bytes": 462274560, "latency_s": 1.934253249783069, "n_emitted": 32, "ok": true, "process_footprint_bytes": 82345984, "prompt_kind": "short", "t_relative_s": 7693.301423999947}, {"decode_latency_s": 0.0680665625113761, "kv_live_bytes": 462274560, "latency_s": 2.179544833023101, "n_emitted": 32, "ok": true, "process_footprint_bytes": 80740352, "prompt_kind": "long", "t_relative_s": 7698.311509625055}, {"decode_latency_s": 0.060618592440732755, "kv_live_bytes": 462274560, "latency_s": 1.9406378329731524, "n_emitted": 32, "ok": true, "process_footprint_bytes": 81281024, "prompt_kind": "short", "t_relative_s": 7703.321575125214}, {"decode_latency_s": 0.06767777343338821, "kv_live_bytes": 462274560, "latency_s": 2.16796191688627, "n_emitted": 32, "ok": true, "process_footprint_bytes": 81379328, "prompt_kind": "long", "t_relative_s": 7708.3316618329845}, {"decode_latency_s": 0.06053177475405391, "kv_live_bytes": 462274560, "latency_s": 1.937835458200425, "n_emitted": 32, "ok": true, "process_footprint_bytes": 81379328, "prompt_kind": "short", "t_relative_s": 7713.333047500346}, {"decode_latency_s": 0.06833736460248474, "kv_live_bytes": 462274560, "latency_s": 2.188853583764285, "n_emitted": 32, "ok": true, "process_footprint_bytes": 81412096, "prompt_kind": "long", "t_relative_s": 7718.343099167105}, {"decode_latency_s": 0.06043743621557951, "kv_live_bytes": 462274560, "latency_s": 1.9349419581703842, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79249408, "prompt_kind": "short", "t_relative_s": 7723.35315779224}, {"decode_latency_s": 0.06799870965187438, "kv_live_bytes": 462274560, "latency_s": 2.178669084329158, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79282176, "prompt_kind": "long", "t_relative_s": 7728.36323300004}, {"decode_latency_s": 0.06047173179103993, "kv_live_bytes": 462274560, "latency_s": 1.9358994169160724, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79364096, "prompt_kind": "short", "t_relative_s": 7733.373281457927}, {"decode_latency_s": 0.0679380286601372, "kv_live_bytes": 462274560, "latency_s": 2.1766857081092894, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79396864, "prompt_kind": "long", "t_relative_s": 7738.383346083108}, {"decode_latency_s": 0.06049421353964135, "kv_live_bytes": 462274560, "latency_s": 1.9365792092867196, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79446016, "prompt_kind": "short", "t_relative_s": 7743.393428250216}, {"decode_latency_s": 0.06046458984201308, "kv_live_bytes": 462274560, "latency_s": 1.9361309171654284, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79446016, "prompt_kind": "long", "t_relative_s": 7748.403488458134}, {"decode_latency_s": 0.06023388540779706, "kv_live_bytes": 462274560, "latency_s": 1.928411084227264, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79904768, "prompt_kind": "short", "t_relative_s": 7753.4135560831055}, {"decode_latency_s": 0.06026119271700736, "kv_live_bytes": 462274560, "latency_s": 1.9292402910068631, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79446016, "prompt_kind": "long", "t_relative_s": 7758.4236435000785}, {"decode_latency_s": 0.06056269658438396, "kv_live_bytes": 462274560, "latency_s": 1.9388349582441151, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79986688, "prompt_kind": "short", "t_relative_s": 7763.433711207937}, {"decode_latency_s": 0.06808899086900055, "kv_live_bytes": 462274560, "latency_s": 2.181974791921675, "n_emitted": 32, "ok": true, "process_footprint_bytes": 80084992, "prompt_kind": "long", "t_relative_s": 7768.443798167165}, {"decode_latency_s": 0.060880261720740236, "kv_live_bytes": 462274560, "latency_s": 1.9489853330887854, "n_emitted": 32, "ok": true, "process_footprint_bytes": 80134144, "prompt_kind": "short", "t_relative_s": 7773.453885958064}, {"decode_latency_s": 0.06049989321036264, "kv_live_bytes": 462274560, "latency_s": 1.9369014999829233, "n_emitted": 32, "ok": true, "process_footprint_bytes": 80150528, "prompt_kind": "long", "t_relative_s": 7778.463934333064}, {"decode_latency_s": 0.061084128901711665, "kv_live_bytes": 462274560, "latency_s": 1.9555985420010984, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79249408, "prompt_kind": "short", "t_relative_s": 7783.473996125162}, {"decode_latency_s": 0.06429701171873603, "kv_live_bytes": 462274560, "latency_s": 2.0585994170978665, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74760192, "prompt_kind": "long", "t_relative_s": 7788.484068042133}, {"decode_latency_s": 0.060590294291614555, "kv_live_bytes": 462274560, "latency_s": 1.940071249846369, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75087872, "prompt_kind": "short", "t_relative_s": 7795.028333500028}, {"decode_latency_s": 0.06889944399881642, "kv_live_bytes": 462274560, "latency_s": 2.2072855001315475, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67960832, "prompt_kind": "long", "t_relative_s": 7800.038405207917}, {"decode_latency_s": 0.06025044531270396, "kv_live_bytes": 462274560, "latency_s": 1.9289866252802312, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67518464, "prompt_kind": "short", "t_relative_s": 7805.048451917246}, {"decode_latency_s": 0.06707122657098807, "kv_live_bytes": 462274560, "latency_s": 2.14834566693753, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70647808, "prompt_kind": "long", "t_relative_s": 7810.058532875031}, {"decode_latency_s": 0.06073130600270815, "kv_live_bytes": 462274560, "latency_s": 1.9441755833104253, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71122944, "prompt_kind": "short", "t_relative_s": 7815.06860166695}, {"decode_latency_s": 0.060432399724959396, "kv_live_bytes": 462274560, "latency_s": 1.935428540688008, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71204864, "prompt_kind": "long", "t_relative_s": 7820.075868792366}, {"decode_latency_s": 0.06049545049609151, "kv_live_bytes": 462274560, "latency_s": 1.9366557919420302, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71745536, "prompt_kind": "short", "t_relative_s": 7825.085953874979}, {"decode_latency_s": 0.06863233853073325, "kv_live_bytes": 462274560, "latency_s": 2.198310417123139, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71843840, "prompt_kind": "long", "t_relative_s": 7830.09600995807}, {"decode_latency_s": 0.06044345312693622, "kv_live_bytes": 462274560, "latency_s": 1.9351307908073068, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71892992, "prompt_kind": "short", "t_relative_s": 7835.1060968749225}, {"decode_latency_s": 0.06797323437058367, "kv_live_bytes": 462274560, "latency_s": 2.1767185418866575, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71909376, "prompt_kind": "long", "t_relative_s": 7840.116187292151}, {"decode_latency_s": 0.060581205718335696, "kv_live_bytes": 462274560, "latency_s": 1.9394103749655187, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71876608, "prompt_kind": "short", "t_relative_s": 7845.119569832925}, {"decode_latency_s": 0.06567297395667993, "kv_live_bytes": 462274560, "latency_s": 2.1030265828594565, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72024064, "prompt_kind": "long", "t_relative_s": 7850.129628832918}, {"decode_latency_s": 0.06118410546332598, "kv_live_bytes": 462274560, "latency_s": 1.9586349581368268, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72482816, "prompt_kind": "short", "t_relative_s": 7855.139703917317}, {"decode_latency_s": 0.06094087372184731, "kv_live_bytes": 462274560, "latency_s": 1.951093875337392, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72531968, "prompt_kind": "long", "t_relative_s": 7860.149739332963}, {"decode_latency_s": 0.06037838544580154, "kv_live_bytes": 462274560, "latency_s": 1.9330372917465866, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72155136, "prompt_kind": "short", "t_relative_s": 7865.15984262526}, {"decode_latency_s": 0.06827385028009303, "kv_live_bytes": 462274560, "latency_s": 2.186699707992375, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71974912, "prompt_kind": "long", "t_relative_s": 7870.1699400423095}, {"decode_latency_s": 0.060395355481887236, "kv_live_bytes": 462274560, "latency_s": 1.933584333397448, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71925760, "prompt_kind": "short", "t_relative_s": 7875.179996833205}, {"decode_latency_s": 0.067791740904795, "kv_live_bytes": 462274560, "latency_s": 2.1713739996775985, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72695808, "prompt_kind": "long", "t_relative_s": 7880.190052208025}, {"decode_latency_s": 0.06049935286864638, "kv_live_bytes": 462274560, "latency_s": 1.9368295408785343, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71876608, "prompt_kind": "short", "t_relative_s": 7885.200136333238}, {"decode_latency_s": 0.06805674089991953, "kv_live_bytes": 462274560, "latency_s": 2.1796172498725355, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72220672, "prompt_kind": "long", "t_relative_s": 7890.210221624933}, {"decode_latency_s": 0.06047913931251969, "kv_live_bytes": 462274560, "latency_s": 1.9362139999866486, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72417280, "prompt_kind": "short", "t_relative_s": 7895.220255000051}, {"decode_latency_s": 0.06791724868526217, "kv_live_bytes": 462274560, "latency_s": 2.175848958082497, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72482816, "prompt_kind": "long", "t_relative_s": 7900.230330542196}, {"decode_latency_s": 0.0606665638188133, "kv_live_bytes": 462274560, "latency_s": 1.9421230829320848, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72843264, "prompt_kind": "short", "t_relative_s": 7905.240431792103}, {"decode_latency_s": 0.13782751304097474, "kv_live_bytes": 462274560, "latency_s": 4.41132283443585, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69173248, "prompt_kind": "long", "t_relative_s": 7910.250487208366}, {"decode_latency_s": 0.06038067315239459, "kv_live_bytes": 462274560, "latency_s": 1.9330936251208186, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69271552, "prompt_kind": "short", "t_relative_s": 7919.277554542292}, {"decode_latency_s": 0.06845641929248814, "kv_live_bytes": 462274560, "latency_s": 2.192249875050038, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69877760, "prompt_kind": "long", "t_relative_s": 7924.287635750137}, {"decode_latency_s": 0.060412243474274874, "kv_live_bytes": 462274560, "latency_s": 1.93452825024724, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70647808, "prompt_kind": "short", "t_relative_s": 7929.297687042039}, {"decode_latency_s": 0.0680943880288396, "kv_live_bytes": 462274560, "latency_s": 2.1812799582257867, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70041600, "prompt_kind": "long", "t_relative_s": 7934.307768625207}, {"decode_latency_s": 0.060571166657609865, "kv_live_bytes": 462274560, "latency_s": 1.9390991660766304, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70598656, "prompt_kind": "short", "t_relative_s": 7939.317870375235}, {"decode_latency_s": 0.06787764062755741, "kv_live_bytes": 462274560, "latency_s": 2.1742301248013973, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70615040, "prompt_kind": "long", "t_relative_s": 7944.327917499933}, {"decode_latency_s": 0.06067382685432676, "kv_live_bytes": 462274560, "latency_s": 1.94249645806849, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71139328, "prompt_kind": "short", "t_relative_s": 7949.338001166936}, {"decode_latency_s": 0.06817901298927609, "kv_live_bytes": 462274560, "latency_s": 2.18364941701293, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71139328, "prompt_kind": "long", "t_relative_s": 7954.348075082991}, {"decode_latency_s": 0.06051036328426562, "kv_live_bytes": 462274560, "latency_s": 1.9372510421089828, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71172096, "prompt_kind": "short", "t_relative_s": 7959.358139375225}, {"decode_latency_s": 0.06795035937102512, "kv_live_bytes": 462274560, "latency_s": 2.177136749960482, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70615040, "prompt_kind": "long", "t_relative_s": 7964.368212375324}, {"decode_latency_s": 0.06054515756841283, "kv_live_bytes": 462274560, "latency_s": 1.9382467921823263, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71041024, "prompt_kind": "short", "t_relative_s": 7969.378282958176}, {"decode_latency_s": 0.06795243750093505, "kv_live_bytes": 462274560, "latency_s": 2.17648124974221, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71041024, "prompt_kind": "long", "t_relative_s": 7974.3883380419575}, {"decode_latency_s": 0.060358755217748694, "kv_live_bytes": 462274560, "latency_s": 1.9323791251517832, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71581696, "prompt_kind": "short", "t_relative_s": 7979.398409458343}, {"decode_latency_s": 0.06819762372469995, "kv_live_bytes": 462274560, "latency_s": 2.183954792097211, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71024640, "prompt_kind": "long", "t_relative_s": 7984.408494625241}, {"decode_latency_s": 0.06041945572360419, "kv_live_bytes": 462274560, "latency_s": 1.9345865421928465, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71106560, "prompt_kind": "short", "t_relative_s": 7989.418563708197}, {"decode_latency_s": 0.067802351564751, "kv_live_bytes": 462274560, "latency_s": 2.172213749960065, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71139328, "prompt_kind": "long", "t_relative_s": 7994.4286370002665}, {"decode_latency_s": 0.06650342709326651, "kv_live_bytes": 462274560, "latency_s": 2.1297885831445456, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71663616, "prompt_kind": "short", "t_relative_s": 7999.438718917314}, {"decode_latency_s": 0.06806469659204595, "kv_live_bytes": 462274560, "latency_s": 2.1801792080514133, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71548928, "prompt_kind": "long", "t_relative_s": 8004.448771500029}, {"decode_latency_s": 0.0602770755358506, "kv_live_bytes": 462274560, "latency_s": 1.9297712086699903, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72089600, "prompt_kind": "short", "t_relative_s": 8009.458868375048}, {"decode_latency_s": 0.06090348699945025, "kv_live_bytes": 462274560, "latency_s": 1.9501274591311812, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72073216, "prompt_kind": "long", "t_relative_s": 8014.468950625043}, {"decode_latency_s": 0.06656474999908824, "kv_live_bytes": 462274560, "latency_s": 2.1316438750363886, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72237056, "prompt_kind": "short", "t_relative_s": 8019.4790060832165}, {"decode_latency_s": 0.06799462500202935, "kv_live_bytes": 462274560, "latency_s": 2.1783810830675066, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72253440, "prompt_kind": "long", "t_relative_s": 8024.4891115003265}, {"decode_latency_s": 0.06681233465496916, "kv_live_bytes": 462274560, "latency_s": 2.1396749997511506, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72794112, "prompt_kind": "short", "t_relative_s": 8029.499191750307}, {"decode_latency_s": 0.06803398956253659, "kv_live_bytes": 462274560, "latency_s": 2.1793037080205977, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72810496, "prompt_kind": "long", "t_relative_s": 8034.509240750223}, {"decode_latency_s": 0.06021203125419561, "kv_live_bytes": 462274560, "latency_s": 1.927762750070542, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72679424, "prompt_kind": "short", "t_relative_s": 8039.51932995813}, {"decode_latency_s": 0.06818226953328121, "kv_live_bytes": 462274560, "latency_s": 2.184154541697353, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72728576, "prompt_kind": "long", "t_relative_s": 8044.529414417222}, {"decode_latency_s": 0.06101706769550219, "kv_live_bytes": 462274560, "latency_s": 1.953466999810189, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72220672, "prompt_kind": "short", "t_relative_s": 8049.539451166987}, {"decode_latency_s": 0.06817223175312392, "kv_live_bytes": 462274560, "latency_s": 2.183917374815792, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72171520, "prompt_kind": "long", "t_relative_s": 8054.549535833299}, {"decode_latency_s": 0.06073467187525239, "kv_live_bytes": 462274560, "latency_s": 1.9443256659433246, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72761344, "prompt_kind": "short", "t_relative_s": 8059.55962758325}, {"decode_latency_s": 0.06746205336821731, "kv_live_bytes": 462274560, "latency_s": 2.1614432078786194, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72777728, "prompt_kind": "long", "t_relative_s": 8064.5647905422375}, {"decode_latency_s": 0.06033393096004147, "kv_live_bytes": 462274560, "latency_s": 1.931588708423078, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73269248, "prompt_kind": "short", "t_relative_s": 8069.574871625286}, {"decode_latency_s": 0.06809265365882311, "kv_live_bytes": 462274560, "latency_s": 2.180839290842414, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73400320, "prompt_kind": "long", "t_relative_s": 8074.5849454579875}, {"decode_latency_s": 0.060937022120924667, "kv_live_bytes": 462274560, "latency_s": 1.9509436669759452, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "short", "t_relative_s": 8079.594994333107}, {"decode_latency_s": 0.09673426431254484, "kv_live_bytes": 462274560, "latency_s": 3.096962624695152, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69222400, "prompt_kind": "long", "t_relative_s": 8084.597600375302}, {"decode_latency_s": 0.06122827602666803, "kv_live_bytes": 462274560, "latency_s": 1.9604332502931356, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70057984, "prompt_kind": "short", "t_relative_s": 8093.624274708331}, {"decode_latency_s": 0.06823580077616498, "kv_live_bytes": 462274560, "latency_s": 2.1861744159832597, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70107136, "prompt_kind": "long", "t_relative_s": 8098.6343231252395}, {"decode_latency_s": 0.060360002622473985, "kv_live_bytes": 462274560, "latency_s": 1.9324573748745024, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70565888, "prompt_kind": "short", "t_relative_s": 8103.644407042302}, {"decode_latency_s": 0.06804121221648529, "kv_live_bytes": 462274560, "latency_s": 2.178840709384531, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70631424, "prompt_kind": "long", "t_relative_s": 8108.654479625169}, {"decode_latency_s": 0.06046320703171659, "kv_live_bytes": 462274560, "latency_s": 1.935659249778837, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70647808, "prompt_kind": "short", "t_relative_s": 8113.664538625162}, {"decode_latency_s": 0.06747037630702835, "kv_live_bytes": 462274560, "latency_s": 2.161805334035307, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70107136, "prompt_kind": "long", "t_relative_s": 8118.674604458269}, {"decode_latency_s": 0.06052086850104388, "kv_live_bytes": 462274560, "latency_s": 1.9374380423687398, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70696960, "prompt_kind": "short", "t_relative_s": 8123.6846809582785}, {"decode_latency_s": 0.06845882290508598, "kv_live_bytes": 462274560, "latency_s": 2.192663208115846, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70565888, "prompt_kind": "long", "t_relative_s": 8128.694736250211}, {"decode_latency_s": 0.06032472528750077, "kv_live_bytes": 462274560, "latency_s": 1.9313374580815434, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70483968, "prompt_kind": "short", "t_relative_s": 8133.704805542249}, {"decode_latency_s": 0.06816847655863967, "kv_live_bytes": 462274560, "latency_s": 2.183418875094503, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71057408, "prompt_kind": "long", "t_relative_s": 8138.714887917042}, {"decode_latency_s": 0.0606751653685933, "kv_live_bytes": 462274560, "latency_s": 1.9424505410715938, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70565888, "prompt_kind": "short", "t_relative_s": 8143.724253542256}, {"decode_latency_s": 0.13867930209380575, "kv_live_bytes": 462274560, "latency_s": 4.440033750142902, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68861952, "prompt_kind": "long", "t_relative_s": 8148.733001917135}, {"decode_latency_s": 0.060484865884063765, "kv_live_bytes": 462274560, "latency_s": 1.9365692920982838, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69468160, "prompt_kind": "short", "t_relative_s": 8157.910600916948}, {"decode_latency_s": 0.0680285156267928, "kv_live_bytes": 462274560, "latency_s": 2.178443167358637, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70057984, "prompt_kind": "long", "t_relative_s": 8162.920691625215}, {"decode_latency_s": 0.0607202812389005, "kv_live_bytes": 462274560, "latency_s": 1.9438787498511374, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70615040, "prompt_kind": "short", "t_relative_s": 8167.930750458036}, {"decode_latency_s": 0.060664726566756144, "kv_live_bytes": 462274560, "latency_s": 1.9425597498193383, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70582272, "prompt_kind": "long", "t_relative_s": 8172.940872125328}, {"decode_latency_s": 0.06091751043277327, "kv_live_bytes": 462274560, "latency_s": 1.9502572501078248, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70713344, "prompt_kind": "short", "t_relative_s": 8177.950466041919}, {"decode_latency_s": 0.06830218358663842, "kv_live_bytes": 462274560, "latency_s": 2.187994208186865, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70795264, "prompt_kind": "long", "t_relative_s": 8182.960516917054}, {"decode_latency_s": 0.06039163544483017, "kv_live_bytes": 462274560, "latency_s": 1.9334478746168315, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70893568, "prompt_kind": "short", "t_relative_s": 8187.970594292041}, {"decode_latency_s": 0.06834077996609267, "kv_live_bytes": 462274560, "latency_s": 2.1886161249130964, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70975488, "prompt_kind": "long", "t_relative_s": 8192.980686792172}, {"decode_latency_s": 0.060535699216416106, "kv_live_bytes": 462274560, "latency_s": 1.9384385831654072, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71548928, "prompt_kind": "short", "t_relative_s": 8197.990743208211}, {"decode_latency_s": 0.06795954296831042, "kv_live_bytes": 462274560, "latency_s": 2.177028249949217, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71598080, "prompt_kind": "long", "t_relative_s": 8203.000833833124}, {"decode_latency_s": 0.06083050390589051, "kv_live_bytes": 462274560, "latency_s": 1.9473730837926269, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71663616, "prompt_kind": "short", "t_relative_s": 8208.01091687521}, {"decode_latency_s": 0.06789616537571419, "kv_live_bytes": 462274560, "latency_s": 2.1747124162502587, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71663616, "prompt_kind": "long", "t_relative_s": 8213.02096850006}, {"decode_latency_s": 0.060354152345098555, "kv_live_bytes": 462274560, "latency_s": 1.9322488331235945, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72171520, "prompt_kind": "short", "t_relative_s": 8218.031059917063}, {"decode_latency_s": 0.06841352212359197, "kv_live_bytes": 462274560, "latency_s": 2.191247916780412, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71647232, "prompt_kind": "long", "t_relative_s": 8223.034062167164}, {"decode_latency_s": 0.060621220065513626, "kv_live_bytes": 462274560, "latency_s": 1.9407467497512698, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72187904, "prompt_kind": "short", "t_relative_s": 8228.044128458016}, {"decode_latency_s": 0.06836829165695235, "kv_live_bytes": 462274560, "latency_s": 2.1903685838915408, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72220672, "prompt_kind": "long", "t_relative_s": 8233.054218542296}, {"decode_latency_s": 0.060563901031855494, "kv_live_bytes": 462274560, "latency_s": 1.9393640421330929, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72269824, "prompt_kind": "short", "t_relative_s": 8238.064291458111}, {"decode_latency_s": 0.0678738515562145, "kv_live_bytes": 462274560, "latency_s": 2.1739826248958707, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72302592, "prompt_kind": "long", "t_relative_s": 8243.074342958163}, {"decode_latency_s": 0.06047992449020967, "kv_live_bytes": 462274560, "latency_s": 1.9363095830194652, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72089600, "prompt_kind": "short", "t_relative_s": 8248.084428875241}, {"decode_latency_s": 0.06769566927687265, "kv_live_bytes": 462274560, "latency_s": 2.1685368749313056, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72155136, "prompt_kind": "long", "t_relative_s": 8253.094512917101}, {"decode_latency_s": 0.06062965888122562, "kv_live_bytes": 462274560, "latency_s": 1.9410345419310033, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72712192, "prompt_kind": "short", "t_relative_s": 8258.104562417138}, {"decode_latency_s": 0.06768497005396057, "kv_live_bytes": 462274560, "latency_s": 2.168649291153997, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72810496, "prompt_kind": "long", "t_relative_s": 8263.114631250035}, {"decode_latency_s": 0.06328573178325314, "kv_live_bytes": 462274560, "latency_s": 2.026108083780855, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70107136, "prompt_kind": "short", "t_relative_s": 8268.124731500167}, {"decode_latency_s": 0.06765979036572389, "kv_live_bytes": 462274560, "latency_s": 2.16738195810467, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72024064, "prompt_kind": "long", "t_relative_s": 8273.13481812505}, {"decode_latency_s": 0.060715468745911494, "kv_live_bytes": 462274560, "latency_s": 1.9437432498671114, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72613888, "prompt_kind": "short", "t_relative_s": 8278.144875457976}, {"decode_latency_s": 0.06776206768699922, "kv_live_bytes": 462274560, "latency_s": 2.170744334347546, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "long", "t_relative_s": 8283.154923500028}, {"decode_latency_s": 0.06050607420911547, "kv_live_bytes": 462274560, "latency_s": 1.9379326249472797, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73220096, "prompt_kind": "short", "t_relative_s": 8288.164989542216}, {"decode_latency_s": 0.06804925650067162, "kv_live_bytes": 462274560, "latency_s": 2.1792804999276996, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73121792, "prompt_kind": "long", "t_relative_s": 8293.175065125339}, {"decode_latency_s": 0.06048960027692374, "kv_live_bytes": 462274560, "latency_s": 1.9365565003827214, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73023488, "prompt_kind": "short", "t_relative_s": 8298.185120542068}, {"decode_latency_s": 0.06753730731725227, "kv_live_bytes": 462274560, "latency_s": 2.1630190829746425, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73105408, "prompt_kind": "long", "t_relative_s": 8303.19520016713}, {"decode_latency_s": 0.06054709374438971, "kv_live_bytes": 462274560, "latency_s": 1.938312916085124, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73646080, "prompt_kind": "short", "t_relative_s": 8308.205280708149}, {"decode_latency_s": 0.060369542974513024, "kv_live_bytes": 462274560, "latency_s": 1.9327880409546196, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73678848, "prompt_kind": "long", "t_relative_s": 8313.215333207976}, {"decode_latency_s": 0.06053461068950128, "kv_live_bytes": 462274560, "latency_s": 1.9385634581558406, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73760768, "prompt_kind": "short", "t_relative_s": 8318.22539854236}, {"decode_latency_s": 0.06803999478870537, "kv_live_bytes": 462274560, "latency_s": 2.178991583175957, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73121792, "prompt_kind": "long", "t_relative_s": 8323.23548120819}, {"decode_latency_s": 0.06039005989441648, "kv_live_bytes": 462274560, "latency_s": 1.9333218336105347, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72531968, "prompt_kind": "short", "t_relative_s": 8328.245531542227}, {"decode_latency_s": 0.06789514713454992, "kv_live_bytes": 462274560, "latency_s": 2.174722250085324, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73203712, "prompt_kind": "long", "t_relative_s": 8333.255611458328}, {"decode_latency_s": 0.06413711978530046, "kv_live_bytes": 462274560, "latency_s": 2.0532800406217575, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72744960, "prompt_kind": "short", "t_relative_s": 8338.265666374937}, {"decode_latency_s": 0.06753101172216702, "kv_live_bytes": 462274560, "latency_s": 2.1631617075763643, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73072640, "prompt_kind": "long", "t_relative_s": 8343.275740833022}, {"decode_latency_s": 0.06037449087307323, "kv_live_bytes": 462274560, "latency_s": 1.9328967495821416, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73744384, "prompt_kind": "short", "t_relative_s": 8348.285790707916}, {"decode_latency_s": 0.0678766132768942, "kv_live_bytes": 462274560, "latency_s": 2.174216417130083, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73826304, "prompt_kind": "long", "t_relative_s": 8353.295872667339}, {"decode_latency_s": 0.06074533856008202, "kv_live_bytes": 462274560, "latency_s": 1.9446433749981225, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73170944, "prompt_kind": "short", "t_relative_s": 8358.305988583248}, {"decode_latency_s": 0.06800317317538429, "kv_live_bytes": 462274560, "latency_s": 2.1786476247943938, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73662464, "prompt_kind": "long", "t_relative_s": 8363.31604283303}, {"decode_latency_s": 0.0604612421739148, "kv_live_bytes": 462274560, "latency_s": 1.9356225421652198, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73744384, "prompt_kind": "short", "t_relative_s": 8368.326121792197}, {"decode_latency_s": 0.06780699477531016, "kv_live_bytes": 462274560, "latency_s": 2.17151654092595, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73678848, "prompt_kind": "long", "t_relative_s": 8373.336200000253}, {"decode_latency_s": 0.06049679817806464, "kv_live_bytes": 462274560, "latency_s": 1.9367919997312129, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74219520, "prompt_kind": "short", "t_relative_s": 8378.346251333132}, {"decode_latency_s": 0.06165771093219519, "kv_live_bytes": 462274560, "latency_s": 1.9740874161943793, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73105408, "prompt_kind": "long", "t_relative_s": 8383.356326542329}, {"decode_latency_s": 0.060822087223641574, "kv_live_bytes": 462274560, "latency_s": 1.947138000279665, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72974336, "prompt_kind": "short", "t_relative_s": 8388.360391250346}, {"decode_latency_s": 0.06806772135314532, "kv_live_bytes": 462274560, "latency_s": 2.1807074160315096, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73744384, "prompt_kind": "long", "t_relative_s": 8393.370443375316}, {"decode_latency_s": 0.06054451434465591, "kv_live_bytes": 462274560, "latency_s": 1.9382075420580804, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74301440, "prompt_kind": "short", "t_relative_s": 8398.37655858323}, {"decode_latency_s": 0.06826565884693991, "kv_live_bytes": 462274560, "latency_s": 2.186602917034179, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74235904, "prompt_kind": "long", "t_relative_s": 8403.386608958244}, {"decode_latency_s": 0.0602821250067791, "kv_live_bytes": 462274560, "latency_s": 1.9299305831082165, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74809344, "prompt_kind": "short", "t_relative_s": 8408.396659417078}, {"decode_latency_s": 0.06049435547902249, "kv_live_bytes": 462274560, "latency_s": 1.936731334310025, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74907648, "prompt_kind": "long", "t_relative_s": 8413.406746208202}, {"decode_latency_s": 0.06069669789576437, "kv_live_bytes": 462274560, "latency_s": 1.9430882916785777, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75022336, "prompt_kind": "short", "t_relative_s": 8418.416791874915}, {"decode_latency_s": 0.06778094659966882, "kv_live_bytes": 462274560, "latency_s": 2.1710820831358433, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75071488, "prompt_kind": "long", "t_relative_s": 8423.42683629226}, {"decode_latency_s": 0.06060481118038297, "kv_live_bytes": 462274560, "latency_s": 1.9410095838829875, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75595776, "prompt_kind": "short", "t_relative_s": 8428.436904042028}, {"decode_latency_s": 0.060327238286845386, "kv_live_bytes": 462274560, "latency_s": 1.9313815832138062, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75612160, "prompt_kind": "long", "t_relative_s": 8433.446952749975}, {"decode_latency_s": 0.06041054424713366, "kv_live_bytes": 462274560, "latency_s": 1.9340403750538826, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75612160, "prompt_kind": "short", "t_relative_s": 8438.45701083308}, {"decode_latency_s": 0.06869993881264236, "kv_live_bytes": 462274560, "latency_s": 2.200495250057429, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75677696, "prompt_kind": "long", "t_relative_s": 8443.467091292143}, {"decode_latency_s": 0.06082927083480172, "kv_live_bytes": 462274560, "latency_s": 1.9473651251755655, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75202560, "prompt_kind": "short", "t_relative_s": 8448.47715683328}, {"decode_latency_s": 0.06325447134440765, "kv_live_bytes": 462274560, "latency_s": 2.0255441670306027, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75169792, "prompt_kind": "long", "t_relative_s": 8453.487215583213}, {"decode_latency_s": 0.06669915234670043, "kv_live_bytes": 462274560, "latency_s": 2.1362388329580426, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75792384, "prompt_kind": "short", "t_relative_s": 8458.497303333133}, {"decode_latency_s": 0.06834405859990511, "kv_live_bytes": 462274560, "latency_s": 2.1888032909482718, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75595776, "prompt_kind": "long", "t_relative_s": 8463.507363833021}, {"decode_latency_s": 0.060347509119310416, "kv_live_bytes": 462274560, "latency_s": 1.931992833968252, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75677696, "prompt_kind": "short", "t_relative_s": 8468.51743341703}, {"decode_latency_s": 0.06816218487801962, "kv_live_bytes": 462274560, "latency_s": 2.1831147079356015, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76152832, "prompt_kind": "long", "t_relative_s": 8473.527507999912}, {"decode_latency_s": 0.06104916274489369, "kv_live_bytes": 462274560, "latency_s": 1.9544425839558244, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75644928, "prompt_kind": "short", "t_relative_s": 8478.537570333108}, {"decode_latency_s": 0.06826502474723384, "kv_live_bytes": 462274560, "latency_s": 2.1875905408523977, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75677696, "prompt_kind": "long", "t_relative_s": 8483.54764600005}, {"decode_latency_s": 0.060725225252099335, "kv_live_bytes": 462274560, "latency_s": 1.9440460000187159, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75710464, "prompt_kind": "short", "t_relative_s": 8488.557735417038}, {"decode_latency_s": 0.06722948310198262, "kv_live_bytes": 462274560, "latency_s": 2.153201666660607, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75563008, "prompt_kind": "long", "t_relative_s": 8493.567804791965}, {"decode_latency_s": 0.06043748959200457, "kv_live_bytes": 462274560, "latency_s": 1.9348694160580635, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75595776, "prompt_kind": "short", "t_relative_s": 8498.577876375057}, {"decode_latency_s": 0.06854333852243144, "kv_live_bytes": 462274560, "latency_s": 2.1953394170850515, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75530240, "prompt_kind": "long", "t_relative_s": 8503.587955917232}, {"decode_latency_s": 0.060505403656861745, "kv_live_bytes": 462274560, "latency_s": 1.9370564171113074, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75661312, "prompt_kind": "short", "t_relative_s": 8508.598002583254}, {"decode_latency_s": 0.06819016014924273, "kv_live_bytes": 462274560, "latency_s": 2.184612500015646, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75661312, "prompt_kind": "long", "t_relative_s": 8513.608093082905}, {"decode_latency_s": 0.06665089452872053, "kv_live_bytes": 462274560, "latency_s": 2.134750208351761, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75694080, "prompt_kind": "short", "t_relative_s": 8518.618168250192}, {"decode_latency_s": 0.1116988776193466, "kv_live_bytes": 462274560, "latency_s": 3.575291041750461, "n_emitted": 32, "ok": true, "process_footprint_bytes": 63668224, "prompt_kind": "long", "t_relative_s": 8523.622669707984}, {"decode_latency_s": 0.06080760678742081, "kv_live_bytes": 462274560, "latency_s": 1.9467477910220623, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66404352, "prompt_kind": "short", "t_relative_s": 8533.619501917157}, {"decode_latency_s": 0.0681567460851511, "kv_live_bytes": 462274560, "latency_s": 2.1827893746085465, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69795840, "prompt_kind": "long", "t_relative_s": 8538.629539625254}, {"decode_latency_s": 0.060327749990392476, "kv_live_bytes": 462274560, "latency_s": 1.9314213329926133, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69795840, "prompt_kind": "short", "t_relative_s": 8543.639589332975}, {"decode_latency_s": 0.060506355468533, "kv_live_bytes": 462274560, "latency_s": 1.9373628753237426, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69910528, "prompt_kind": "long", "t_relative_s": 8548.649662000127}, {"decode_latency_s": 0.06074310546682682, "kv_live_bytes": 462274560, "latency_s": 1.9446475417353213, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70615040, "prompt_kind": "short", "t_relative_s": 8553.659714375157}, {"decode_latency_s": 0.06782161846058443, "kv_live_bytes": 462274560, "latency_s": 2.1728416667319834, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70713344, "prompt_kind": "long", "t_relative_s": 8558.66977712512}, {"decode_latency_s": 0.06138978907256387, "kv_live_bytes": 462274560, "latency_s": 1.9652810832485557, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71303168, "prompt_kind": "short", "t_relative_s": 8563.679863000289}, {"decode_latency_s": 0.06790707549953368, "kv_live_bytes": 462274560, "latency_s": 2.175723375286907, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70680576, "prompt_kind": "long", "t_relative_s": 8568.689921125304}, {"decode_latency_s": 0.060393585939891636, "kv_live_bytes": 462274560, "latency_s": 1.933534708339721, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70582272, "prompt_kind": "short", "t_relative_s": 8573.699991708156}, {"decode_latency_s": 0.06034038022335153, "kv_live_bytes": 462274560, "latency_s": 1.9318391671404243, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70500352, "prompt_kind": "long", "t_relative_s": 8578.71006920794}, {"decode_latency_s": 0.06073480338091031, "kv_live_bytes": 462274560, "latency_s": 1.9444614578969777, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71090176, "prompt_kind": "short", "t_relative_s": 8583.720123542007}, {"decode_latency_s": 0.06784913540468551, "kv_live_bytes": 462274560, "latency_s": 2.173183666076511, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71057408, "prompt_kind": "long", "t_relative_s": 8588.730196875054}, {"decode_latency_s": 0.0606877760583302, "kv_live_bytes": 462274560, "latency_s": 1.9427752080373466, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71647232, "prompt_kind": "short", "t_relative_s": 8593.740277375095}, {"decode_latency_s": 0.06810818228404969, "kv_live_bytes": 462274560, "latency_s": 2.1819256246089935, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71581696, "prompt_kind": "long", "t_relative_s": 8598.750319499988}, {"decode_latency_s": 0.06048133593867533, "kv_live_bytes": 462274560, "latency_s": 1.9363515838049352, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71139328, "prompt_kind": "short", "t_relative_s": 8603.756394000258}, {"decode_latency_s": 0.06851033722341526, "kv_live_bytes": 462274560, "latency_s": 2.1943855830468237, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71155712, "prompt_kind": "long", "t_relative_s": 8608.766474792268}, {"decode_latency_s": 0.06064219401741866, "kv_live_bytes": 462274560, "latency_s": 1.9414806249551475, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71712768, "prompt_kind": "short", "t_relative_s": 8613.77652179217}, {"decode_latency_s": 0.06095402083883528, "kv_live_bytes": 462274560, "latency_s": 1.9515986670739949, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71155712, "prompt_kind": "long", "t_relative_s": 8618.786611082964}, {"decode_latency_s": 0.0606179361930117, "kv_live_bytes": 462274560, "latency_s": 1.9406247078441083, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70959104, "prompt_kind": "short", "t_relative_s": 8623.79665758321}, {"decode_latency_s": 0.06799052606220357, "kv_live_bytes": 462274560, "latency_s": 2.1775244157761335, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70418432, "prompt_kind": "long", "t_relative_s": 8628.80673733307}, {"decode_latency_s": 0.06696117318642791, "kv_live_bytes": 462274560, "latency_s": 2.1443134169094265, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71221248, "prompt_kind": "short", "t_relative_s": 8633.816777457949}, {"decode_latency_s": 0.06827120180241764, "kv_live_bytes": 462274560, "latency_s": 2.186854540836066, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71745536, "prompt_kind": "long", "t_relative_s": 8638.826860208064}, {"decode_latency_s": 0.06052709897630848, "kv_live_bytes": 462274560, "latency_s": 1.9376454171724617, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72335360, "prompt_kind": "short", "t_relative_s": 8643.836949625053}, {"decode_latency_s": 0.0680368945322698, "kv_live_bytes": 462274560, "latency_s": 2.179319457616657, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72400896, "prompt_kind": "long", "t_relative_s": 8648.847007791977}, {"decode_latency_s": 0.06038997393625323, "kv_live_bytes": 462274560, "latency_s": 1.9333949168212712, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72237056, "prompt_kind": "short", "t_relative_s": 8653.857069374993}, {"decode_latency_s": 0.06794967578025535, "kv_live_bytes": 462274560, "latency_s": 2.176081624813378, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72302592, "prompt_kind": "long", "t_relative_s": 8658.867130000144}, {"decode_latency_s": 0.06721401953836903, "kv_live_bytes": 462274560, "latency_s": 2.152751292102039, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72417280, "prompt_kind": "short", "t_relative_s": 8663.87719587516}, {"decode_latency_s": 0.06774355209199712, "kv_live_bytes": 462274560, "latency_s": 2.1702105002477765, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72482816, "prompt_kind": "long", "t_relative_s": 8668.887263750192}, {"decode_latency_s": 0.060533014315296896, "kv_live_bytes": 462274560, "latency_s": 1.9378969580866396, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73023488, "prompt_kind": "short", "t_relative_s": 8673.897365666926}, {"decode_latency_s": 0.06782767969707493, "kv_live_bytes": 462274560, "latency_s": 2.1728578340262175, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72990720, "prompt_kind": "long", "t_relative_s": 8678.907417792361}, {"decode_latency_s": 0.0637326926807873, "kv_live_bytes": 462274560, "latency_s": 2.0407245829701424, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69812224, "prompt_kind": "short", "t_relative_s": 8683.917497167364}, {"decode_latency_s": 0.06801529166114051, "kv_live_bytes": 462274560, "latency_s": 2.1789779998362064, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72630272, "prompt_kind": "long", "t_relative_s": 8688.927570917178}, {"decode_latency_s": 0.060522031257278286, "kv_live_bytes": 462274560, "latency_s": 1.937463500071317, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72679424, "prompt_kind": "short", "t_relative_s": 8693.937673208304}, {"decode_latency_s": 0.06791585808969103, "kv_live_bytes": 462274560, "latency_s": 2.1755359582602978, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72761344, "prompt_kind": "long", "t_relative_s": 8698.947723249905}, {"decode_latency_s": 0.06036620441591367, "kv_live_bytes": 462274560, "latency_s": 1.932653000112623, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72744960, "prompt_kind": "short", "t_relative_s": 8703.95778341731}, {"decode_latency_s": 0.06826430989895016, "kv_live_bytes": 462274560, "latency_s": 2.1869747089222074, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73334784, "prompt_kind": "long", "t_relative_s": 8708.967857500073}, {"decode_latency_s": 0.06063365102454554, "kv_live_bytes": 462274560, "latency_s": 1.9411051669158041, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73957376, "prompt_kind": "short", "t_relative_s": 8713.97791529214}, {"decode_latency_s": 0.06833312239905354, "kv_live_bytes": 462274560, "latency_s": 2.188863333314657, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74022912, "prompt_kind": "long", "t_relative_s": 8718.987746917177}, {"decode_latency_s": 0.06056473568605725, "kv_live_bytes": 462274560, "latency_s": 1.9388889172114432, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74072064, "prompt_kind": "short", "t_relative_s": 8723.99782570824}, {"decode_latency_s": 0.0680312395998044, "kv_live_bytes": 462274560, "latency_s": 2.178698708768934, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73531392, "prompt_kind": "long", "t_relative_s": 8729.007870458066}, {"decode_latency_s": 0.060596028648433276, "kv_live_bytes": 462274560, "latency_s": 1.9399442500434816, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73498624, "prompt_kind": "short", "t_relative_s": 8734.0179101252}, {"decode_latency_s": 0.06919274087704252, "kv_live_bytes": 462274560, "latency_s": 2.216787583194673, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73940992, "prompt_kind": "long", "t_relative_s": 8739.027964542154}, {"decode_latency_s": 0.06309192709159106, "kv_live_bytes": 462274560, "latency_s": 2.019834291189909, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73056256, "prompt_kind": "short", "t_relative_s": 8744.038048750255}, {"decode_latency_s": 0.0677800703124376, "kv_live_bytes": 462274560, "latency_s": 2.1711312090046704, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74137600, "prompt_kind": "long", "t_relative_s": 8749.04811520828}, {"decode_latency_s": 0.06301746352983173, "kv_live_bytes": 462274560, "latency_s": 2.0179609581828117, "n_emitted": 32, "ok": true, "process_footprint_bytes": 37535744, "prompt_kind": "short", "t_relative_s": 8754.058195750229}, {"decode_latency_s": 0.06767724736710079, "kv_live_bytes": 462274560, "latency_s": 2.168446875177324, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "long", "t_relative_s": 8759.068251542281}, {"decode_latency_s": 0.060397524721338414, "kv_live_bytes": 462274560, "latency_s": 1.9336392083205283, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73695232, "prompt_kind": "short", "t_relative_s": 8764.078322542366}, {"decode_latency_s": 0.06793117840425111, "kv_live_bytes": 462274560, "latency_s": 2.175332374870777, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73908224, "prompt_kind": "long", "t_relative_s": 8769.088396375068}, {"decode_latency_s": 0.0604411315289326, "kv_live_bytes": 462274560, "latency_s": 1.9349941248074174, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73891840, "prompt_kind": "short", "t_relative_s": 8774.09177750023}, {"decode_latency_s": 0.06852447136770934, "kv_live_bytes": 462274560, "latency_s": 2.1948566250503063, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73531392, "prompt_kind": "long", "t_relative_s": 8779.101856250316}, {"decode_latency_s": 0.06073101299989503, "kv_live_bytes": 462274560, "latency_s": 1.944239666685462, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73580544, "prompt_kind": "short", "t_relative_s": 8784.111910042353}, {"decode_latency_s": 0.0682250143145211, "kv_live_bytes": 462274560, "latency_s": 2.1856607496738434, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73711616, "prompt_kind": "long", "t_relative_s": 8789.121969917323}, {"decode_latency_s": 0.06064376822905615, "kv_live_bytes": 462274560, "latency_s": 1.941401000134647, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74285056, "prompt_kind": "short", "t_relative_s": 8794.132043333258}, {"decode_latency_s": 0.06798675390018616, "kv_live_bytes": 462274560, "latency_s": 2.1771217500790954, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74383360, "prompt_kind": "long", "t_relative_s": 8799.142102083191}, {"decode_latency_s": 0.06180107162799686, "kv_live_bytes": 462274560, "latency_s": 1.9784362497739494, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74350592, "prompt_kind": "short", "t_relative_s": 8804.152176083066}, {"decode_latency_s": 0.0673347812553402, "kv_live_bytes": 462274560, "latency_s": 2.1569827082566917, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74956800, "prompt_kind": "long", "t_relative_s": 8809.162229792215}, {"decode_latency_s": 0.060505604167701676, "kv_live_bytes": 462274560, "latency_s": 1.936975124757737, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75022336, "prompt_kind": "short", "t_relative_s": 8814.172303417232}, {"decode_latency_s": 0.06801467968034558, "kv_live_bytes": 462274560, "latency_s": 2.1785592497326434, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73924608, "prompt_kind": "long", "t_relative_s": 8819.182404167019}, {"decode_latency_s": 0.060272549468209036, "kv_live_bytes": 462274560, "latency_s": 1.9297020831145346, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74366976, "prompt_kind": "short", "t_relative_s": 8824.192460542079}, {"decode_latency_s": 0.06772245180036407, "kv_live_bytes": 462274560, "latency_s": 2.1686204159632325, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74973184, "prompt_kind": "long", "t_relative_s": 8829.20253720833}, {"decode_latency_s": 0.06053095443348866, "kv_live_bytes": 462274560, "latency_s": 1.937910791952163, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75513856, "prompt_kind": "short", "t_relative_s": 8834.212594000157}, {"decode_latency_s": 0.06821589716128074, "kv_live_bytes": 462274560, "latency_s": 2.1852466254495084, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75038720, "prompt_kind": "long", "t_relative_s": 8839.222643833142}, {"decode_latency_s": 0.060535338532645255, "kv_live_bytes": 462274560, "latency_s": 1.9378928747028112, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75071488, "prompt_kind": "short", "t_relative_s": 8844.23272600025}, {"decode_latency_s": 0.060583080718060955, "kv_live_bytes": 462274560, "latency_s": 1.9399138749577105, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74465280, "prompt_kind": "long", "t_relative_s": 8849.242771500256}, {"decode_latency_s": 0.06074260674358811, "kv_live_bytes": 462274560, "latency_s": 1.9447540417313576, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70041600, "prompt_kind": "short", "t_relative_s": 8854.25284849992}, {"decode_latency_s": 0.06745818100171164, "kv_live_bytes": 462274560, "latency_s": 2.1603909581899643, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74022912, "prompt_kind": "long", "t_relative_s": 8859.26292704232}, {"decode_latency_s": 0.06055219272093382, "kv_live_bytes": 462274560, "latency_s": 1.938580417074263, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74661888, "prompt_kind": "short", "t_relative_s": 8864.273000125308}, {"decode_latency_s": 0.06491708462999668, "kv_live_bytes": 462274560, "latency_s": 2.0782854170538485, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73400320, "prompt_kind": "long", "t_relative_s": 8869.283068249933}, {"decode_latency_s": 0.06300598305824678, "kv_live_bytes": 462274560, "latency_s": 2.017277750186622, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68698112, "prompt_kind": "short", "t_relative_s": 8874.293147667311}, {"decode_latency_s": 0.06821177602978423, "kv_live_bytes": 462274560, "latency_s": 2.1854515420272946, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73252864, "prompt_kind": "long", "t_relative_s": 8879.303246208001}, {"decode_latency_s": 0.06071849608269986, "kv_live_bytes": 462274560, "latency_s": 1.9438389167189598, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72941568, "prompt_kind": "short", "t_relative_s": 8884.31332425028}, {"decode_latency_s": 0.06884665496181697, "kv_live_bytes": 462274560, "latency_s": 2.205154957715422, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73383936, "prompt_kind": "long", "t_relative_s": 8889.323378083296}, {"decode_latency_s": 0.06046047915879171, "kv_live_bytes": 462274560, "latency_s": 1.9356745830737054, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73940992, "prompt_kind": "short", "t_relative_s": 8894.33345287526}, {"decode_latency_s": 0.06402303646609653, "kv_live_bytes": 462274560, "latency_s": 2.050802582874894, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70336512, "prompt_kind": "long", "t_relative_s": 8899.343512208201}, {"decode_latency_s": 0.06097684244741686, "kv_live_bytes": 462274560, "latency_s": 1.9521777499467134, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71106560, "prompt_kind": "short", "t_relative_s": 8905.491083958186}, {"decode_latency_s": 0.06850515624682885, "kv_live_bytes": 462274560, "latency_s": 2.1941823749803007, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71221248, "prompt_kind": "long", "t_relative_s": 8910.501173208002}, {"decode_latency_s": 0.060823602878372185, "kv_live_bytes": 462274560, "latency_s": 1.9471560842357576, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71794688, "prompt_kind": "short", "t_relative_s": 8915.511260792147}, {"decode_latency_s": 0.06792969268281013, "kv_live_bytes": 462274560, "latency_s": 2.1758302920497954, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71892992, "prompt_kind": "long", "t_relative_s": 8920.521307874937}, {"decode_latency_s": 0.06073774218384642, "kv_live_bytes": 462274560, "latency_s": 1.9443649579770863, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72417280, "prompt_kind": "short", "t_relative_s": 8925.531375292223}, {"decode_latency_s": 0.0680632564908592, "kv_live_bytes": 462274560, "latency_s": 2.1796655422076583, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72499200, "prompt_kind": "long", "t_relative_s": 8930.54145487491}, {"decode_latency_s": 0.060317531242617406, "kv_live_bytes": 462274560, "latency_s": 1.931067958008498, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72564736, "prompt_kind": "short", "t_relative_s": 8935.551489292178}, {"decode_latency_s": 0.06771092188137118, "kv_live_bytes": 462274560, "latency_s": 2.169043666217476, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "long", "t_relative_s": 8940.561579792295}, {"decode_latency_s": 0.060521324223373085, "kv_live_bytes": 462274560, "latency_s": 1.9374551251530647, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73187328, "prompt_kind": "short", "t_relative_s": 8945.571651583072}, {"decode_latency_s": 0.06040774872235488, "kv_live_bytes": 462274560, "latency_s": 1.9347097496502101, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73220096, "prompt_kind": "long", "t_relative_s": 8950.581699542236}, {"decode_latency_s": 0.06053189843078144, "kv_live_bytes": 462274560, "latency_s": 1.9379799580201507, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73285632, "prompt_kind": "short", "t_relative_s": 8955.591775750276}, {"decode_latency_s": 0.06048424218897708, "kv_live_bytes": 462274560, "latency_s": 1.9364127502776682, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73367552, "prompt_kind": "long", "t_relative_s": 8960.601851208135}, {"decode_latency_s": 0.06027536069450434, "kv_live_bytes": 462274560, "latency_s": 1.9296790002845228, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "short", "t_relative_s": 8965.611904042307}, {"decode_latency_s": 0.06775979687517975, "kv_live_bytes": 462274560, "latency_s": 2.1706132078543305, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73465856, "prompt_kind": "long", "t_relative_s": 8970.621988167055}, {"decode_latency_s": 0.06090552997193299, "kv_live_bytes": 462274560, "latency_s": 1.9496766249649227, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73990144, "prompt_kind": "short", "t_relative_s": 8975.632063041907}, {"decode_latency_s": 0.06781045052048285, "kv_live_bytes": 462274560, "latency_s": 2.172975040972233, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74055680, "prompt_kind": "long", "t_relative_s": 8980.64211799996}, {"decode_latency_s": 0.060409111974877305, "kv_live_bytes": 462274560, "latency_s": 1.9339992920868099, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74055680, "prompt_kind": "short", "t_relative_s": 8985.652178042103}, {"decode_latency_s": 0.06694165234512184, "kv_live_bytes": 462274560, "latency_s": 2.1439077081158757, "n_emitted": 32, "ok": true, "process_footprint_bytes": 65978368, "prompt_kind": "long", "t_relative_s": 8990.662244332954}, {"decode_latency_s": 0.0607876549474895, "kv_live_bytes": 462274560, "latency_s": 1.9461464169435203, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70467584, "prompt_kind": "short", "t_relative_s": 8995.672321625054}, {"decode_latency_s": 0.06743762371479534, "kv_live_bytes": 462274560, "latency_s": 2.1603376250714064, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72974336, "prompt_kind": "long", "t_relative_s": 9000.682382042054}, {"decode_latency_s": 0.060634571593254805, "kv_live_bytes": 462274560, "latency_s": 1.941194707993418, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73285632, "prompt_kind": "short", "t_relative_s": 9005.69244466722}, {"decode_latency_s": 0.06780964975769166, "kv_live_bytes": 462274560, "latency_s": 2.171483499929309, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73351168, "prompt_kind": "long", "t_relative_s": 9010.702525583096}, {"decode_latency_s": 0.06060421746224165, "kv_live_bytes": 462274560, "latency_s": 1.940200749784708, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73973760, "prompt_kind": "short", "t_relative_s": 9015.712575333193}, {"decode_latency_s": 0.06788979559496511, "kv_live_bytes": 462274560, "latency_s": 2.1747916657477617, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74153984, "prompt_kind": "long", "t_relative_s": 9020.722654667217}, {"decode_latency_s": 0.06648079687147401, "kv_live_bytes": 462274560, "latency_s": 2.1290982919745147, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74268672, "prompt_kind": "short", "t_relative_s": 9025.732764542103}, {"decode_latency_s": 0.06832660936925095, "kv_live_bytes": 462274560, "latency_s": 2.188561666291207, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74137600, "prompt_kind": "long", "t_relative_s": 9030.742819332983}, {"decode_latency_s": 0.060453195314039476, "kv_live_bytes": 462274560, "latency_s": 1.9353621248155832, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74072064, "prompt_kind": "short", "t_relative_s": 9035.752900208347}, {"decode_latency_s": 0.06784740624425467, "kv_live_bytes": 462274560, "latency_s": 2.1728655421175063, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74252288, "prompt_kind": "long", "t_relative_s": 9040.76296620816}, {"decode_latency_s": 0.06700635678134859, "kv_live_bytes": 462274560, "latency_s": 2.145736124832183, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74874880, "prompt_kind": "short", "t_relative_s": 9045.773025000002}, {"decode_latency_s": 0.0671588606928708, "kv_live_bytes": 462274560, "latency_s": 2.150950583163649, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74842112, "prompt_kind": "long", "t_relative_s": 9050.783116792329}, {"decode_latency_s": 0.06606869140523486, "kv_live_bytes": 462274560, "latency_s": 2.115923209115863, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74989568, "prompt_kind": "short", "t_relative_s": 9055.793182957917}, {"decode_latency_s": 0.06772060805815272, "kv_live_bytes": 462274560, "latency_s": 2.1700042500160635, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74940416, "prompt_kind": "long", "t_relative_s": 9060.803229582962}, {"decode_latency_s": 0.06049445184180513, "kv_live_bytes": 462274560, "latency_s": 1.9367032912559807, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75202560, "prompt_kind": "short", "t_relative_s": 9065.81334408326}, {"decode_latency_s": 0.06091388281492982, "kv_live_bytes": 462274560, "latency_s": 1.9504820420406759, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74547200, "prompt_kind": "long", "t_relative_s": 9070.823413208127}, {"decode_latency_s": 0.060525118504301645, "kv_live_bytes": 462274560, "latency_s": 1.937755708117038, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74596352, "prompt_kind": "short", "t_relative_s": 9075.833463000134}, {"decode_latency_s": 0.0680617734324187, "kv_live_bytes": 462274560, "latency_s": 2.1807733750902116, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75235328, "prompt_kind": "long", "t_relative_s": 9080.843549333047}, {"decode_latency_s": 0.0607366601616377, "kv_live_bytes": 462274560, "latency_s": 1.9443773748353124, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75431936, "prompt_kind": "short", "t_relative_s": 9085.853649500292}, {"decode_latency_s": 0.0678208814933896, "kv_live_bytes": 462274560, "latency_s": 2.173624916933477, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75579392, "prompt_kind": "long", "t_relative_s": 9090.863711792044}, {"decode_latency_s": 0.060520257815369405, "kv_live_bytes": 462274560, "latency_s": 1.9374919580295682, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75661312, "prompt_kind": "short", "t_relative_s": 9095.873813542072}, {"decode_latency_s": 0.06813572657119948, "kv_live_bytes": 462274560, "latency_s": 2.1829580408520997, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74416128, "prompt_kind": "long", "t_relative_s": 9100.883900958113}, {"decode_latency_s": 0.06066863281012047, "kv_live_bytes": 462274560, "latency_s": 1.9422544999979436, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75104256, "prompt_kind": "short", "t_relative_s": 9105.893984083086}, {"decode_latency_s": 0.06886635937553365, "kv_live_bytes": 462274560, "latency_s": 2.2062189583666623, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75431936, "prompt_kind": "long", "t_relative_s": 9110.904031666927}, {"decode_latency_s": 0.060272785151028074, "kv_live_bytes": 462274560, "latency_s": 1.9297472909092903, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74809344, "prompt_kind": "short", "t_relative_s": 9115.914102375042}, {"decode_latency_s": 0.06783394400554243, "kv_live_bytes": 462274560, "latency_s": 2.173052624799311, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74891264, "prompt_kind": "long", "t_relative_s": 9120.924186292104}, {"decode_latency_s": 0.06698399347078521, "kv_live_bytes": 462274560, "latency_s": 2.145289999898523, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75481088, "prompt_kind": "short", "t_relative_s": 9125.934237292036}, {"decode_latency_s": 0.0675167903827969, "kv_live_bytes": 462274560, "latency_s": 2.1630049580708146, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75563008, "prompt_kind": "long", "t_relative_s": 9130.944310875144}, {"decode_latency_s": 0.06089839975174982, "kv_live_bytes": 462274560, "latency_s": 1.9495169171132147, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76152832, "prompt_kind": "short", "t_relative_s": 9135.954391000327}, {"decode_latency_s": 0.06779149218345992, "kv_live_bytes": 462274560, "latency_s": 2.1716235000640154, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76054528, "prompt_kind": "long", "t_relative_s": 9140.964441708289}, {"decode_latency_s": 0.060393460938939825, "kv_live_bytes": 462274560, "latency_s": 1.9335162080824375, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75235328, "prompt_kind": "short", "t_relative_s": 9145.974503458012}, {"decode_latency_s": 0.06828545313328505, "kv_live_bytes": 462274560, "latency_s": 2.1871182080358267, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75317248, "prompt_kind": "long", "t_relative_s": 9150.984594583046}, {"decode_latency_s": 0.060806497363955714, "kv_live_bytes": 462274560, "latency_s": 1.9466765830293298, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75431936, "prompt_kind": "short", "t_relative_s": 9155.994654917158}, {"decode_latency_s": 0.0680529140663566, "kv_live_bytes": 462274560, "latency_s": 2.1802960000932217, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75481088, "prompt_kind": "long", "t_relative_s": 9161.004723750055}, {"decode_latency_s": 0.06671669399656821, "kv_live_bytes": 462274560, "latency_s": 2.1367303328588605, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75366400, "prompt_kind": "short", "t_relative_s": 9166.014796667267}, {"decode_latency_s": 0.06771039453451522, "kv_live_bytes": 462274560, "latency_s": 2.1689282502047718, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75382784, "prompt_kind": "long", "t_relative_s": 9171.024856000207}, {"decode_latency_s": 0.06064581511600409, "kv_live_bytes": 462274560, "latency_s": 1.9415915827266872, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74825728, "prompt_kind": "short", "t_relative_s": 9176.034940292127}, {"decode_latency_s": 0.06726530859305058, "kv_live_bytes": 462274560, "latency_s": 2.1545755406841636, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75399168, "prompt_kind": "long", "t_relative_s": 9181.045012833085}, {"decode_latency_s": 0.0607361731963465, "kv_live_bytes": 462274560, "latency_s": 1.9444714160636067, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76005376, "prompt_kind": "short", "t_relative_s": 9186.055066000205}, {"decode_latency_s": 0.06768599087081384, "kv_live_bytes": 462274560, "latency_s": 2.168330792337656, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76218368, "prompt_kind": "long", "t_relative_s": 9191.065145791974}, {"decode_latency_s": 0.06083925912389532, "kv_live_bytes": 462274560, "latency_s": 1.9476677500642836, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76775424, "prompt_kind": "short", "t_relative_s": 9196.075226624962}, {"decode_latency_s": 0.06786182681389619, "kv_live_bytes": 462274560, "latency_s": 2.1741126249544322, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76808192, "prompt_kind": "long", "t_relative_s": 9201.085273708217}, {"decode_latency_s": 0.0605369817931205, "kv_live_bytes": 462274560, "latency_s": 1.9380905842408538, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76709888, "prompt_kind": "short", "t_relative_s": 9206.09535362525}, {"decode_latency_s": 0.06033743097214028, "kv_live_bytes": 462274560, "latency_s": 1.9317395417019725, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76234752, "prompt_kind": "long", "t_relative_s": 9211.105438208207}, {"decode_latency_s": 0.06076571875018999, "kv_live_bytes": 462274560, "latency_s": 1.9453827082179487, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76251136, "prompt_kind": "short", "t_relative_s": 9216.115492417011}, {"decode_latency_s": 0.060613540379563347, "kv_live_bytes": 462274560, "latency_s": 1.9413735000416636, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76431360, "prompt_kind": "long", "t_relative_s": 9221.125584042165}, {"decode_latency_s": 0.06696347397519276, "kv_live_bytes": 462274560, "latency_s": 2.144568667281419, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76906496, "prompt_kind": "short", "t_relative_s": 9226.135669375304}, {"decode_latency_s": 0.06774325127480552, "kv_live_bytes": 462274560, "latency_s": 2.169894667342305, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76775424, "prompt_kind": "long", "t_relative_s": 9231.145719042048}, {"decode_latency_s": 0.06033219271921553, "kv_live_bytes": 462274560, "latency_s": 1.9315564162097871, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76218368, "prompt_kind": "short", "t_relative_s": 9236.155805250164}, {"decode_latency_s": 0.06744172528851777, "kv_live_bytes": 462274560, "latency_s": 2.1598627092316747, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76218368, "prompt_kind": "long", "t_relative_s": 9241.16587566724}, {"decode_latency_s": 0.06697017708211206, "kv_live_bytes": 462274560, "latency_s": 2.1448520408011973, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76267520, "prompt_kind": "short", "t_relative_s": 9246.175930083264}, {"decode_latency_s": 0.06776298697513994, "kv_live_bytes": 462274560, "latency_s": 2.1709255832247436, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76349440, "prompt_kind": "long", "t_relative_s": 9251.186046542134}, {"decode_latency_s": 0.060695611959090456, "kv_live_bytes": 462274560, "latency_s": 1.943020333070308, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76644352, "prompt_kind": "short", "t_relative_s": 9256.19615712529}, {"decode_latency_s": 0.06824043359665666, "kv_live_bytes": 462274560, "latency_s": 2.185782916843891, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76693504, "prompt_kind": "long", "t_relative_s": 9261.206213749945}, {"decode_latency_s": 0.060394397121854126, "kv_live_bytes": 462274560, "latency_s": 1.933516625314951, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76677120, "prompt_kind": "short", "t_relative_s": 9266.216302208137}, {"decode_latency_s": 0.06789934897096828, "kv_live_bytes": 462274560, "latency_s": 2.174520082771778, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76709888, "prompt_kind": "long", "t_relative_s": 9271.226391583215}, {"decode_latency_s": 0.06065126562316436, "kv_live_bytes": 462274560, "latency_s": 1.9416938750073314, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76775424, "prompt_kind": "short", "t_relative_s": 9276.236449375283}, {"decode_latency_s": 0.06799774087266997, "kv_live_bytes": 462274560, "latency_s": 2.17829104186967, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76857344, "prompt_kind": "long", "t_relative_s": 9281.246522833128}, {"decode_latency_s": 0.060709023440722376, "kv_live_bytes": 462274560, "latency_s": 1.9437852501869202, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72318976, "prompt_kind": "short", "t_relative_s": 9286.256632833276}, {"decode_latency_s": 0.06815992968040518, "kv_live_bytes": 462274560, "latency_s": 2.1832378329709172, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75923456, "prompt_kind": "long", "t_relative_s": 9291.266707000323}, {"decode_latency_s": 0.060686735683702864, "kv_live_bytes": 462274560, "latency_s": 1.942838708870113, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75366400, "prompt_kind": "short", "t_relative_s": 9296.276767666917}, {"decode_latency_s": 0.06822001822001766, "kv_live_bytes": 462274560, "latency_s": 2.185594040900469, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74956800, "prompt_kind": "long", "t_relative_s": 9301.286834333092}, {"decode_latency_s": 0.0607518059987342, "kv_live_bytes": 462274560, "latency_s": 1.9448811248876154, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75022336, "prompt_kind": "short", "t_relative_s": 9306.296901250258}, {"decode_latency_s": 0.06824149868043605, "kv_live_bytes": 462274560, "latency_s": 2.1866890001110733, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73465856, "prompt_kind": "long", "t_relative_s": 9311.306978750043}, {"decode_latency_s": 0.06029413543001283, "kv_live_bytes": 462274560, "latency_s": 1.930280499625951, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74629120, "prompt_kind": "short", "t_relative_s": 9316.317038042005}, {"decode_latency_s": 0.0679226914071478, "kv_live_bytes": 462274560, "latency_s": 2.1751989163458347, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74891264, "prompt_kind": "long", "t_relative_s": 9321.327109333128}, {"decode_latency_s": 0.0604177174391225, "kv_live_bytes": 462274560, "latency_s": 1.9342620419338346, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74956800, "prompt_kind": "short", "t_relative_s": 9326.337180375122}, {"decode_latency_s": 0.06825960677815601, "kv_live_bytes": 462274560, "latency_s": 2.1869760840199888, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75005952, "prompt_kind": "long", "t_relative_s": 9331.347264125012}, {"decode_latency_s": 0.060624443998676725, "kv_live_bytes": 462274560, "latency_s": 1.94077012501657, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75104256, "prompt_kind": "short", "t_relative_s": 9336.357347917277}, {"decode_latency_s": 0.06824826430238318, "kv_live_bytes": 462274560, "latency_s": 2.1860711248591542, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75235328, "prompt_kind": "long", "t_relative_s": 9341.367400208022}, {"decode_latency_s": 0.06026567058870569, "kv_live_bytes": 462274560, "latency_s": 1.9293937920592725, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75792384, "prompt_kind": "short", "t_relative_s": 9346.37746229209}, {"decode_latency_s": 0.06791585155588109, "kv_live_bytes": 462274560, "latency_s": 2.1750486250966787, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75284480, "prompt_kind": "long", "t_relative_s": 9351.387528917287}, {"decode_latency_s": 0.06575966274249367, "kv_live_bytes": 462274560, "latency_s": 2.1060167918913066, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75333632, "prompt_kind": "short", "t_relative_s": 9356.397588416934}, {"decode_latency_s": 0.0677021510200575, "kv_live_bytes": 462274560, "latency_s": 2.168929624836892, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75448320, "prompt_kind": "long", "t_relative_s": 9361.407702458091}, {"decode_latency_s": 0.06680506639531814, "kv_live_bytes": 462274560, "latency_s": 2.1395035409368575, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75431936, "prompt_kind": "short", "t_relative_s": 9366.417788917199}, {"decode_latency_s": 0.06222752343455795, "kv_live_bytes": 462274560, "latency_s": 1.9926808751188219, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75448320, "prompt_kind": "long", "t_relative_s": 9371.427865874954}, {"decode_latency_s": 0.06079390626109671, "kv_live_bytes": 462274560, "latency_s": 1.9462752919644117, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74825728, "prompt_kind": "short", "t_relative_s": 9376.43796870811}, {"decode_latency_s": 0.06794067578448448, "kv_live_bytes": 462274560, "latency_s": 2.1758503327146173, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75087872, "prompt_kind": "long", "t_relative_s": 9381.448060917202}, {"decode_latency_s": 0.06071338930632919, "kv_live_bytes": 462274560, "latency_s": 1.9438160830177367, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75137024, "prompt_kind": "short", "t_relative_s": 9386.458107292186}, {"decode_latency_s": 0.06815260025905445, "kv_live_bytes": 462274560, "latency_s": 2.1832719170488417, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75235328, "prompt_kind": "long", "t_relative_s": 9391.46820945805}, {"decode_latency_s": 0.0667324570385972, "kv_live_bytes": 462274560, "latency_s": 2.137153666932136, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75284480, "prompt_kind": "short", "t_relative_s": 9396.478302417323}, {"decode_latency_s": 0.0683263359387638, "kv_live_bytes": 462274560, "latency_s": 2.1883402089588344, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75284480, "prompt_kind": "long", "t_relative_s": 9401.48834658321}, {"decode_latency_s": 0.060829850248410366, "kv_live_bytes": 462274560, "latency_s": 1.9474652921780944, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73711616, "prompt_kind": "short", "t_relative_s": 9406.498447916936}, {"decode_latency_s": 0.06770594923000317, "kv_live_bytes": 462274560, "latency_s": 2.16830737516284, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75563008, "prompt_kind": "long", "t_relative_s": 9411.508530625142}, {"decode_latency_s": 0.06073744271998294, "kv_live_bytes": 462274560, "latency_s": 1.9444999159313738, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75677696, "prompt_kind": "short", "t_relative_s": 9416.51858333312}, {"decode_latency_s": 0.06771098566241562, "kv_live_bytes": 462274560, "latency_s": 2.1690643751062453, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75726848, "prompt_kind": "long", "t_relative_s": 9421.528670458123}, {"decode_latency_s": 0.06681634244159795, "kv_live_bytes": 462274560, "latency_s": 2.139723500236869, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75759616, "prompt_kind": "short", "t_relative_s": 9426.537982082926}, {"decode_latency_s": 0.0679177460988285, "kv_live_bytes": 462274560, "latency_s": 2.17536941729486, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75382784, "prompt_kind": "long", "t_relative_s": 9431.5480184583}, {"decode_latency_s": 0.06026388281316031, "kv_live_bytes": 462274560, "latency_s": 1.9293398750014603, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74694656, "prompt_kind": "short", "t_relative_s": 9436.558099499904}, {"decode_latency_s": 0.06830114584590774, "kv_live_bytes": 462274560, "latency_s": 2.187325374688953, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75350016, "prompt_kind": "long", "t_relative_s": 9441.56819345802}, {"decode_latency_s": 0.06052491665468551, "kv_live_bytes": 462274560, "latency_s": 1.937700083013624, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75448320, "prompt_kind": "short", "t_relative_s": 9446.578244583216}, {"decode_latency_s": 0.06728167450637557, "kv_live_bytes": 462274560, "latency_s": 2.1554314168170094, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75530240, "prompt_kind": "long", "t_relative_s": 9451.588321249932}, {"decode_latency_s": 0.06058010156266391, "kv_live_bytes": 462274560, "latency_s": 1.9393632910214365, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75563008, "prompt_kind": "short", "t_relative_s": 9456.598429583013}, {"decode_latency_s": 0.06790387761429884, "kv_live_bytes": 462274560, "latency_s": 2.1749545000493526, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75431936, "prompt_kind": "long", "t_relative_s": 9461.60848091729}, {"decode_latency_s": 0.06045158593042288, "kv_live_bytes": 462274560, "latency_s": 1.9353502499870956, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75431936, "prompt_kind": "short", "t_relative_s": 9466.618563167285}, {"decode_latency_s": 0.06815836587338708, "kv_live_bytes": 462274560, "latency_s": 2.1828174591064453, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76054528, "prompt_kind": "long", "t_relative_s": 9471.628642333206}, {"decode_latency_s": 0.0605138294340577, "kv_live_bytes": 462274560, "latency_s": 1.9373732502572238, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76218368, "prompt_kind": "short", "t_relative_s": 9476.638702417258}, {"decode_latency_s": 0.06738259896519594, "kv_live_bytes": 462274560, "latency_s": 2.158578457776457, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76365824, "prompt_kind": "long", "t_relative_s": 9481.648782667238}, {"decode_latency_s": 0.060635201807599515, "kv_live_bytes": 462274560, "latency_s": 1.941111583262682, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76414976, "prompt_kind": "short", "t_relative_s": 9486.658888542093}, {"decode_latency_s": 0.06040422135265544, "kv_live_bytes": 462274560, "latency_s": 1.9339819997549057, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76447744, "prompt_kind": "long", "t_relative_s": 9491.668931417167}, {"decode_latency_s": 0.06069433984521311, "kv_live_bytes": 462274560, "latency_s": 1.9431346249766648, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73711616, "prompt_kind": "short", "t_relative_s": 9496.679014916997}, {"decode_latency_s": 0.06828447006409988, "kv_live_bytes": 462274560, "latency_s": 2.187406832817942, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75382784, "prompt_kind": "long", "t_relative_s": 9501.689084333368}, {"decode_latency_s": 0.06037682943861, "kv_live_bytes": 462274560, "latency_s": 1.9329471252858639, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75546624, "prompt_kind": "short", "t_relative_s": 9506.699129958171}, {"decode_latency_s": 0.06811341016145889, "kv_live_bytes": 462274560, "latency_s": 2.1818949161097407, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75612160, "prompt_kind": "long", "t_relative_s": 9511.709217917174}, {"decode_latency_s": 0.06054125649097841, "kv_live_bytes": 462274560, "latency_s": 1.9381121252663434, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75628544, "prompt_kind": "short", "t_relative_s": 9516.719314292073}, {"decode_latency_s": 0.06828496875823475, "kv_live_bytes": 462274560, "latency_s": 2.1870622923597693, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75874304, "prompt_kind": "long", "t_relative_s": 9521.72937183315}, {"decode_latency_s": 0.06037841796933208, "kv_live_bytes": 462274560, "latency_s": 1.9330162908881903, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76283904, "prompt_kind": "short", "t_relative_s": 9526.739454207942}, {"decode_latency_s": 0.06805843622714747, "kv_live_bytes": 462274560, "latency_s": 2.1796924169175327, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76300288, "prompt_kind": "long", "t_relative_s": 9531.749546166975}, {"decode_latency_s": 0.06651087368663866, "kv_live_bytes": 462274560, "latency_s": 2.1302935839630663, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76513280, "prompt_kind": "short", "t_relative_s": 9536.759601583239}, {"decode_latency_s": 0.06808749999618158, "kv_live_bytes": 462274560, "latency_s": 2.180907292291522, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76562432, "prompt_kind": "long", "t_relative_s": 9541.769703042228}, {"decode_latency_s": 0.060643317687208764, "kv_live_bytes": 462274560, "latency_s": 1.941364916972816, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76595200, "prompt_kind": "short", "t_relative_s": 9546.779782833066}, {"decode_latency_s": 0.06819510286732111, "kv_live_bytes": 462274560, "latency_s": 2.1847334587946534, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75628544, "prompt_kind": "long", "t_relative_s": 9551.789837958291}, {"decode_latency_s": 0.0624046380398795, "kv_live_bytes": 462274560, "latency_s": 1.9977325000800192, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72466432, "prompt_kind": "short", "t_relative_s": 9556.799893542193}, {"decode_latency_s": 0.06814590885187499, "kv_live_bytes": 462274560, "latency_s": 2.1831662501208484, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75415552, "prompt_kind": "long", "t_relative_s": 9561.809957083315}, {"decode_latency_s": 0.0606035247183172, "kv_live_bytes": 462274560, "latency_s": 1.9401198751293123, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75464704, "prompt_kind": "short", "t_relative_s": 9566.82003991725}, {"decode_latency_s": 0.06807094531541225, "kv_live_bytes": 462274560, "latency_s": 2.180344333872199, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74743808, "prompt_kind": "long", "t_relative_s": 9571.830072875135}, {"decode_latency_s": 0.06068572527146898, "kv_live_bytes": 462274560, "latency_s": 1.9427685420960188, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72613888, "prompt_kind": "short", "t_relative_s": 9576.840159958228}, {"decode_latency_s": 0.06778775913699064, "kv_live_bytes": 462274560, "latency_s": 2.1707237916998565, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73859072, "prompt_kind": "long", "t_relative_s": 9581.850206542294}, {"decode_latency_s": 0.060723016897100024, "kv_live_bytes": 462274560, "latency_s": 1.9439484160393476, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73695232, "prompt_kind": "short", "t_relative_s": 9586.860255500302}, {"decode_latency_s": 0.06780910675297491, "kv_live_bytes": 462274560, "latency_s": 2.171333166770637, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75530240, "prompt_kind": "long", "t_relative_s": 9591.868515999988}, {"decode_latency_s": 0.060318514340906404, "kv_live_bytes": 462274560, "latency_s": 1.931088542100042, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75595776, "prompt_kind": "short", "t_relative_s": 9596.869015583303}, {"decode_latency_s": 0.06772771483520046, "kv_live_bytes": 462274560, "latency_s": 2.1695184581913054, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75808768, "prompt_kind": "long", "t_relative_s": 9601.879110082984}, {"decode_latency_s": 0.06059711199486628, "kv_live_bytes": 462274560, "latency_s": 1.93992062471807, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75841536, "prompt_kind": "short", "t_relative_s": 9606.889154667035}, {"decode_latency_s": 0.06835158984176815, "kv_live_bytes": 462274560, "latency_s": 2.1897840830497444, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75923456, "prompt_kind": "long", "t_relative_s": 9611.899199374951}, {"decode_latency_s": 0.06260151561582461, "kv_live_bytes": 462274560, "latency_s": 2.0042178332805634, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75317248, "prompt_kind": "short", "t_relative_s": 9616.90928570833}, {"decode_latency_s": 0.06780091796827037, "kv_live_bytes": 462274560, "latency_s": 2.171704832930118, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75956224, "prompt_kind": "long", "t_relative_s": 9621.91935420828}, {"decode_latency_s": 0.06756031382246874, "kv_live_bytes": 462274560, "latency_s": 2.1636087084189057, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75317248, "prompt_kind": "short", "t_relative_s": 9626.92946966691}, {"decode_latency_s": 0.06769167841412127, "kv_live_bytes": 462274560, "latency_s": 2.168541958089918, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75218944, "prompt_kind": "long", "t_relative_s": 9631.939508500043}, {"decode_latency_s": 0.06047025522275362, "kv_live_bytes": 462274560, "latency_s": 1.9359723753295839, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75235328, "prompt_kind": "short", "t_relative_s": 9636.94961479213}, {"decode_latency_s": 0.06851094921876211, "kv_live_bytes": 462274560, "latency_s": 2.1939241248182952, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72499200, "prompt_kind": "long", "t_relative_s": 9641.959697416984}, {"decode_latency_s": 0.06664452735276427, "kv_live_bytes": 462274560, "latency_s": 2.1349327908828855, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74596352, "prompt_kind": "short", "t_relative_s": 9646.96979141701}, {"decode_latency_s": 0.06557664844149258, "kv_live_bytes": 462274560, "latency_s": 2.1005362919531763, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75153408, "prompt_kind": "long", "t_relative_s": 9651.979846208356}, {"decode_latency_s": 0.06042671615432482, "kv_live_bytes": 462274560, "latency_s": 1.9345801249146461, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74301440, "prompt_kind": "short", "t_relative_s": 9656.98992375005}, {"decode_latency_s": 0.06555982813006267, "kv_live_bytes": 462274560, "latency_s": 2.099562041927129, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69795840, "prompt_kind": "long", "t_relative_s": 9661.999976708088}, {"decode_latency_s": 0.06081181380432099, "kv_live_bytes": 462274560, "latency_s": 1.946898290887475, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73596928, "prompt_kind": "short", "t_relative_s": 9667.010047625285}, {"decode_latency_s": 0.06064314712421037, "kv_live_bytes": 462274560, "latency_s": 1.941627792082727, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72843264, "prompt_kind": "long", "t_relative_s": 9672.020082333125}, {"decode_latency_s": 0.06238199608924333, "kv_live_bytes": 462274560, "latency_s": 1.9970148750580847, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72073216, "prompt_kind": "short", "t_relative_s": 9677.030130042229}, {"decode_latency_s": 0.06820717579103075, "kv_live_bytes": 462274560, "latency_s": 2.1849121251143515, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74727424, "prompt_kind": "long", "t_relative_s": 9682.040179625154}, {"decode_latency_s": 0.060641877600573935, "kv_live_bytes": 462274560, "latency_s": 1.9412925001233816, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74842112, "prompt_kind": "short", "t_relative_s": 9687.050265708007}, {"decode_latency_s": 0.060633113273070194, "kv_live_bytes": 462274560, "latency_s": 1.941096667200327, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75284480, "prompt_kind": "long", "t_relative_s": 9692.060342042241}, {"decode_latency_s": 0.060484230460133404, "kv_live_bytes": 462274560, "latency_s": 1.9364958750084043, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74940416, "prompt_kind": "short", "t_relative_s": 9697.070395042188}, {"decode_latency_s": 0.06878479165607132, "kv_live_bytes": 462274560, "latency_s": 2.2030757078900933, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75104256, "prompt_kind": "long", "t_relative_s": 9702.080469958019}, {"decode_latency_s": 0.0612303567759227, "kv_live_bytes": 462274560, "latency_s": 1.9602850838564336, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74989568, "prompt_kind": "short", "t_relative_s": 9707.090516000055}, {"decode_latency_s": 0.06791855210030917, "kv_live_bytes": 462274560, "latency_s": 2.174893250223249, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73400320, "prompt_kind": "long", "t_relative_s": 9712.10062158294}, {"decode_latency_s": 0.06031438543868717, "kv_live_bytes": 462274560, "latency_s": 1.9309337921440601, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74678272, "prompt_kind": "short", "t_relative_s": 9717.11067495821}, {"decode_latency_s": 0.0679314322915161, "kv_live_bytes": 462274560, "latency_s": 2.1757936673238873, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75284480, "prompt_kind": "long", "t_relative_s": 9722.120742625091}, {"decode_latency_s": 0.06073309505882207, "kv_live_bytes": 462274560, "latency_s": 1.944253166206181, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75317248, "prompt_kind": "short", "t_relative_s": 9727.130807792302}, {"decode_latency_s": 0.0683636627800297, "kv_live_bytes": 462274560, "latency_s": 2.190413041971624, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75120640, "prompt_kind": "long", "t_relative_s": 9732.140865750145}, {"decode_latency_s": 0.062881442689104, "kv_live_bytes": 462274560, "latency_s": 2.0131461252458394, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74661888, "prompt_kind": "short", "t_relative_s": 9737.150960708037}, {"decode_latency_s": 0.06804323568940163, "kv_live_bytes": 462274560, "latency_s": 2.179894792381674, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75694080, "prompt_kind": "long", "t_relative_s": 9742.161033832934}, {"decode_latency_s": 0.06248862236679997, "kv_live_bytes": 462274560, "latency_s": 2.0004605003632605, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90193920, "prompt_kind": "short", "t_relative_s": 9747.171111875214}, {"decode_latency_s": 0.06801878774422221, "kv_live_bytes": 462274560, "latency_s": 2.1791366669349372, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90210304, "prompt_kind": "long", "t_relative_s": 9752.181156042032}, {"decode_latency_s": 0.06051574740558863, "kv_live_bytes": 462274560, "latency_s": 1.9374216659925878, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90112000, "prompt_kind": "short", "t_relative_s": 9757.191227250267}, {"decode_latency_s": 0.06859125000482891, "kv_live_bytes": 462274560, "latency_s": 2.1966065410524607, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88539136, "prompt_kind": "long", "t_relative_s": 9762.201305625029}, {"decode_latency_s": 0.06761985806224402, "kv_live_bytes": 462274560, "latency_s": 2.165699750185013, "n_emitted": 32, "ok": true, "process_footprint_bytes": 89276416, "prompt_kind": "short", "t_relative_s": 9767.211361750029}, {"decode_latency_s": 0.06765882813488133, "kv_live_bytes": 462274560, "latency_s": 2.167114249896258, "n_emitted": 32, "ok": true, "process_footprint_bytes": 89948160, "prompt_kind": "long", "t_relative_s": 9772.221469750162}, {"decode_latency_s": 0.06060289063316304, "kv_live_bytes": 462274560, "latency_s": 1.9401282076723874, "n_emitted": 32, "ok": true, "process_footprint_bytes": 89964544, "prompt_kind": "short", "t_relative_s": 9777.231577167287}, {"decode_latency_s": 0.06789551433757879, "kv_live_bytes": 462274560, "latency_s": 2.1749971667304635, "n_emitted": 32, "ok": true, "process_footprint_bytes": 89964544, "prompt_kind": "long", "t_relative_s": 9782.241635249928}, {"decode_latency_s": 0.06032345835410524, "kv_live_bytes": 462274560, "latency_s": 1.9310989580117166, "n_emitted": 32, "ok": true, "process_footprint_bytes": 89964544, "prompt_kind": "short", "t_relative_s": 9787.251724125352}, {"decode_latency_s": 0.06805143489327747, "kv_live_bytes": 462274560, "latency_s": 2.1791366660036147, "n_emitted": 32, "ok": true, "process_footprint_bytes": 87703552, "prompt_kind": "long", "t_relative_s": 9792.261810542084}, {"decode_latency_s": 0.06091050780378282, "kv_live_bytes": 462274560, "latency_s": 1.9499710001982749, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88424448, "prompt_kind": "short", "t_relative_s": 9797.271886250004}, {"decode_latency_s": 0.06786165104131214, "kv_live_bytes": 462274560, "latency_s": 2.1735169580206275, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88932352, "prompt_kind": "long", "t_relative_s": 9802.281943375245}, {"decode_latency_s": 0.06027551040460821, "kv_live_bytes": 462274560, "latency_s": 1.9297746657393873, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88834048, "prompt_kind": "short", "t_relative_s": 9807.291999208275}, {"decode_latency_s": 0.060558895842405036, "kv_live_bytes": 462274560, "latency_s": 1.9387914161197841, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88752128, "prompt_kind": "long", "t_relative_s": 9812.302071417216}, {"decode_latency_s": 0.060673250001855195, "kv_live_bytes": 462274560, "latency_s": 1.9424327500164509, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88883200, "prompt_kind": "short", "t_relative_s": 9817.31211454235}, {"decode_latency_s": 0.06836445444787387, "kv_live_bytes": 462274560, "latency_s": 2.1898033749312162, "n_emitted": 32, "ok": true, "process_footprint_bytes": 87441408, "prompt_kind": "long", "t_relative_s": 9822.322207458317}, {"decode_latency_s": 0.0667968216148438, "kv_live_bytes": 462274560, "latency_s": 2.1391367078758776, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88145920, "prompt_kind": "short", "t_relative_s": 9827.33229308296}, {"decode_latency_s": 0.06801744659605902, "kv_live_bytes": 462274560, "latency_s": 2.1784318750724196, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74940416, "prompt_kind": "long", "t_relative_s": 9832.34235400008}, {"decode_latency_s": 0.06035529034852516, "kv_live_bytes": 462274560, "latency_s": 1.9322581668384373, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75005952, "prompt_kind": "short", "t_relative_s": 9837.352423958015}, {"decode_latency_s": 0.0679013398475945, "kv_live_bytes": 462274560, "latency_s": 2.1742174169048667, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75137024, "prompt_kind": "long", "t_relative_s": 9842.362497624941}, {"decode_latency_s": 0.06351848956546746, "kv_live_bytes": 462274560, "latency_s": 2.0334972497075796, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75218944, "prompt_kind": "short", "t_relative_s": 9847.372545875143}, {"decode_latency_s": 0.06829838540579658, "kv_live_bytes": 462274560, "latency_s": 2.187780833337456, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75235328, "prompt_kind": "long", "t_relative_s": 9852.382628499996}, {"decode_latency_s": 0.06055714584363159, "kv_live_bytes": 462274560, "latency_s": 1.9386255419813097, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75759616, "prompt_kind": "short", "t_relative_s": 9857.392711667344}, {"decode_latency_s": 0.06790785936755128, "kv_live_bytes": 462274560, "latency_s": 2.175347290933132, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75841536, "prompt_kind": "long", "t_relative_s": 9862.40276366705}, {"decode_latency_s": 0.06023623957298696, "kv_live_bytes": 462274560, "latency_s": 1.928433082997799, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75038720, "prompt_kind": "short", "t_relative_s": 9867.412837458309}, {"decode_latency_s": 0.06035486586915795, "kv_live_bytes": 462274560, "latency_s": 1.9322514589875937, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75300864, "prompt_kind": "long", "t_relative_s": 9872.422916542273}, {"decode_latency_s": 0.060233552096178755, "kv_live_bytes": 462274560, "latency_s": 1.9283479996956885, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75300864, "prompt_kind": "short", "t_relative_s": 9877.43296950031}, {"decode_latency_s": 0.06838738544320222, "kv_live_bytes": 462274560, "latency_s": 2.190600708127022, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75415552, "prompt_kind": "long", "t_relative_s": 9882.4430455002}, {"decode_latency_s": 0.06600769399665296, "kv_live_bytes": 462274560, "latency_s": 2.1139379581436515, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74383360, "prompt_kind": "short", "t_relative_s": 9887.453161833342}, {"decode_latency_s": 0.06774815887911245, "kv_live_bytes": 462274560, "latency_s": 2.16881295805797, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69632000, "prompt_kind": "long", "t_relative_s": 9892.46323391702}, {"decode_latency_s": 0.06078150909161195, "kv_live_bytes": 462274560, "latency_s": 1.945783874951303, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69959680, "prompt_kind": "short", "t_relative_s": 9899.140753167216}, {"decode_latency_s": 0.06830529816215858, "kv_live_bytes": 462274560, "latency_s": 2.18761625001207, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68878336, "prompt_kind": "long", "t_relative_s": 9904.150836958084}, {"decode_latency_s": 0.06021749218052719, "kv_live_bytes": 462274560, "latency_s": 1.9278783751651645, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69910528, "prompt_kind": "short", "t_relative_s": 9909.160885208286}, {"decode_latency_s": 0.068127339851344, "kv_live_bytes": 462274560, "latency_s": 2.182158250361681, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70156288, "prompt_kind": "long", "t_relative_s": 9914.170958333183}, {"decode_latency_s": 0.06115946744102985, "kv_live_bytes": 462274560, "latency_s": 1.957890666089952, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70696960, "prompt_kind": "short", "t_relative_s": 9919.17715062527}, {"decode_latency_s": 0.06788977865653578, "kv_live_bytes": 462274560, "latency_s": 2.1751840417273343, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69468160, "prompt_kind": "long", "t_relative_s": 9924.187205625232}, {"decode_latency_s": 0.06730376822815742, "kv_live_bytes": 462274560, "latency_s": 2.1554558337666094, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69779456, "prompt_kind": "short", "t_relative_s": 9929.197296583094}, {"decode_latency_s": 0.06040716664574575, "kv_live_bytes": 462274560, "latency_s": 1.9338442077860236, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69844992, "prompt_kind": "long", "t_relative_s": 9934.20734391734}, {"decode_latency_s": 0.06036778257112019, "kv_live_bytes": 462274560, "latency_s": 1.932655541691929, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70483968, "prompt_kind": "short", "t_relative_s": 9939.217412875034}, {"decode_latency_s": 0.06825542316073552, "kv_live_bytes": 462274560, "latency_s": 2.1867667082697153, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70516736, "prompt_kind": "long", "t_relative_s": 9944.22745083319}, {"decode_latency_s": 0.06038929555506911, "kv_live_bytes": 462274560, "latency_s": 1.9332027090713382, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71057408, "prompt_kind": "short", "t_relative_s": 9949.237537083216}, {"decode_latency_s": 0.06656644790200517, "kv_live_bytes": 462274560, "latency_s": 2.1319045843556523, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71106560, "prompt_kind": "long", "t_relative_s": 9954.247621749993}, {"decode_latency_s": 0.06044232162821572, "kv_live_bytes": 462274560, "latency_s": 1.9350528749637306, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71139328, "prompt_kind": "short", "t_relative_s": 9959.257677874994}, {"decode_latency_s": 0.06871475653315429, "kv_live_bytes": 462274560, "latency_s": 2.2012724997475743, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71319552, "prompt_kind": "long", "t_relative_s": 9964.267762708012}, {"decode_latency_s": 0.060637063812464476, "kv_live_bytes": 462274560, "latency_s": 1.941211124882102, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71892992, "prompt_kind": "short", "t_relative_s": 9969.27786820801}, {"decode_latency_s": 0.06824430859705899, "kv_live_bytes": 462274560, "latency_s": 2.1864616670645773, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71483392, "prompt_kind": "long", "t_relative_s": 9974.287920708302}, {"decode_latency_s": 0.06048107812239323, "kv_live_bytes": 462274560, "latency_s": 1.936212666798383, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71417856, "prompt_kind": "short", "t_relative_s": 9979.297991583124}, {"decode_latency_s": 0.06796660284453537, "kv_live_bytes": 462274560, "latency_s": 2.176666708663106, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71860224, "prompt_kind": "long", "t_relative_s": 9984.308039499912}, {"decode_latency_s": 0.060238277350435965, "kv_live_bytes": 462274560, "latency_s": 1.928597167134285, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70516736, "prompt_kind": "short", "t_relative_s": 9989.318090583198}, {"decode_latency_s": 0.06828857812797651, "kv_live_bytes": 462274560, "latency_s": 2.187305625062436, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71614464, "prompt_kind": "long", "t_relative_s": 9994.328155792318}, {"decode_latency_s": 0.06043149478500709, "kv_live_bytes": 462274560, "latency_s": 1.9346124157309532, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72122368, "prompt_kind": "short", "t_relative_s": 9999.338221500162}, {"decode_latency_s": 0.06762654424528591, "kv_live_bytes": 462274560, "latency_s": 2.1663845414295793, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72204288, "prompt_kind": "long", "t_relative_s": 10004.348287832923}, {"decode_latency_s": 0.06053580209845677, "kv_live_bytes": 462274560, "latency_s": 1.9380000000819564, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72433664, "prompt_kind": "short", "t_relative_s": 10009.358372500166}, {"decode_latency_s": 0.06801692317822017, "kv_live_bytes": 462274560, "latency_s": 2.178311957977712, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71680000, "prompt_kind": "long", "t_relative_s": 10014.368432083167}, {"decode_latency_s": 0.06026401824783534, "kv_live_bytes": 462274560, "latency_s": 1.9293957501649857, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71286784, "prompt_kind": "short", "t_relative_s": 10019.378474917263}, {"decode_latency_s": 0.06831786588008981, "kv_live_bytes": 462274560, "latency_s": 2.187805207911879, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71860224, "prompt_kind": "long", "t_relative_s": 10024.38855104195}, {"decode_latency_s": 0.06054205728287343, "kv_live_bytes": 462274560, "latency_s": 1.938202042132616, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72466432, "prompt_kind": "short", "t_relative_s": 10029.398610292003}, {"decode_latency_s": 0.06805157422786579, "kv_live_bytes": 462274560, "latency_s": 2.1799073331058025, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72515584, "prompt_kind": "long", "t_relative_s": 10034.408682167064}, {"decode_latency_s": 0.060949887993047014, "kv_live_bytes": 462274560, "latency_s": 1.9512428753077984, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70156288, "prompt_kind": "short", "t_relative_s": 10039.418732041959}, {"decode_latency_s": 0.06792924999899697, "kv_live_bytes": 462274560, "latency_s": 2.175799749791622, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71892992, "prompt_kind": "long", "t_relative_s": 10044.428800042253}, {"decode_latency_s": 0.06052144792920444, "kv_live_bytes": 462274560, "latency_s": 1.9374854159541428, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72007680, "prompt_kind": "short", "t_relative_s": 10049.438875125255}, {"decode_latency_s": 0.06838379424880259, "kv_live_bytes": 462274560, "latency_s": 2.1909556249156594, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72089600, "prompt_kind": "long", "t_relative_s": 10054.448943417054}, {"decode_latency_s": 0.060544207037310116, "kv_live_bytes": 462274560, "latency_s": 1.9381692497991025, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72728576, "prompt_kind": "short", "t_relative_s": 10059.459048416931}, {"decode_latency_s": 0.06836973437748384, "kv_live_bytes": 462274560, "latency_s": 2.1900440002791584, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71778304, "prompt_kind": "long", "t_relative_s": 10064.463478042278}, {"decode_latency_s": 0.06666979037981946, "kv_live_bytes": 462274560, "latency_s": 2.135360292159021, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71925760, "prompt_kind": "short", "t_relative_s": 10069.473549458198}, {"decode_latency_s": 0.06774811455397867, "kv_live_bytes": 462274560, "latency_s": 2.1701329168863595, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72564736, "prompt_kind": "long", "t_relative_s": 10074.483598249964}, {"decode_latency_s": 0.060305253908154555, "kv_live_bytes": 462274560, "latency_s": 1.9306317502632737, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72744960, "prompt_kind": "short", "t_relative_s": 10079.49365662504}, {"decode_latency_s": 0.0685477942606667, "kv_live_bytes": 462274560, "latency_s": 2.1958199157379568, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72761344, "prompt_kind": "long", "t_relative_s": 10084.50373608293}, {"decode_latency_s": 0.06680486199911684, "kv_live_bytes": 462274560, "latency_s": 2.139448500238359, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73318400, "prompt_kind": "short", "t_relative_s": 10089.513795082923}, {"decode_latency_s": 0.06810383594711311, "kv_live_bytes": 462274560, "latency_s": 2.1817730832844973, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73383936, "prompt_kind": "long", "t_relative_s": 10094.523898000363}, {"decode_latency_s": 0.06044171484245453, "kv_live_bytes": 462274560, "latency_s": 1.934910542331636, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73400320, "prompt_kind": "short", "t_relative_s": 10099.5339540001}, {"decode_latency_s": 0.060514287775731646, "kv_live_bytes": 462274560, "latency_s": 1.9375425837934017, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "long", "t_relative_s": 10104.544000875205}, {"decode_latency_s": 0.06033175651100464, "kv_live_bytes": 462274560, "latency_s": 1.931648749858141, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73400320, "prompt_kind": "short", "t_relative_s": 10109.554058917332}, {"decode_latency_s": 0.06840296094014775, "kv_live_bytes": 462274560, "latency_s": 2.191085583064705, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73498624, "prompt_kind": "long", "t_relative_s": 10114.564143833239}, {"decode_latency_s": 0.06066532162367366, "kv_live_bytes": 462274560, "latency_s": 1.942152834031731, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74153984, "prompt_kind": "short", "t_relative_s": 10119.574238958303}, {"decode_latency_s": 0.06778712758386973, "kv_live_bytes": 462274560, "latency_s": 2.172062083147466, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74301440, "prompt_kind": "long", "t_relative_s": 10124.584293541964}, {"decode_latency_s": 0.06652068489347585, "kv_live_bytes": 462274560, "latency_s": 2.130374124739319, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74350592, "prompt_kind": "short", "t_relative_s": 10129.594384250231}, {"decode_latency_s": 0.06805521092610434, "kv_live_bytes": 462274560, "latency_s": 2.1799665419384837, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74334208, "prompt_kind": "long", "t_relative_s": 10134.604438417125}, {"decode_latency_s": 0.06052732684474904, "kv_live_bytes": 462274560, "latency_s": 1.9377748752012849, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72335360, "prompt_kind": "short", "t_relative_s": 10139.614493333269}, {"decode_latency_s": 0.06788859114749357, "kv_live_bytes": 462274560, "latency_s": 2.1746405828744173, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73875456, "prompt_kind": "long", "t_relative_s": 10144.624553916976}, {"decode_latency_s": 0.06047223959467374, "kv_live_bytes": 462274560, "latency_s": 1.9360173749737442, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73908224, "prompt_kind": "short", "t_relative_s": 10149.634615958203}, {"decode_latency_s": 0.06780995181179605, "kv_live_bytes": 462274560, "latency_s": 2.1718086251057684, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72794112, "prompt_kind": "long", "t_relative_s": 10154.644703583326}, {"decode_latency_s": 0.06031677735154517, "kv_live_bytes": 462274560, "latency_s": 1.9311748328618705, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73498624, "prompt_kind": "short", "t_relative_s": 10159.654748833273}, {"decode_latency_s": 0.0681354778498644, "kv_live_bytes": 462274560, "latency_s": 2.182690708898008, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74104832, "prompt_kind": "long", "t_relative_s": 10164.664820167236}, {"decode_latency_s": 0.0606668059626827, "kv_live_bytes": 462274560, "latency_s": 1.9421276673674583, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74137600, "prompt_kind": "short", "t_relative_s": 10169.67490645824}, {"decode_latency_s": 0.06785089061304461, "kv_live_bytes": 462274560, "latency_s": 2.1734535829164088, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74235904, "prompt_kind": "long", "t_relative_s": 10174.684956167359}, {"decode_latency_s": 0.060379192684195004, "kv_live_bytes": 462274560, "latency_s": 1.9330131248570979, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74776576, "prompt_kind": "short", "t_relative_s": 10179.695048875175}, {"decode_latency_s": 0.06823705468559638, "kv_live_bytes": 462274560, "latency_s": 2.1848687501624227, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69746688, "prompt_kind": "long", "t_relative_s": 10184.70512333326}, {"decode_latency_s": 0.06083893097820692, "kv_live_bytes": 462274560, "latency_s": 1.947646166663617, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71499776, "prompt_kind": "short", "t_relative_s": 10189.715216042008}, {"decode_latency_s": 0.0682502421841491, "kv_live_bytes": 462274560, "latency_s": 2.1862532920204103, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73269248, "prompt_kind": "long", "t_relative_s": 10194.725270875264}, {"decode_latency_s": 0.06040840105561074, "kv_live_bytes": 462274560, "latency_s": 1.9339737920090556, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72777728, "prompt_kind": "short", "t_relative_s": 10199.735351417214}, {"decode_latency_s": 0.06835074609261937, "kv_live_bytes": 462274560, "latency_s": 2.189068084117025, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72630272, "prompt_kind": "long", "t_relative_s": 10204.745431458112}, {"decode_latency_s": 0.06045992187864613, "kv_live_bytes": 462274560, "latency_s": 1.9356533330865204, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72744960, "prompt_kind": "short", "t_relative_s": 10209.755469250027}, {"decode_latency_s": 0.06960319398785941, "kv_live_bytes": 462274560, "latency_s": 2.2291951249353588, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72220672, "prompt_kind": "long", "t_relative_s": 10214.76553758327}, {"decode_latency_s": 0.060314648450003006, "kv_live_bytes": 462274560, "latency_s": 1.930971666239202, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72859648, "prompt_kind": "short", "t_relative_s": 10219.775606708135}, {"decode_latency_s": 0.06850482942536473, "kv_live_bytes": 462274560, "latency_s": 2.1941657080315053, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73433088, "prompt_kind": "long", "t_relative_s": 10224.785692125093}, {"decode_latency_s": 0.06700727994029876, "kv_live_bytes": 462274560, "latency_s": 2.146055458113551, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73531392, "prompt_kind": "short", "t_relative_s": 10229.795750500169}, {"decode_latency_s": 0.06087461719289422, "kv_live_bytes": 462274560, "latency_s": 1.948968917131424, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73629696, "prompt_kind": "long", "t_relative_s": 10234.805828667246}, {"decode_latency_s": 0.06064003384381067, "kv_live_bytes": 462274560, "latency_s": 1.9412727090530097, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73711616, "prompt_kind": "short", "t_relative_s": 10239.815897500142}, {"decode_latency_s": 0.0681307695194846, "kv_live_bytes": 462274560, "latency_s": 2.182729249820113, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73777152, "prompt_kind": "long", "t_relative_s": 10244.818069499917}, {"decode_latency_s": 0.0603881237184396, "kv_live_bytes": 462274560, "latency_s": 1.9333896250464022, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73334784, "prompt_kind": "short", "t_relative_s": 10249.828153250273}, {"decode_latency_s": 0.06710228255542461, "kv_live_bytes": 462274560, "latency_s": 2.1489571668207645, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73859072, "prompt_kind": "long", "t_relative_s": 10254.838206707966}, {"decode_latency_s": 0.06056265103688929, "kv_live_bytes": 462274560, "latency_s": 1.9388775830157101, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73891840, "prompt_kind": "short", "t_relative_s": 10259.848252667114}, {"decode_latency_s": 0.0685386041586753, "kv_live_bytes": 462274560, "latency_s": 2.196037291083485, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74006528, "prompt_kind": "long", "t_relative_s": 10264.858354333322}, {"decode_latency_s": 0.060694529936881736, "kv_live_bytes": 462274560, "latency_s": 1.9429860417731106, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74006528, "prompt_kind": "short", "t_relative_s": 10269.868426207919}, {"decode_latency_s": 0.06826598959742114, "kv_live_bytes": 462274560, "latency_s": 2.1869723326526582, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71565312, "prompt_kind": "long", "t_relative_s": 10274.878474292345}, {"decode_latency_s": 0.06046051824523602, "kv_live_bytes": 462274560, "latency_s": 1.935670250095427, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73138176, "prompt_kind": "short", "t_relative_s": 10279.888547332957}, {"decode_latency_s": 0.06812335156428162, "kv_live_bytes": 462274560, "latency_s": 2.1814500829204917, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73646080, "prompt_kind": "long", "t_relative_s": 10284.898623333313}, {"decode_latency_s": 0.06681500261765905, "kv_live_bytes": 462274560, "latency_s": 2.1399519159458578, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73744384, "prompt_kind": "short", "t_relative_s": 10289.908687999938}, {"decode_latency_s": 0.06534415365604218, "kv_live_bytes": 462274560, "latency_s": 2.0924217076972127, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73809920, "prompt_kind": "long", "t_relative_s": 10294.91877620807}, {"decode_latency_s": 0.060560994781553745, "kv_live_bytes": 462274560, "latency_s": 1.9388237078674138, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74366976, "prompt_kind": "short", "t_relative_s": 10299.928852250334}, {"decode_latency_s": 0.06763475522166118, "kv_live_bytes": 462274560, "latency_s": 2.166310125030577, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73367552, "prompt_kind": "long", "t_relative_s": 10304.93889720831}, {"decode_latency_s": 0.06042908334347885, "kv_live_bytes": 462274560, "latency_s": 1.9346614172682166, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71647232, "prompt_kind": "short", "t_relative_s": 10309.948993749917}, {"decode_latency_s": 0.06802858856099192, "kv_live_bytes": 462274560, "latency_s": 2.1789345000870526, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74399744, "prompt_kind": "long", "t_relative_s": 10314.95907037519}, {"decode_latency_s": 0.06066759634995833, "kv_live_bytes": 462274560, "latency_s": 1.942228791769594, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74547200, "prompt_kind": "short", "t_relative_s": 10319.969121042173}, {"decode_latency_s": 0.06827258074190468, "kv_live_bytes": 462274560, "latency_s": 2.187305834144354, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74711040, "prompt_kind": "long", "t_relative_s": 10324.979208667297}, {"decode_latency_s": 0.06061697787663434, "kv_live_bytes": 462274560, "latency_s": 1.9405914582312107, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75268096, "prompt_kind": "short", "t_relative_s": 10329.989318250213}, {"decode_latency_s": 0.06797965755686164, "kv_live_bytes": 462274560, "latency_s": 2.177767042070627, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74514432, "prompt_kind": "long", "t_relative_s": 10334.99936245801}, {"decode_latency_s": 0.06023077343706973, "kv_live_bytes": 462274560, "latency_s": 1.9282993329688907, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74399744, "prompt_kind": "short", "t_relative_s": 10340.009428875055}, {"decode_latency_s": 0.06795536327990703, "kv_live_bytes": 462274560, "latency_s": 2.1763493339531124, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75038720, "prompt_kind": "long", "t_relative_s": 10345.019504250027}, {"decode_latency_s": 0.06052796743460931, "kv_live_bytes": 462274560, "latency_s": 1.937784249894321, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75137024, "prompt_kind": "short", "t_relative_s": 10350.029560707975}, {"decode_latency_s": 0.06803154687804636, "kv_live_bytes": 462274560, "latency_s": 2.1792453746311367, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75186176, "prompt_kind": "long", "t_relative_s": 10355.039657332934}, {"decode_latency_s": 0.060553192713996395, "kv_live_bytes": 462274560, "latency_s": 1.9385312502272427, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75218944, "prompt_kind": "short", "t_relative_s": 10360.049735083245}, {"decode_latency_s": 0.06789427994226571, "kv_live_bytes": 462274560, "latency_s": 2.1757045830599964, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75169792, "prompt_kind": "long", "t_relative_s": 10365.059779291973}, {"decode_latency_s": 0.06032896743272431, "kv_live_bytes": 462274560, "latency_s": 1.931438332889229, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74629120, "prompt_kind": "short", "t_relative_s": 10370.069842542056}, {"decode_latency_s": 0.06795682030497119, "kv_live_bytes": 462274560, "latency_s": 2.177188124973327, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74727424, "prompt_kind": "long", "t_relative_s": 10375.079877708107}, {"decode_latency_s": 0.060344065132085234, "kv_live_bytes": 462274560, "latency_s": 1.931783666368574, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74743808, "prompt_kind": "short", "t_relative_s": 10380.089967333246}, {"decode_latency_s": 0.060386368495528586, "kv_live_bytes": 462274560, "latency_s": 1.933236625045538, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74842112, "prompt_kind": "long", "t_relative_s": 10385.10004887497}, {"decode_latency_s": 0.060338184906868264, "kv_live_bytes": 462274560, "latency_s": 1.9317322922870517, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74727424, "prompt_kind": "short", "t_relative_s": 10390.110089791939}, {"decode_latency_s": 0.06791257031727582, "kv_live_bytes": 462274560, "latency_s": 2.1750637497752905, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74809344, "prompt_kind": "long", "t_relative_s": 10395.120166874956}, {"decode_latency_s": 0.06060425649047829, "kv_live_bytes": 462274560, "latency_s": 1.9401815831661224, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75415552, "prompt_kind": "short", "t_relative_s": 10400.130208583083}, {"decode_latency_s": 0.06033076823223382, "kv_live_bytes": 462274560, "latency_s": 1.931452624965459, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75448320, "prompt_kind": "long", "t_relative_s": 10405.140253708232}, {"decode_latency_s": 0.060391591119696386, "kv_live_bytes": 462274560, "latency_s": 1.933290584012866, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75513856, "prompt_kind": "short", "t_relative_s": 10410.150334957987}, {"decode_latency_s": 0.06810429815959651, "kv_live_bytes": 462274560, "latency_s": 2.1816272498108447, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75268096, "prompt_kind": "long", "t_relative_s": 10415.160416916944}, {"decode_latency_s": 0.06034217968408484, "kv_live_bytes": 462274560, "latency_s": 1.9318544575944543, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75350016, "prompt_kind": "short", "t_relative_s": 10420.170471041929}, {"decode_latency_s": 0.06810311197477859, "kv_live_bytes": 462274560, "latency_s": 2.18150816578418, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75251712, "prompt_kind": "long", "t_relative_s": 10425.180553167127}, {"decode_latency_s": 0.06059183203615248, "kv_live_bytes": 462274560, "latency_s": 1.939729958306998, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75415552, "prompt_kind": "short", "t_relative_s": 10430.19064937532}, {"decode_latency_s": 0.06041700261994265, "kv_live_bytes": 462274560, "latency_s": 1.9344914169050753, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75497472, "prompt_kind": "long", "t_relative_s": 10435.200705999974}, {"decode_latency_s": 0.060305476567009464, "kv_live_bytes": 462274560, "latency_s": 1.9306503748521209, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75481088, "prompt_kind": "short", "t_relative_s": 10440.210784832947}, {"decode_latency_s": 0.06815337367879692, "kv_live_bytes": 462274560, "latency_s": 2.182560999877751, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75612160, "prompt_kind": "long", "t_relative_s": 10445.220858583227}, {"decode_latency_s": 0.06056026824808214, "kv_live_bytes": 462274560, "latency_s": 1.939045624807477, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75644928, "prompt_kind": "short", "t_relative_s": 10450.230909750331}, {"decode_latency_s": 0.061370463532512076, "kv_live_bytes": 462274560, "latency_s": 1.9648185828700662, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74366976, "prompt_kind": "long", "t_relative_s": 10455.240975708235}, {"decode_latency_s": 0.060471843753475696, "kv_live_bytes": 462274560, "latency_s": 1.9360129591077566, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73252864, "prompt_kind": "short", "t_relative_s": 10460.251018125098}, {"decode_latency_s": 0.06780096614966169, "kv_live_bytes": 462274560, "latency_s": 2.171313290949911, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75464704, "prompt_kind": "long", "t_relative_s": 10465.261099667288}, {"decode_latency_s": 0.060626532562309876, "kv_live_bytes": 462274560, "latency_s": 1.940895417239517, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75513856, "prompt_kind": "short", "t_relative_s": 10470.271162583027}, {"decode_latency_s": 0.06073579429357778, "kv_live_bytes": 462274560, "latency_s": 1.9448239589110017, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75563008, "prompt_kind": "long", "t_relative_s": 10475.28120524995}, {"decode_latency_s": 0.06075315366615541, "kv_live_bytes": 462274560, "latency_s": 1.9448896669782698, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75579392, "prompt_kind": "short", "t_relative_s": 10480.291273500305}, {"decode_latency_s": 0.06506621222069953, "kv_live_bytes": 462274560, "latency_s": 2.0844864998944104, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72843264, "prompt_kind": "long", "t_relative_s": 10485.301314875018}, {"decode_latency_s": 0.06065034506900702, "kv_live_bytes": 462274560, "latency_s": 1.9416695414111018, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72859648, "prompt_kind": "short", "t_relative_s": 10490.311361625325}, {"decode_latency_s": 0.06782470182224642, "kv_live_bytes": 462274560, "latency_s": 2.172621625009924, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73269248, "prompt_kind": "long", "t_relative_s": 10495.321409041993}, {"decode_latency_s": 0.06069907812343445, "kv_live_bytes": 462274560, "latency_s": 1.9431141251698136, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74268672, "prompt_kind": "short", "t_relative_s": 10500.331497624982}, {"decode_latency_s": 0.06795527602662332, "kv_live_bytes": 462274560, "latency_s": 2.176874625030905, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73596928, "prompt_kind": "long", "t_relative_s": 10505.34154412523}, {"decode_latency_s": 0.06153184507275, "kv_live_bytes": 462274560, "latency_s": 1.96998087503016, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71811072, "prompt_kind": "short", "t_relative_s": 10510.351635125}, {"decode_latency_s": 0.060948842437937856, "kv_live_bytes": 462274560, "latency_s": 1.9513588747940958, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72400896, "prompt_kind": "long", "t_relative_s": 10515.361705916934}, {"decode_latency_s": 0.06088544272643048, "kv_live_bytes": 462274560, "latency_s": 1.9492390416562557, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73039872, "prompt_kind": "short", "t_relative_s": 10520.371755292173}, {"decode_latency_s": 0.06862470571650192, "kv_live_bytes": 462274560, "latency_s": 2.198279249947518, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74432512, "prompt_kind": "long", "t_relative_s": 10525.381835167296}, {"decode_latency_s": 0.06053874609642662, "kv_live_bytes": 462274560, "latency_s": 1.9380437079817057, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74579968, "prompt_kind": "short", "t_relative_s": 10530.391930125188}, {"decode_latency_s": 0.06805688413442113, "kv_live_bytes": 462274560, "latency_s": 2.180238999892026, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74579968, "prompt_kind": "long", "t_relative_s": 10535.399177458137}, {"decode_latency_s": 0.06029879297420848, "kv_live_bytes": 462274560, "latency_s": 1.9304949999786913, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74645504, "prompt_kind": "short", "t_relative_s": 10540.409259499982}, {"decode_latency_s": 0.06085755077947397, "kv_live_bytes": 462274560, "latency_s": 1.9484031246975064, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72679424, "prompt_kind": "long", "t_relative_s": 10545.419349292293}, {"decode_latency_s": 0.06038139325391967, "kv_live_bytes": 462274560, "latency_s": 1.9332667919807136, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72220672, "prompt_kind": "short", "t_relative_s": 10550.42939429218}, {"decode_latency_s": 0.06512330468103755, "kv_live_bytes": 462274560, "latency_s": 2.085046958178282, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73564160, "prompt_kind": "long", "t_relative_s": 10555.439484333154}, {"decode_latency_s": 0.06053167187201325, "kv_live_bytes": 462274560, "latency_s": 1.9378378330729902, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73760768, "prompt_kind": "short", "t_relative_s": 10560.449538375251}, {"decode_latency_s": 0.06099301823996939, "kv_live_bytes": 462274560, "latency_s": 1.952726375311613, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73302016, "prompt_kind": "long", "t_relative_s": 10565.459636125248}, {"decode_latency_s": 0.06069481901067775, "kv_live_bytes": 462274560, "latency_s": 1.9430565419606864, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73351168, "prompt_kind": "short", "t_relative_s": 10570.469733375125}, {"decode_latency_s": 0.06879942578962073, "kv_live_bytes": 462274560, "latency_s": 2.2038799161091447, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73318400, "prompt_kind": "long", "t_relative_s": 10575.479806667194}, {"decode_latency_s": 0.060634986992226914, "kv_live_bytes": 462274560, "latency_s": 1.941105583216995, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74088448, "prompt_kind": "short", "t_relative_s": 10580.489867125172}, {"decode_latency_s": 0.06758278905181214, "kv_live_bytes": 462274560, "latency_s": 2.1647134167142212, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74186752, "prompt_kind": "long", "t_relative_s": 10585.499918708112}, {"decode_latency_s": 0.06054729296010919, "kv_live_bytes": 462274560, "latency_s": 1.9383127917535603, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74301440, "prompt_kind": "short", "t_relative_s": 10590.509997417219}, {"decode_latency_s": 0.0683635507884901, "kv_live_bytes": 462274560, "latency_s": 2.189124791882932, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73547776, "prompt_kind": "long", "t_relative_s": 10595.520079292357}, {"decode_latency_s": 0.06031673827965278, "kv_live_bytes": 462274560, "latency_s": 1.9310847502201796, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73203712, "prompt_kind": "short", "t_relative_s": 10600.5301337922}, {"decode_latency_s": 0.06774228256836068, "kv_live_bytes": 462274560, "latency_s": 2.169726707972586, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73809920, "prompt_kind": "long", "t_relative_s": 10605.540194250178}, {"decode_latency_s": 0.06067479168996215, "kv_live_bytes": 462274560, "latency_s": 1.9424317916855216, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74416128, "prompt_kind": "short", "t_relative_s": 10610.55026937509}, {"decode_latency_s": 0.06773745053214952, "kv_live_bytes": 462274560, "latency_s": 2.1700994577258825, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74530816, "prompt_kind": "long", "t_relative_s": 10615.550335207954}, {"decode_latency_s": 0.06057756509108003, "kv_live_bytes": 462274560, "latency_s": 1.93925566598773, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74629120, "prompt_kind": "short", "t_relative_s": 10620.560418083332}, {"decode_latency_s": 0.0633227486978285, "kv_live_bytes": 462274560, "latency_s": 2.0274508339352906, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70238208, "prompt_kind": "long", "t_relative_s": 10625.568361292128}, {"decode_latency_s": 0.06025804555974901, "kv_live_bytes": 462274560, "latency_s": 1.9291251669637859, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69648384, "prompt_kind": "short", "t_relative_s": 10630.744953624904}, {"decode_latency_s": 0.06836911587743089, "kv_live_bytes": 462274560, "latency_s": 2.19016549969092, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68501504, "prompt_kind": "long", "t_relative_s": 10635.755046833307}, {"decode_latency_s": 0.06059992060181685, "kv_live_bytes": 462274560, "latency_s": 1.93999854195863, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70893568, "prompt_kind": "short", "t_relative_s": 10640.765125083271}, {"decode_latency_s": 0.06785580077848863, "kv_live_bytes": 462274560, "latency_s": 2.174058166332543, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71008256, "prompt_kind": "long", "t_relative_s": 10645.775179666933}, {"decode_latency_s": 0.060363384094671346, "kv_live_bytes": 462274560, "latency_s": 1.9324264591559768, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70991872, "prompt_kind": "short", "t_relative_s": 10650.785269750282}, {"decode_latency_s": 0.06845726825122256, "kv_live_bytes": 462274560, "latency_s": 2.1924427920021117, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69746688, "prompt_kind": "long", "t_relative_s": 10655.795338707976}, {"decode_latency_s": 0.060275714844465256, "kv_live_bytes": 462274560, "latency_s": 1.9297836660407484, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69992448, "prompt_kind": "short", "t_relative_s": 10660.805383542087}, {"decode_latency_s": 0.06839506250980776, "kv_live_bytes": 462274560, "latency_s": 2.1911537502892315, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70402048, "prompt_kind": "long", "t_relative_s": 10665.815473542083}, {"decode_latency_s": 0.06070330990769435, "kv_live_bytes": 462274560, "latency_s": 1.943388540763408, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71450624, "prompt_kind": "short", "t_relative_s": 10670.825557083357}, {"decode_latency_s": 0.06045887632353697, "kv_live_bytes": 462274560, "latency_s": 1.935793333221227, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71467008, "prompt_kind": "long", "t_relative_s": 10675.835598292295}, {"decode_latency_s": 0.06047240625775885, "kv_live_bytes": 462274560, "latency_s": 1.9358734576962888, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71532544, "prompt_kind": "short", "t_relative_s": 10680.84567595832}, {"decode_latency_s": 0.068130494793877, "kv_live_bytes": 462274560, "latency_s": 2.1817719168029726, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70926336, "prompt_kind": "long", "t_relative_s": 10685.855751583353}, {"decode_latency_s": 0.06038058853300754, "kv_live_bytes": 462274560, "latency_s": 1.9330730000510812, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70713344, "prompt_kind": "short", "t_relative_s": 10690.86334716715}, {"decode_latency_s": 0.06846554687945172, "kv_live_bytes": 462274560, "latency_s": 2.1928387503139675, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70926336, "prompt_kind": "long", "t_relative_s": 10695.873417958152}, {"decode_latency_s": 0.06575274090573657, "kv_live_bytes": 462274560, "latency_s": 2.1050750422291458, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66404352, "prompt_kind": "short", "t_relative_s": 10700.883460625075}, {"decode_latency_s": 0.06791072527994402, "kv_live_bytes": 462274560, "latency_s": 2.1750900419428945, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68943872, "prompt_kind": "long", "t_relative_s": 10705.89355041692}, {"decode_latency_s": 0.06076608202420175, "kv_live_bytes": 462274560, "latency_s": 1.945280082989484, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69599232, "prompt_kind": "short", "t_relative_s": 10710.903602542356}, {"decode_latency_s": 0.060788446629885584, "kv_live_bytes": 462274560, "latency_s": 1.9463212909176946, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69763072, "prompt_kind": "long", "t_relative_s": 10715.91365183331}, {"decode_latency_s": 0.06062146743352059, "kv_live_bytes": 462274560, "latency_s": 1.9407146251760423, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70303744, "prompt_kind": "short", "t_relative_s": 10720.923733042087}, {"decode_latency_s": 0.06828571484948043, "kv_live_bytes": 462274560, "latency_s": 2.187423000112176, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69533696, "prompt_kind": "long", "t_relative_s": 10725.93385308329}, {"decode_latency_s": 0.06081932420784142, "kv_live_bytes": 462274560, "latency_s": 1.9469832917675376, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69484544, "prompt_kind": "short", "t_relative_s": 10730.943976708222}, {"decode_latency_s": 0.06799333203525748, "kv_live_bytes": 462274560, "latency_s": 2.1780312499031425, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70008832, "prompt_kind": "long", "t_relative_s": 10735.954032917041}, {"decode_latency_s": 0.06039175781188533, "kv_live_bytes": 462274560, "latency_s": 1.9333679592236876, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70074368, "prompt_kind": "short", "t_relative_s": 10740.964155917056}, {"decode_latency_s": 0.06843472266336903, "kv_live_bytes": 462274560, "latency_s": 2.1913966247811913, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68976640, "prompt_kind": "long", "t_relative_s": 10745.974231041968}, {"decode_latency_s": 0.06715711067954544, "kv_live_bytes": 462274560, "latency_s": 2.1508987080305815, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69943296, "prompt_kind": "short", "t_relative_s": 10750.984329583123}, {"decode_latency_s": 0.06902291536971461, "kv_live_bytes": 462274560, "latency_s": 2.210654624737799, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69042176, "prompt_kind": "long", "t_relative_s": 10755.994407332968}, {"decode_latency_s": 0.06042542187788058, "kv_live_bytes": 462274560, "latency_s": 1.9344534161500633, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68698112, "prompt_kind": "short", "t_relative_s": 10761.004470292013}, {"decode_latency_s": 0.06818880340142641, "kv_live_bytes": 462274560, "latency_s": 2.183777166996151, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70434816, "prompt_kind": "long", "t_relative_s": 10766.014551333152}, {"decode_latency_s": 0.06049271352821961, "kv_live_bytes": 462274560, "latency_s": 1.93659762525931, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70500352, "prompt_kind": "short", "t_relative_s": 10771.015064458363}, {"decode_latency_s": 0.06819852734042797, "kv_live_bytes": 462274560, "latency_s": 2.1849105418659747, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70615040, "prompt_kind": "long", "t_relative_s": 10776.02514112508}, {"decode_latency_s": 0.06066282684332691, "kv_live_bytes": 462274560, "latency_s": 1.9419785412028432, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71139328, "prompt_kind": "short", "t_relative_s": 10781.035255458206}, {"decode_latency_s": 0.0697372981812805, "kv_live_bytes": 462274560, "latency_s": 2.2336712921969593, "n_emitted": 32, "ok": true, "process_footprint_bytes": 65552384, "prompt_kind": "long", "t_relative_s": 10786.045301917009}, {"decode_latency_s": 0.06656075263163075, "kv_live_bytes": 462274560, "latency_s": 2.1317875417880714, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68485120, "prompt_kind": "short", "t_relative_s": 10791.054444000125}, {"decode_latency_s": 0.06803742967895232, "kv_live_bytes": 462274560, "latency_s": 2.1795164169743657, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69943296, "prompt_kind": "long", "t_relative_s": 10796.06452295836}, {"decode_latency_s": 0.06104636850068346, "kv_live_bytes": 462274560, "latency_s": 1.954294040799141, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69517312, "prompt_kind": "short", "t_relative_s": 10801.074597250205}, {"decode_latency_s": 0.06938776302558836, "kv_live_bytes": 462274560, "latency_s": 2.2218194170854986, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70205440, "prompt_kind": "long", "t_relative_s": 10806.084641917143}, {"decode_latency_s": 0.06039639453229029, "kv_live_bytes": 462274560, "latency_s": 1.9336273749358952, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70729728, "prompt_kind": "short", "t_relative_s": 10811.094723958056}, {"decode_latency_s": 0.06802148959832266, "kv_live_bytes": 462274560, "latency_s": 2.178303875029087, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69795840, "prompt_kind": "long", "t_relative_s": 10816.104796583299}, {"decode_latency_s": 0.060445933588198386, "kv_live_bytes": 462274560, "latency_s": 1.9351733326911926, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70254592, "prompt_kind": "short", "t_relative_s": 10821.11484841723}, {"decode_latency_s": 0.06812234893732239, "kv_live_bytes": 462274560, "latency_s": 2.1818613749928772, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71221248, "prompt_kind": "long", "t_relative_s": 10826.124930292368}, {"decode_latency_s": 0.06308216533216182, "kv_live_bytes": 462274560, "latency_s": 2.019558749627322, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68009984, "prompt_kind": "short", "t_relative_s": 10831.126189458184}, {"decode_latency_s": 0.0681605703139212, "kv_live_bytes": 462274560, "latency_s": 2.1838855827227235, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68698112, "prompt_kind": "long", "t_relative_s": 10836.13624329213}, {"decode_latency_s": 0.060423722650739364, "kv_live_bytes": 462274560, "latency_s": 1.934425333980471, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69222400, "prompt_kind": "short", "t_relative_s": 10841.146299083252}, {"decode_latency_s": 0.0683020625001518, "kv_live_bytes": 462274560, "latency_s": 2.1872572917491198, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69255168, "prompt_kind": "long", "t_relative_s": 10846.155300583225}, {"decode_latency_s": 0.060858768250909634, "kv_live_bytes": 462274560, "latency_s": 1.9483586251735687, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69058560, "prompt_kind": "short", "t_relative_s": 10851.16534562502}, {"decode_latency_s": 0.06769912369782105, "kv_live_bytes": 462274560, "latency_s": 2.1680376660078764, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69337088, "prompt_kind": "long", "t_relative_s": 10856.175419458188}, {"decode_latency_s": 0.060871300782309845, "kv_live_bytes": 462274560, "latency_s": 1.9486858327873051, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69320704, "prompt_kind": "short", "t_relative_s": 10861.185479958076}, {"decode_latency_s": 0.06820842968591023, "kv_live_bytes": 462274560, "latency_s": 2.184758000075817, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69304320, "prompt_kind": "long", "t_relative_s": 10866.195532416925}, {"decode_latency_s": 0.06046143748972099, "kv_live_bytes": 462274560, "latency_s": 1.9356518746353686, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69402624, "prompt_kind": "short", "t_relative_s": 10871.20559104206}, {"decode_latency_s": 0.06795451694051735, "kv_live_bytes": 462274560, "latency_s": 2.1774445828050375, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68222976, "prompt_kind": "long", "t_relative_s": 10876.215649208054}, {"decode_latency_s": 0.060625494792475365, "kv_live_bytes": 462274560, "latency_s": 1.9407740831375122, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70402048, "prompt_kind": "short", "t_relative_s": 10881.225722458214}, {"decode_latency_s": 0.06811770831700414, "kv_live_bytes": 462274560, "latency_s": 2.1813135417178273, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69926912, "prompt_kind": "long", "t_relative_s": 10886.235800249968}, {"decode_latency_s": 0.06034467059362214, "kv_live_bytes": 462274560, "latency_s": 1.9319630828686059, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69992448, "prompt_kind": "short", "t_relative_s": 10891.245855417103}, {"decode_latency_s": 0.0684455676964717, "kv_live_bytes": 462274560, "latency_s": 2.192241624929011, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70107136, "prompt_kind": "long", "t_relative_s": 10896.255932417233}, {"decode_latency_s": 0.06070457422174513, "kv_live_bytes": 462274560, "latency_s": 1.9434061250649393, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70647808, "prompt_kind": "short", "t_relative_s": 10901.263328208122}, {"decode_latency_s": 0.06837624477338977, "kv_live_bytes": 462274560, "latency_s": 2.1908719167113304, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69943296, "prompt_kind": "long", "t_relative_s": 10906.273371417075}, {"decode_latency_s": 0.060597799470997415, "kv_live_bytes": 462274560, "latency_s": 1.9399238750338554, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70582272, "prompt_kind": "short", "t_relative_s": 10911.283434667159}, {"decode_latency_s": 0.06782299480983056, "kv_live_bytes": 462274560, "latency_s": 2.1726435418240726, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71122944, "prompt_kind": "long", "t_relative_s": 10916.293478208128}, {"decode_latency_s": 0.0603042421862483, "kv_live_bytes": 462274560, "latency_s": 1.9306375002488494, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70647808, "prompt_kind": "short", "t_relative_s": 10921.303507000208}, {"decode_latency_s": 0.06861032293818425, "kv_live_bytes": 462274560, "latency_s": 2.1976582501083612, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71286784, "prompt_kind": "long", "t_relative_s": 10926.313588958234}, {"decode_latency_s": 0.06078628252726048, "kv_live_bytes": 462274560, "latency_s": 1.9460223326459527, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71860224, "prompt_kind": "short", "t_relative_s": 10931.323646957986}, {"decode_latency_s": 0.06819014974462334, "kv_live_bytes": 462274560, "latency_s": 2.1846148329786956, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71286784, "prompt_kind": "long", "t_relative_s": 10936.333702375181}, {"decode_latency_s": 0.06323646356759127, "kv_live_bytes": 462274560, "latency_s": 2.024502709042281, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71008256, "prompt_kind": "short", "t_relative_s": 10941.34377537528}, {"decode_latency_s": 0.06770804297411814, "kv_live_bytes": 462274560, "latency_s": 2.169210707768798, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71942144, "prompt_kind": "long", "t_relative_s": 10946.353842624929}, {"decode_latency_s": 0.06054865365149453, "kv_live_bytes": 462274560, "latency_s": 1.938385207671672, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72040448, "prompt_kind": "short", "t_relative_s": 10951.363997499924}, {"decode_latency_s": 0.06781295961991418, "kv_live_bytes": 462274560, "latency_s": 2.172359958756715, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72122368, "prompt_kind": "long", "t_relative_s": 10956.37404050026}, {"decode_latency_s": 0.060651536463410594, "kv_live_bytes": 462274560, "latency_s": 1.9417556668631732, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71974912, "prompt_kind": "short", "t_relative_s": 10961.384115917142}, {"decode_latency_s": 0.06877186066412833, "kv_live_bytes": 462274560, "latency_s": 2.2023321660235524, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74743808, "prompt_kind": "long", "t_relative_s": 10966.394191583153}, {"decode_latency_s": 0.06061430991394445, "kv_live_bytes": 462274560, "latency_s": 1.940600958187133, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75448320, "prompt_kind": "short", "t_relative_s": 10971.404247042257}, {"decode_latency_s": 0.06803558465617243, "kv_live_bytes": 462274560, "latency_s": 2.178928917273879, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76546048, "prompt_kind": "long", "t_relative_s": 10976.41431783326}, {"decode_latency_s": 0.060734454440535046, "kv_live_bytes": 462274560, "latency_s": 1.9443988339044154, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76595200, "prompt_kind": "short", "t_relative_s": 10981.424406958278}, {"decode_latency_s": 0.06810049217892811, "kv_live_bytes": 462274560, "latency_s": 2.1815372919663787, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76513280, "prompt_kind": "long", "t_relative_s": 10986.434459500015}, {"decode_latency_s": 0.060435894527472556, "kv_live_bytes": 462274560, "latency_s": 1.9346932917833328, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77053952, "prompt_kind": "short", "t_relative_s": 10991.444530792069}, {"decode_latency_s": 0.06861910028965212, "kv_live_bytes": 462274560, "latency_s": 2.1978227919898927, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74432512, "prompt_kind": "long", "t_relative_s": 10996.454633791931}, {"decode_latency_s": 0.060763601562939584, "kv_live_bytes": 462274560, "latency_s": 1.9451976660639048, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76627968, "prompt_kind": "short", "t_relative_s": 11001.464723750018}, {"decode_latency_s": 0.06778998437221162, "kv_live_bytes": 462274560, "latency_s": 2.171857957728207, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76791808, "prompt_kind": "long", "t_relative_s": 11006.474778250325}, {"decode_latency_s": 0.06031496745708864, "kv_live_bytes": 462274560, "latency_s": 1.9309516670182347, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76611584, "prompt_kind": "short", "t_relative_s": 11011.484842792153}, {"decode_latency_s": 0.06830421225458849, "kv_live_bytes": 462274560, "latency_s": 2.18746625026688, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76677120, "prompt_kind": "long", "t_relative_s": 11016.494918208104}, {"decode_latency_s": 0.06714751954132225, "kv_live_bytes": 462274560, "latency_s": 2.150457375217229, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77316096, "prompt_kind": "short", "t_relative_s": 11021.504972583149}, {"decode_latency_s": 0.06795894012611825, "kv_live_bytes": 462274560, "latency_s": 2.177737249992788, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77217792, "prompt_kind": "long", "t_relative_s": 11026.515055292286}, {"decode_latency_s": 0.06719486849033274, "kv_live_bytes": 462274560, "latency_s": 2.151953165885061, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77381632, "prompt_kind": "short", "t_relative_s": 11031.525125416927}, {"decode_latency_s": 0.06764565233606845, "kv_live_bytes": 462274560, "latency_s": 2.1669547921046615, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77185024, "prompt_kind": "long", "t_relative_s": 11036.535193417221}, {"decode_latency_s": 0.06047145182674285, "kv_live_bytes": 462274560, "latency_s": 1.9359916248358786, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76840960, "prompt_kind": "short", "t_relative_s": 11041.545250541996}, {"decode_latency_s": 0.06831881508696824, "kv_live_bytes": 462274560, "latency_s": 2.187838749960065, "n_emitted": 32, "ok": true, "process_footprint_bytes": 76873728, "prompt_kind": "long", "t_relative_s": 11046.555331707932}, {"decode_latency_s": 0.060657734371488914, "kv_live_bytes": 462274560, "latency_s": 1.9419071250595152, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77414400, "prompt_kind": "short", "t_relative_s": 11051.565401874948}, {"decode_latency_s": 0.06794698047451675, "kv_live_bytes": 462274560, "latency_s": 2.1766339587047696, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77479936, "prompt_kind": "long", "t_relative_s": 11056.575471375138}, {"decode_latency_s": 0.06070974610338453, "kv_live_bytes": 462274560, "latency_s": 1.9435141254216433, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78069760, "prompt_kind": "short", "t_relative_s": 11061.585554417223}, {"decode_latency_s": 0.06802535025053658, "kv_live_bytes": 462274560, "latency_s": 2.178896375000477, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78086144, "prompt_kind": "long", "t_relative_s": 11066.595603791997}, {"decode_latency_s": 0.06047835024946835, "kv_live_bytes": 462274560, "latency_s": 1.936181957833469, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74694656, "prompt_kind": "short", "t_relative_s": 11071.605656458065}, {"decode_latency_s": 0.06803367055545095, "kv_live_bytes": 462274560, "latency_s": 2.178952833171934, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74711040, "prompt_kind": "long", "t_relative_s": 11076.615741458256}, {"decode_latency_s": 0.06623783333634492, "kv_live_bytes": 462274560, "latency_s": 2.1212582918815315, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74694656, "prompt_kind": "short", "t_relative_s": 11081.625806625001}, {"decode_latency_s": 0.06511213150224648, "kv_live_bytes": 462274560, "latency_s": 2.085017249919474, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67796992, "prompt_kind": "long", "t_relative_s": 11086.635870042257}, {"decode_latency_s": 0.06727492705977056, "kv_live_bytes": 462274560, "latency_s": 2.1548832920379937, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71614464, "prompt_kind": "short", "t_relative_s": 11091.645938416943}, {"decode_latency_s": 0.06806653515377548, "kv_live_bytes": 462274560, "latency_s": 2.1796882078051567, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70860800, "prompt_kind": "long", "t_relative_s": 11096.653444041964}, {"decode_latency_s": 0.06122893486462999, "kv_live_bytes": 462274560, "latency_s": 1.9601367502473295, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71680000, "prompt_kind": "short", "t_relative_s": 11101.663553917315}, {"decode_latency_s": 0.0679228645603871, "kv_live_bytes": 462274560, "latency_s": 2.1757516246289015, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72253440, "prompt_kind": "long", "t_relative_s": 11106.67362358328}, {"decode_latency_s": 0.060467307281214744, "kv_live_bytes": 462274560, "latency_s": 1.9358437079936266, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72777728, "prompt_kind": "short", "t_relative_s": 11111.683705416974}, {"decode_latency_s": 0.06803236978885252, "kv_live_bytes": 462274560, "latency_s": 2.17867024987936, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72990720, "prompt_kind": "long", "t_relative_s": 11116.693785833195}, {"decode_latency_s": 0.06711052864557132, "kv_live_bytes": 462274560, "latency_s": 2.1491990839131176, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73564160, "prompt_kind": "short", "t_relative_s": 11121.699826333206}, {"decode_latency_s": 0.06042030337266624, "kv_live_bytes": 462274560, "latency_s": 1.934425000101328, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73580544, "prompt_kind": "long", "t_relative_s": 11126.709908000194}, {"decode_latency_s": 0.06063891795929521, "kv_live_bytes": 462274560, "latency_s": 1.9412491251714528, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73629696, "prompt_kind": "short", "t_relative_s": 11131.719999332912}, {"decode_latency_s": 0.060477029939647764, "kv_live_bytes": 462274560, "latency_s": 1.9365155412815511, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73678848, "prompt_kind": "long", "t_relative_s": 11136.730051750317}, {"decode_latency_s": 0.06072484374453779, "kv_live_bytes": 462274560, "latency_s": 1.944124750327319, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73629696, "prompt_kind": "short", "t_relative_s": 11141.740130042192}, {"decode_latency_s": 0.06074837368214503, "kv_live_bytes": 462274560, "latency_s": 1.9455770421773195, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72990720, "prompt_kind": "long", "t_relative_s": 11146.750185417011}, {"decode_latency_s": 0.170569811220048, "kv_live_bytes": 462274560, "latency_s": 5.459243708755821, "n_emitted": 32, "ok": true, "process_footprint_bytes": 65159168, "prompt_kind": "short", "t_relative_s": 11151.760267667007}, {"decode_latency_s": 0.06750672655471135, "kv_live_bytes": 462274560, "latency_s": 2.1622861670330167, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68042752, "prompt_kind": "long", "t_relative_s": 11161.147078332957}, {"decode_latency_s": 0.06053491927741561, "kv_live_bytes": 462274560, "latency_s": 1.9379984168335795, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68763648, "prompt_kind": "short", "t_relative_s": 11166.157131500077}, {"decode_latency_s": 0.06784219140536152, "kv_live_bytes": 462274560, "latency_s": 2.1729509588330984, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69058560, "prompt_kind": "long", "t_relative_s": 11171.16721162526}, {"decode_latency_s": 0.06092326431826223, "kv_live_bytes": 462274560, "latency_s": 1.950398666318506, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69599232, "prompt_kind": "short", "t_relative_s": 11176.177275999915}, {"decode_latency_s": 0.06782219267915934, "kv_live_bytes": 462274560, "latency_s": 2.1724945418536663, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69779456, "prompt_kind": "long", "t_relative_s": 11181.187322708312}, {"decode_latency_s": 0.060617967435973696, "kv_live_bytes": 462274560, "latency_s": 1.9405817077495158, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70303744, "prompt_kind": "short", "t_relative_s": 11186.197400792036}, {"decode_latency_s": 0.06777217319176998, "kv_live_bytes": 462274560, "latency_s": 2.1706595830619335, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70369280, "prompt_kind": "long", "t_relative_s": 11191.207483208273}, {"decode_latency_s": 0.06053897656966001, "kv_live_bytes": 462274560, "latency_s": 1.938134958036244, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70434816, "prompt_kind": "short", "t_relative_s": 11196.21754045831}, {"decode_latency_s": 0.06832458724966273, "kv_live_bytes": 462274560, "latency_s": 2.1884665829129517, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70582272, "prompt_kind": "long", "t_relative_s": 11201.22762400005}, {"decode_latency_s": 0.06066759375971742, "kv_live_bytes": 462274560, "latency_s": 1.9421505411155522, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71188480, "prompt_kind": "short", "t_relative_s": 11206.237707042135}, {"decode_latency_s": 0.06789865625614766, "kv_live_bytes": 462274560, "latency_s": 2.1752792918123305, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71286784, "prompt_kind": "long", "t_relative_s": 11211.247758542188}, {"decode_latency_s": 0.06046705337939784, "kv_live_bytes": 462274560, "latency_s": 1.935704582836479, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71188480, "prompt_kind": "short", "t_relative_s": 11216.257821250241}, {"decode_latency_s": 0.060492889329907484, "kv_live_bytes": 462274560, "latency_s": 1.9366689161397517, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71286784, "prompt_kind": "long", "t_relative_s": 11221.267899292056}, {"decode_latency_s": 0.06043507684080396, "kv_live_bytes": 462274560, "latency_s": 1.9348056665621698, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70680576, "prompt_kind": "short", "t_relative_s": 11226.277951792348}, {"decode_latency_s": 0.06764444531290792, "kv_live_bytes": 462274560, "latency_s": 2.1672033751383424, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70844416, "prompt_kind": "long", "t_relative_s": 11231.288036042359}, {"decode_latency_s": 0.060751296870876104, "kv_live_bytes": 462274560, "latency_s": 1.9448669161647558, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71385088, "prompt_kind": "short", "t_relative_s": 11236.298135333229}, {"decode_latency_s": 0.06113083071250003, "kv_live_bytes": 462274560, "latency_s": 1.95757225016132, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71467008, "prompt_kind": "long", "t_relative_s": 11241.30819116719}, {"decode_latency_s": 0.06052170571638271, "kv_live_bytes": 462274560, "latency_s": 1.937572750262916, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71909376, "prompt_kind": "short", "t_relative_s": 11246.318269125186}, {"decode_latency_s": 0.06806059893278871, "kv_live_bytes": 462274560, "latency_s": 2.1799620841629803, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71876608, "prompt_kind": "long", "t_relative_s": 11251.328346417286}, {"decode_latency_s": 0.06060806510504335, "kv_live_bytes": 462274560, "latency_s": 1.940388583112508, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71942144, "prompt_kind": "short", "t_relative_s": 11256.338402042165}, {"decode_latency_s": 0.06088859765441157, "kv_live_bytes": 462274560, "latency_s": 1.9499179162085056, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71925760, "prompt_kind": "long", "t_relative_s": 11261.348473333288}, {"decode_latency_s": 0.060706278658472, "kv_live_bytes": 462274560, "latency_s": 1.9434062498621643, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71991296, "prompt_kind": "short", "t_relative_s": 11266.35854850011}, {"decode_latency_s": 0.06811907941300888, "kv_live_bytes": 462274560, "latency_s": 2.1822989997453988, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72040448, "prompt_kind": "long", "t_relative_s": 11271.368592625018}, {"decode_latency_s": 0.060466631533927284, "kv_live_bytes": 462274560, "latency_s": 1.9359042919240892, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72597504, "prompt_kind": "short", "t_relative_s": 11276.378676542081}, {"decode_latency_s": 0.06750307031325065, "kv_live_bytes": 462274560, "latency_s": 2.1616192501969635, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72024064, "prompt_kind": "long", "t_relative_s": 11281.388749583159}, {"decode_latency_s": 0.06708205991890281, "kv_live_bytes": 462274560, "latency_s": 2.148596915882081, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72138752, "prompt_kind": "short", "t_relative_s": 11286.398805249948}, {"decode_latency_s": 0.06817887758370489, "kv_live_bytes": 462274560, "latency_s": 2.184098625089973, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72253440, "prompt_kind": "long", "t_relative_s": 11291.40887466725}, {"decode_latency_s": 0.06074797917972319, "kv_live_bytes": 462274560, "latency_s": 1.9447245839983225, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72826880, "prompt_kind": "short", "t_relative_s": 11296.418960625306}, {"decode_latency_s": 0.0677998723986093, "kv_live_bytes": 462274560, "latency_s": 2.1717922911047935, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72908800, "prompt_kind": "long", "t_relative_s": 11301.429004082922}, {"decode_latency_s": 0.06036296093952842, "kv_live_bytes": 462274560, "latency_s": 1.9324996657669544, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73416704, "prompt_kind": "short", "t_relative_s": 11306.439082249999}, {"decode_latency_s": 0.06809398178302217, "kv_live_bytes": 462274560, "latency_s": 2.181131082586944, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73383936, "prompt_kind": "long", "t_relative_s": 11311.449180708267}, {"decode_latency_s": 0.060863350285217166, "kv_live_bytes": 462274560, "latency_s": 1.948502709157765, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73531392, "prompt_kind": "short", "t_relative_s": 11316.459249667358}, {"decode_latency_s": 0.06808765497407876, "kv_live_bytes": 462274560, "latency_s": 2.1812752918340266, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73646080, "prompt_kind": "long", "t_relative_s": 11321.469316666946}, {"decode_latency_s": 0.0667248789104633, "kv_live_bytes": 462274560, "latency_s": 2.1369757913053036, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74268672, "prompt_kind": "short", "t_relative_s": 11326.479246250354}, {"decode_latency_s": 0.068289389309939, "kv_live_bytes": 462274560, "latency_s": 2.187741417437792, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74317824, "prompt_kind": "long", "t_relative_s": 11331.489298083354}, {"decode_latency_s": 0.060257869787164964, "kv_live_bytes": 462274560, "latency_s": 1.929121000226587, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74088448, "prompt_kind": "short", "t_relative_s": 11336.499375124928}, {"decode_latency_s": 0.06796756903349888, "kv_live_bytes": 462274560, "latency_s": 2.1770233749412, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74121216, "prompt_kind": "long", "t_relative_s": 11341.509465833195}, {"decode_latency_s": 0.06073621223913506, "kv_live_bytes": 462274560, "latency_s": 1.944390666205436, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74219520, "prompt_kind": "short", "t_relative_s": 11346.519517833367}, {"decode_latency_s": 0.06803769399994053, "kv_live_bytes": 462274560, "latency_s": 2.17961816675961, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74268672, "prompt_kind": "long", "t_relative_s": 11351.529594458174}, {"decode_latency_s": 0.060829770838608965, "kv_live_bytes": 462274560, "latency_s": 1.9473370420746505, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73957376, "prompt_kind": "short", "t_relative_s": 11356.539678083267}, {"decode_latency_s": 0.06759347135084681, "kv_live_bytes": 462274560, "latency_s": 2.165106583852321, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73891840, "prompt_kind": "long", "t_relative_s": 11361.549727207981}, {"decode_latency_s": 0.06035843618155923, "kv_live_bytes": 462274560, "latency_s": 1.9324307087808847, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73924608, "prompt_kind": "short", "t_relative_s": 11366.559794625267}, {"decode_latency_s": 0.06827624086872675, "kv_live_bytes": 462274560, "latency_s": 2.1865018331445754, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74498048, "prompt_kind": "long", "t_relative_s": 11371.56986925006}, {"decode_latency_s": 0.06094162500812672, "kv_live_bytes": 462274560, "latency_s": 1.9509980827569962, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75890688, "prompt_kind": "short", "t_relative_s": 11376.579934583046}, {"decode_latency_s": 0.06807808073062915, "kv_live_bytes": 462274560, "latency_s": 2.1810030420310795, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75939840, "prompt_kind": "long", "t_relative_s": 11381.590024583042}, {"decode_latency_s": 0.06610247524804436, "kv_live_bytes": 462274560, "latency_s": 2.1161501253955066, "n_emitted": 32, "ok": true, "process_footprint_bytes": 49954816, "prompt_kind": "short", "t_relative_s": 11386.600103000179}, {"decode_latency_s": 0.06854924609069712, "kv_live_bytes": 462274560, "latency_s": 2.1955764167942107, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70991872, "prompt_kind": "long", "t_relative_s": 11391.610179125331}, {"decode_latency_s": 0.0660105937422486, "kv_live_bytes": 462274560, "latency_s": 2.114098208025098, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73318400, "prompt_kind": "short", "t_relative_s": 11396.620236250106}, {"decode_latency_s": 0.06770689190307166, "kv_live_bytes": 462274560, "latency_s": 2.168703665956855, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72925184, "prompt_kind": "long", "t_relative_s": 11401.630354083143}, {"decode_latency_s": 0.061066167982062325, "kv_live_bytes": 462274560, "latency_s": 1.9549725828692317, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73023488, "prompt_kind": "short", "t_relative_s": 11406.640474624932}, {"decode_latency_s": 0.06781505727849435, "kv_live_bytes": 462274560, "latency_s": 2.1725187497213483, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73187328, "prompt_kind": "long", "t_relative_s": 11411.65051229205}, {"decode_latency_s": 0.06274014321388677, "kv_live_bytes": 462274560, "latency_s": 2.0085462909191847, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73154560, "prompt_kind": "short", "t_relative_s": 11416.660569333006}, {"decode_latency_s": 0.06788562110159546, "kv_live_bytes": 462274560, "latency_s": 2.1748331249691546, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73744384, "prompt_kind": "long", "t_relative_s": 11421.670671250205}, {"decode_latency_s": 0.06083989713806659, "kv_live_bytes": 462274560, "latency_s": 1.9477647920139134, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74366976, "prompt_kind": "short", "t_relative_s": 11426.67469183309}, {"decode_latency_s": 0.06767979946744163, "kv_live_bytes": 462274560, "latency_s": 2.1679427078925073, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74498048, "prompt_kind": "long", "t_relative_s": 11431.684752332978}, {"decode_latency_s": 0.06060454164980911, "kv_live_bytes": 462274560, "latency_s": 1.9402754590846598, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74612736, "prompt_kind": "short", "t_relative_s": 11436.694841207936}, {"decode_latency_s": 0.06086032549501397, "kv_live_bytes": 462274560, "latency_s": 1.9488054169341922, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71073792, "prompt_kind": "long", "t_relative_s": 11441.704931292217}, {"decode_latency_s": 0.06059171483502723, "kv_live_bytes": 462274560, "latency_s": 1.9397932076826692, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73515008, "prompt_kind": "short", "t_relative_s": 11446.71501383325}, {"decode_latency_s": 0.06700419008848257, "kv_live_bytes": 462274560, "latency_s": 2.1465864591300488, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74153984, "prompt_kind": "long", "t_relative_s": 11451.725059458055}, {"decode_latency_s": 0.06107965233968571, "kv_live_bytes": 462274560, "latency_s": 1.9553923751227558, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74711040, "prompt_kind": "short", "t_relative_s": 11456.735157333314}, {"decode_latency_s": 0.0608053671894595, "kv_live_bytes": 462274560, "latency_s": 1.9467642079107463, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71958528, "prompt_kind": "long", "t_relative_s": 11461.745208083186}, {"decode_latency_s": 0.06065760419005528, "kv_live_bytes": 462274560, "latency_s": 1.9417944164015353, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72040448, "prompt_kind": "short", "t_relative_s": 11466.755292583257}, {"decode_latency_s": 0.06848187498690095, "kv_live_bytes": 462274560, "latency_s": 2.1931579592637718, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73940992, "prompt_kind": "long", "t_relative_s": 11471.765328875277}, {"decode_latency_s": 0.06065470834437292, "kv_live_bytes": 462274560, "latency_s": 1.9418955422006547, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73302016, "prompt_kind": "short", "t_relative_s": 11476.775386041962}, {"decode_latency_s": 0.06848780209838878, "kv_live_bytes": 462274560, "latency_s": 2.196363208349794, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74055680, "prompt_kind": "long", "t_relative_s": 11481.785434625112}, {"decode_latency_s": 0.060400825503165834, "kv_live_bytes": 462274560, "latency_s": 1.9336962075904012, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74711040, "prompt_kind": "short", "t_relative_s": 11486.795533375349}, {"decode_latency_s": 0.06847948305949103, "kv_live_bytes": 462274560, "latency_s": 2.192854166030884, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68190208, "prompt_kind": "long", "t_relative_s": 11491.805594542064}, {"decode_latency_s": 0.06093864844297059, "kv_live_bytes": 462274560, "latency_s": 1.9509719996713102, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72843264, "prompt_kind": "short", "t_relative_s": 11496.815671250224}, {"decode_latency_s": 0.06859646743396297, "kv_live_bytes": 462274560, "latency_s": 2.1972457910887897, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74022912, "prompt_kind": "long", "t_relative_s": 11501.82571220817}, {"decode_latency_s": 0.0606056536489632, "kv_live_bytes": 462274560, "latency_s": 1.940283834002912, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73269248, "prompt_kind": "short", "t_relative_s": 11506.835790750105}, {"decode_latency_s": 0.06837882031686604, "kv_live_bytes": 462274560, "latency_s": 2.1890505412593484, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88473600, "prompt_kind": "long", "t_relative_s": 11511.845865542069}, {"decode_latency_s": 0.061301713532884605, "kv_live_bytes": 462274560, "latency_s": 1.9625964588485658, "n_emitted": 32, "ok": true, "process_footprint_bytes": 87048192, "prompt_kind": "short", "t_relative_s": 11516.855916792061}, {"decode_latency_s": 0.06780525262001902, "kv_live_bytes": 462274560, "latency_s": 2.171940708067268, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90554368, "prompt_kind": "long", "t_relative_s": 11521.866012917366}, {"decode_latency_s": 0.06063878643908538, "kv_live_bytes": 462274560, "latency_s": 1.9412225410342216, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90652672, "prompt_kind": "short", "t_relative_s": 11526.876074292231}, {"decode_latency_s": 0.06814776040846482, "kv_live_bytes": 462274560, "latency_s": 2.1830286672338843, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90652672, "prompt_kind": "long", "t_relative_s": 11531.88613395812}, {"decode_latency_s": 0.0629502838710323, "kv_live_bytes": 462274560, "latency_s": 2.0153394169174135, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90275840, "prompt_kind": "short", "t_relative_s": 11536.896231375169}, {"decode_latency_s": 0.0680539895693073, "kv_live_bytes": 462274560, "latency_s": 2.1801775419153273, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90652672, "prompt_kind": "long", "t_relative_s": 11541.90632366715}, {"decode_latency_s": 0.06096551561495289, "kv_live_bytes": 462274560, "latency_s": 1.951694083865732, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91258880, "prompt_kind": "short", "t_relative_s": 11546.906752957962}, {"decode_latency_s": 0.060384703130694106, "kv_live_bytes": 462274560, "latency_s": 1.9334808751009405, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74219520, "prompt_kind": "long", "t_relative_s": 11551.916797833052}, {"decode_latency_s": 0.060539669284480624, "kv_live_bytes": 462274560, "latency_s": 1.9382145423442125, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74268672, "prompt_kind": "short", "t_relative_s": 11556.926869875286}, {"decode_latency_s": 0.06067421221814584, "kv_live_bytes": 462274560, "latency_s": 1.9424814581871033, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70172672, "prompt_kind": "long", "t_relative_s": 11561.93695516698}, {"decode_latency_s": 0.061067835937137716, "kv_live_bytes": 462274560, "latency_s": 1.9550177496857941, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71041024, "prompt_kind": "short", "t_relative_s": 11566.947044833098}, {"decode_latency_s": 0.06044439584366046, "kv_live_bytes": 462274560, "latency_s": 1.9351522498764098, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71385088, "prompt_kind": "long", "t_relative_s": 11571.957122582942}, {"decode_latency_s": 0.060355486974003725, "kv_live_bytes": 462274560, "latency_s": 1.9322434170171618, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71319552, "prompt_kind": "short", "t_relative_s": 11576.967193375342}, {"decode_latency_s": 0.06760601824498735, "kv_live_bytes": 462274560, "latency_s": 2.165399417281151, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73646080, "prompt_kind": "long", "t_relative_s": 11581.977246042341}, {"decode_latency_s": 0.060430930971051566, "kv_live_bytes": 462274560, "latency_s": 1.934576999861747, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73826304, "prompt_kind": "short", "t_relative_s": 11586.987298625056}, {"decode_latency_s": 0.06798665755195543, "kv_live_bytes": 462274560, "latency_s": 2.177375125233084, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73596928, "prompt_kind": "long", "t_relative_s": 11591.997385083232}, {"decode_latency_s": 0.06054967187810689, "kv_live_bytes": 462274560, "latency_s": 1.9384567090310156, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73236480, "prompt_kind": "short", "t_relative_s": 11597.007437082939}, {"decode_latency_s": 0.06801474999519996, "kv_live_bytes": 462274560, "latency_s": 2.1783913327381015, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73826304, "prompt_kind": "long", "t_relative_s": 11602.017526499927}, {"decode_latency_s": 0.06057382293511182, "kv_live_bytes": 462274560, "latency_s": 1.9391420828178525, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74350592, "prompt_kind": "short", "t_relative_s": 11607.027637583204}, {"decode_latency_s": 0.06720506772398949, "kv_live_bytes": 462274560, "latency_s": 2.1530422498472035, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74547200, "prompt_kind": "long", "t_relative_s": 11612.037696917076}, {"decode_latency_s": 0.06054486590437591, "kv_live_bytes": 462274560, "latency_s": 1.9382404158823192, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74645504, "prompt_kind": "short", "t_relative_s": 11617.047787375282}, {"decode_latency_s": 0.061004337258054875, "kv_live_bytes": 462274560, "latency_s": 1.9530354170128703, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69681152, "prompt_kind": "long", "t_relative_s": 11622.057854167186}, {"decode_latency_s": 0.06306271225912496, "kv_live_bytes": 462274560, "latency_s": 2.0192087083123624, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68419584, "prompt_kind": "short", "t_relative_s": 11627.067943083122}, {"decode_latency_s": 0.06819746484688949, "kv_live_bytes": 462274560, "latency_s": 2.1849260423332453, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73089024, "prompt_kind": "long", "t_relative_s": 11632.078015583102}, {"decode_latency_s": 0.06059880732209422, "kv_live_bytes": 462274560, "latency_s": 1.9399665840901434, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73777152, "prompt_kind": "short", "t_relative_s": 11637.088089250028}, {"decode_latency_s": 0.06714502735121641, "kv_live_bytes": 462274560, "latency_s": 2.1514947921968997, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73744384, "prompt_kind": "long", "t_relative_s": 11642.09813775029}, {"decode_latency_s": 0.06800459764781408, "kv_live_bytes": 462274560, "latency_s": 2.1771147921681404, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91193344, "prompt_kind": "short", "t_relative_s": 11647.108226375189}, {"decode_latency_s": 0.06860037631122395, "kv_live_bytes": 462274560, "latency_s": 2.1973653747700155, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90324992, "prompt_kind": "long", "t_relative_s": 11652.118279417045}, {"decode_latency_s": 0.06090137371211313, "kv_live_bytes": 462274560, "latency_s": 1.9499712083488703, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88719360, "prompt_kind": "short", "t_relative_s": 11657.128355250228}, {"decode_latency_s": 0.06854924350045621, "kv_live_bytes": 462274560, "latency_s": 2.195789041928947, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90931200, "prompt_kind": "long", "t_relative_s": 11662.138422042131}, {"decode_latency_s": 0.06034507160075009, "kv_live_bytes": 462274560, "latency_s": 1.9319213340058923, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91570176, "prompt_kind": "short", "t_relative_s": 11667.148471792229}, {"decode_latency_s": 0.06827405469084624, "kv_live_bytes": 462274560, "latency_s": 2.186956999823451, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91766784, "prompt_kind": "long", "t_relative_s": 11672.158578208182}, {"decode_latency_s": 0.0669156627409393, "kv_live_bytes": 462274560, "latency_s": 2.1431044172495604, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91930624, "prompt_kind": "short", "t_relative_s": 11677.168652958237}, {"decode_latency_s": 0.06831838413199876, "kv_live_bytes": 462274560, "latency_s": 2.1880018752999604, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73646080, "prompt_kind": "long", "t_relative_s": 11682.178704208229}, {"decode_latency_s": 0.06680842577770818, "kv_live_bytes": 462274560, "latency_s": 2.1390127083286643, "n_emitted": 32, "ok": true, "process_footprint_bytes": 65667072, "prompt_kind": "short", "t_relative_s": 11687.188751833048}, {"decode_latency_s": 0.0686461562581826, "kv_live_bytes": 462274560, "latency_s": 2.1984247909858823, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73531392, "prompt_kind": "long", "t_relative_s": 11692.198810042348}, {"decode_latency_s": 0.06057084244093858, "kv_live_bytes": 462274560, "latency_s": 1.9392404998652637, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70500352, "prompt_kind": "short", "t_relative_s": 11697.208864083048}, {"decode_latency_s": 0.06643172005715314, "kv_live_bytes": 462274560, "latency_s": 2.1281534167937934, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73711616, "prompt_kind": "long", "t_relative_s": 11702.218918667175}, {"decode_latency_s": 0.06030408461811021, "kv_live_bytes": 462274560, "latency_s": 1.9305972498841584, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73252864, "prompt_kind": "short", "t_relative_s": 11707.228999667335}, {"decode_latency_s": 0.06830015884770546, "kv_live_bytes": 462274560, "latency_s": 2.187455916777253, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73744384, "prompt_kind": "long", "t_relative_s": 11712.239085208159}, {"decode_latency_s": 0.060824096377473325, "kv_live_bytes": 462274560, "latency_s": 1.947316374629736, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68059136, "prompt_kind": "short", "t_relative_s": 11717.249125875067}, {"decode_latency_s": 0.06844604556681588, "kv_live_bytes": 462274560, "latency_s": 2.1927137090824544, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73318400, "prompt_kind": "long", "t_relative_s": 11722.259204083122}, {"decode_latency_s": 0.06052576821821276, "kv_live_bytes": 462274560, "latency_s": 1.9376422921195626, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74039296, "prompt_kind": "short", "t_relative_s": 11727.269270292018}, {"decode_latency_s": 0.0681912304717116, "kv_live_bytes": 462274560, "latency_s": 2.184387667104602, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74498048, "prompt_kind": "long", "t_relative_s": 11732.279319417197}, {"decode_latency_s": 0.060535440090461634, "kv_live_bytes": 462274560, "latency_s": 1.9379112920723855, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74530816, "prompt_kind": "short", "t_relative_s": 11737.289404875133}, {"decode_latency_s": 0.06913800913025625, "kv_live_bytes": 462274560, "latency_s": 2.2143787923268974, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72744960, "prompt_kind": "long", "t_relative_s": 11742.299460417125}, {"decode_latency_s": 0.061814497399609536, "kv_live_bytes": 462274560, "latency_s": 1.9789448748342693, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72105984, "prompt_kind": "short", "t_relative_s": 11747.309533042368}, {"decode_latency_s": 0.17430418096773792, "kv_live_bytes": 462274560, "latency_s": 5.5786101669073105, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67862528, "prompt_kind": "long", "t_relative_s": 11752.31957920827}, {"decode_latency_s": 0.06031286981306039, "kv_live_bytes": 462274560, "latency_s": 1.9309049169532955, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68550656, "prompt_kind": "short", "t_relative_s": 11760.740496166982}, {"decode_latency_s": 0.06870187890308443, "kv_live_bytes": 462274560, "latency_s": 2.200213958043605, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68747264, "prompt_kind": "long", "t_relative_s": 11765.75057420833}, {"decode_latency_s": 0.060516833342262544, "kv_live_bytes": 462274560, "latency_s": 1.937346207909286, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69435392, "prompt_kind": "short", "t_relative_s": 11770.760648875032}, {"decode_latency_s": 0.06901311327237636, "kv_live_bytes": 462274560, "latency_s": 2.210480500012636, "n_emitted": 32, "ok": true, "process_footprint_bytes": 65961984, "prompt_kind": "long", "t_relative_s": 11775.770704458002}, {"decode_latency_s": 0.060323601559503004, "kv_live_bytes": 462274560, "latency_s": 1.931344416923821, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68141056, "prompt_kind": "short", "t_relative_s": 11780.78076654207}, {"decode_latency_s": 0.07049756772175897, "kv_live_bytes": 462274560, "latency_s": 2.2570884586311877, "n_emitted": 32, "ok": true, "process_footprint_bytes": 89915392, "prompt_kind": "long", "t_relative_s": 11785.790827624965}, {"decode_latency_s": 0.061546966127934866, "kv_live_bytes": 462274560, "latency_s": 1.970306457951665, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90488832, "prompt_kind": "short", "t_relative_s": 11790.800893417094}, {"decode_latency_s": 0.06757203774759546, "kv_live_bytes": 462274560, "latency_s": 2.1646915832534432, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90554368, "prompt_kind": "long", "t_relative_s": 11795.810947000049}, {"decode_latency_s": 0.0604895807482535, "kv_live_bytes": 462274560, "latency_s": 1.936432915739715, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90636288, "prompt_kind": "short", "t_relative_s": 11800.820948458277}, {"decode_latency_s": 0.06848775650723837, "kv_live_bytes": 462274560, "latency_s": 2.193626417312771, "n_emitted": 32, "ok": true, "process_footprint_bytes": 89948160, "prompt_kind": "long", "t_relative_s": 11805.831008207984}, {"decode_latency_s": 0.060751554687158205, "kv_live_bytes": 462274560, "latency_s": 1.944864375051111, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90521600, "prompt_kind": "short", "t_relative_s": 11810.841079958249}, {"decode_latency_s": 0.06066574477881659, "kv_live_bytes": 462274560, "latency_s": 1.942583791911602, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90701824, "prompt_kind": "long", "t_relative_s": 11815.851126166992}, {"decode_latency_s": 0.06028447656717617, "kv_live_bytes": 462274560, "latency_s": 1.929960499983281, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90619904, "prompt_kind": "short", "t_relative_s": 11820.861185375135}, {"decode_latency_s": 0.06881077216530684, "kv_live_bytes": 462274560, "latency_s": 2.2037384156137705, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91226112, "prompt_kind": "long", "t_relative_s": 11825.871261416934}, {"decode_latency_s": 0.060457494779257104, "kv_live_bytes": 462274560, "latency_s": 1.9354906249791384, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91357184, "prompt_kind": "short", "t_relative_s": 11830.881308125332}, {"decode_latency_s": 0.06830701041326392, "kv_live_bytes": 462274560, "latency_s": 2.188435500022024, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90587136, "prompt_kind": "long", "t_relative_s": 11835.891407625284}, {"decode_latency_s": 0.06677598696842324, "kv_live_bytes": 462274560, "latency_s": 2.138766041956842, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91242496, "prompt_kind": "short", "t_relative_s": 11840.901477625128}, {"decode_latency_s": 0.06791719920875039, "kv_live_bytes": 462274560, "latency_s": 2.1756909997202456, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91242496, "prompt_kind": "long", "t_relative_s": 11845.91152512515}, {"decode_latency_s": 0.06026782939443365, "kv_live_bytes": 462274560, "latency_s": 1.9294876670464873, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91193344, "prompt_kind": "short", "t_relative_s": 11850.921623167116}, {"decode_latency_s": 0.06042775390960742, "kv_live_bytes": 462274560, "latency_s": 1.9345789169892669, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91193344, "prompt_kind": "long", "t_relative_s": 11855.93169441726}, {"decode_latency_s": 0.06677452866279054, "kv_live_bytes": 462274560, "latency_s": 2.1385985407978296, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91258880, "prompt_kind": "short", "t_relative_s": 11860.941750708036}, {"decode_latency_s": 0.06821777606091928, "kv_live_bytes": 462274560, "latency_s": 2.185297125019133, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91275264, "prompt_kind": "long", "t_relative_s": 11865.951858083252}, {"decode_latency_s": 0.060584731778362766, "kv_live_bytes": 462274560, "latency_s": 1.9394928752444685, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91832320, "prompt_kind": "short", "t_relative_s": 11870.961926292162}, {"decode_latency_s": 0.060566218744497746, "kv_live_bytes": 462274560, "latency_s": 1.9390031662769616, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91832320, "prompt_kind": "long", "t_relative_s": 11875.971989833284}, {"decode_latency_s": 0.060323330719256774, "kv_live_bytes": 462274560, "latency_s": 1.9312187083996832, "n_emitted": 32, "ok": true, "process_footprint_bytes": 92241920, "prompt_kind": "short", "t_relative_s": 11880.982051375322}, {"decode_latency_s": 0.06858194920641836, "kv_live_bytes": 462274560, "latency_s": 2.1962877502664924, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91914240, "prompt_kind": "long", "t_relative_s": 11885.992136875167}, {"decode_latency_s": 0.06051701694377698, "kv_live_bytes": 462274560, "latency_s": 1.9374147499911487, "n_emitted": 32, "ok": true, "process_footprint_bytes": 91963392, "prompt_kind": "short", "t_relative_s": 11891.002205375116}, {"decode_latency_s": 0.06060880079166964, "kv_live_bytes": 462274560, "latency_s": 1.940643667243421, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71188480, "prompt_kind": "long", "t_relative_s": 11896.01228391705}, {"decode_latency_s": 0.06078389716276433, "kv_live_bytes": 462274560, "latency_s": 1.945899416692555, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71843840, "prompt_kind": "short", "t_relative_s": 11901.022362667136}, {"decode_latency_s": 0.06818730078521185, "kv_live_bytes": 462274560, "latency_s": 2.1842637499794364, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71892992, "prompt_kind": "long", "t_relative_s": 11906.027683957946}, {"decode_latency_s": 0.060378395806765184, "kv_live_bytes": 462274560, "latency_s": 1.9330090419389307, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72302592, "prompt_kind": "short", "t_relative_s": 11911.037771875039}, {"decode_latency_s": 0.06807230990671087, "kv_live_bytes": 462274560, "latency_s": 2.1803262503817677, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71860224, "prompt_kind": "long", "t_relative_s": 11916.04785737535}, {"decode_latency_s": 0.06484793880372308, "kv_live_bytes": 462274560, "latency_s": 2.07598870806396, "n_emitted": 32, "ok": true, "process_footprint_bytes": 89505792, "prompt_kind": "short", "t_relative_s": 11921.057898292318}, {"decode_latency_s": 0.06799349481298123, "kv_live_bytes": 462274560, "latency_s": 2.1782364998944104, "n_emitted": 32, "ok": true, "process_footprint_bytes": 89423872, "prompt_kind": "long", "t_relative_s": 11926.06798116723}, {"decode_latency_s": 0.06141072134778369, "kv_live_bytes": 462274560, "latency_s": 1.965949125122279, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90062848, "prompt_kind": "short", "t_relative_s": 11931.07806575019}, {"decode_latency_s": 0.06802014193090145, "kv_live_bytes": 462274560, "latency_s": 2.178968334104866, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88358912, "prompt_kind": "long", "t_relative_s": 11936.08827862516}, {"decode_latency_s": 0.06041471353091765, "kv_live_bytes": 462274560, "latency_s": 1.9340493329800665, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90439680, "prompt_kind": "short", "t_relative_s": 11941.09834758332}, {"decode_latency_s": 0.06828670832328498, "kv_live_bytes": 462274560, "latency_s": 2.1875349576584995, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90456064, "prompt_kind": "long", "t_relative_s": 11946.108400125057}, {"decode_latency_s": 0.060594087219215, "kv_live_bytes": 462274560, "latency_s": 1.9398123752325773, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90406912, "prompt_kind": "short", "t_relative_s": 11951.118492207956}, {"decode_latency_s": 0.0684599935048027, "kv_live_bytes": 462274560, "latency_s": 2.1929817497730255, "n_emitted": 32, "ok": true, "process_footprint_bytes": 89604096, "prompt_kind": "long", "t_relative_s": 11956.12853912497}, {"decode_latency_s": 0.06030586326960474, "kv_live_bytes": 462274560, "latency_s": 1.9307335829362273, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90095616, "prompt_kind": "short", "t_relative_s": 11961.138607167173}, {"decode_latency_s": 0.06776464974973351, "kv_live_bytes": 462274560, "latency_s": 2.1705324999056756, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90210304, "prompt_kind": "long", "t_relative_s": 11966.1486800001}, {"decode_latency_s": 0.06059016668586992, "kv_live_bytes": 462274560, "latency_s": 1.9397534998133779, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90800128, "prompt_kind": "short", "t_relative_s": 11971.158727000002}, {"decode_latency_s": 0.06821944272087421, "kv_live_bytes": 462274560, "latency_s": 2.185310082975775, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90882048, "prompt_kind": "long", "t_relative_s": 11976.168795292266}, {"decode_latency_s": 0.06054946615768131, "kv_live_bytes": 462274560, "latency_s": 1.9383710422553122, "n_emitted": 32, "ok": true, "process_footprint_bytes": 90963968, "prompt_kind": "short", "t_relative_s": 11981.178866750095}, {"decode_latency_s": 0.06854413803375792, "kv_live_bytes": 462274560, "latency_s": 2.1953324167989194, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68747264, "prompt_kind": "long", "t_relative_s": 11986.188920750283}, {"decode_latency_s": 0.06594852084526792, "kv_live_bytes": 462274560, "latency_s": 2.112128082662821, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71483392, "prompt_kind": "short", "t_relative_s": 11991.198982333299}, {"decode_latency_s": 0.06804072916565929, "kv_live_bytes": 462274560, "latency_s": 2.179753000382334, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72712192, "prompt_kind": "long", "t_relative_s": 11996.209062833339}, {"decode_latency_s": 0.06686188280582428, "kv_live_bytes": 462274560, "latency_s": 2.141231916844845, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73351168, "prompt_kind": "short", "t_relative_s": 12001.219128458295}, {"decode_latency_s": 0.06834567578334827, "kv_live_bytes": 462274560, "latency_s": 2.1897309999912977, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73433088, "prompt_kind": "long", "t_relative_s": 12006.229202917311}, {"decode_latency_s": 0.060311343739158474, "kv_live_bytes": 462274560, "latency_s": 1.9308476247824728, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73007104, "prompt_kind": "short", "t_relative_s": 12011.23927487526}, {"decode_latency_s": 0.068412322900258, "kv_live_bytes": 462274560, "latency_s": 2.1907887905836105, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72597504, "prompt_kind": "long", "t_relative_s": 12016.249347958248}, {"decode_latency_s": 0.06697625390370376, "kv_live_bytes": 462274560, "latency_s": 2.144789499696344, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73515008, "prompt_kind": "short", "t_relative_s": 12021.259401042014}, {"decode_latency_s": 0.06803421874064952, "kv_live_bytes": 462274560, "latency_s": 2.1794537086971104, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73744384, "prompt_kind": "long", "t_relative_s": 12026.2695035832}, {"decode_latency_s": 0.06675049867772032, "kv_live_bytes": 462274560, "latency_s": 2.1379614579491317, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74350592, "prompt_kind": "short", "t_relative_s": 12031.279605207965}, {"decode_latency_s": 0.06848659114621114, "kv_live_bytes": 462274560, "latency_s": 2.193657541181892, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74399744, "prompt_kind": "long", "t_relative_s": 12036.289661792107}, {"decode_latency_s": 0.06040168881008867, "kv_live_bytes": 462274560, "latency_s": 1.9337522499263287, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73826304, "prompt_kind": "short", "t_relative_s": 12041.29974012496}, {"decode_latency_s": 0.06867190366028808, "kv_live_bytes": 462274560, "latency_s": 2.19975558295846, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70991872, "prompt_kind": "long", "t_relative_s": 12046.309816000052}, {"decode_latency_s": 0.06063204556994606, "kv_live_bytes": 462274560, "latency_s": 1.9411635003052652, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73613312, "prompt_kind": "short", "t_relative_s": 12051.319876417052}, {"decode_latency_s": 0.0677522825426422, "kv_live_bytes": 462274560, "latency_s": 2.1706743338145316, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78430208, "prompt_kind": "long", "t_relative_s": 12056.329947292339}, {"decode_latency_s": 0.06640959765354637, "kv_live_bytes": 462274560, "latency_s": 2.1267543751746416, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78741504, "prompt_kind": "short", "t_relative_s": 12061.340021792334}, {"decode_latency_s": 0.06783591795829125, "kv_live_bytes": 462274560, "latency_s": 2.173259750008583, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79822848, "prompt_kind": "long", "t_relative_s": 12066.350050583016}, {"decode_latency_s": 0.06374675262486562, "kv_live_bytes": 462274560, "latency_s": 2.0409959158860147, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78053376, "prompt_kind": "short", "t_relative_s": 12071.360115708318}, {"decode_latency_s": 0.06774293880152982, "kv_live_bytes": 462274560, "latency_s": 2.170000209007412, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79134720, "prompt_kind": "long", "t_relative_s": 12076.37017370807}, {"decode_latency_s": 0.06052855467714835, "kv_live_bytes": 462274560, "latency_s": 1.9377029589377344, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79200256, "prompt_kind": "short", "t_relative_s": 12081.380240083206}, {"decode_latency_s": 0.06814198959909845, "kv_live_bytes": 462274560, "latency_s": 2.1827631248161197, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79331328, "prompt_kind": "long", "t_relative_s": 12086.390292041935}, {"decode_latency_s": 0.06048942315101158, "kv_live_bytes": 462274560, "latency_s": 1.9364282498136163, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79790080, "prompt_kind": "short", "t_relative_s": 12091.400369917043}, {"decode_latency_s": 0.06800268232473172, "kv_live_bytes": 462274560, "latency_s": 2.1777124581858516, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78692352, "prompt_kind": "long", "t_relative_s": 12096.410419000313}, {"decode_latency_s": 0.060352079395670444, "kv_live_bytes": 462274560, "latency_s": 1.9322599591687322, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68730880, "prompt_kind": "short", "t_relative_s": 12101.42045537522}, {"decode_latency_s": 0.06841043359600008, "kv_live_bytes": 462274560, "latency_s": 2.19097250001505, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74022912, "prompt_kind": "long", "t_relative_s": 12106.430513292085}, {"decode_latency_s": 0.06108236718864646, "kv_live_bytes": 462274560, "latency_s": 1.9556569578126073, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71630848, "prompt_kind": "short", "t_relative_s": 12111.44056012528}, {"decode_latency_s": 0.0675408749957569, "kv_live_bytes": 462274560, "latency_s": 2.1636573341675103, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72368128, "prompt_kind": "long", "t_relative_s": 12116.450617583003}, {"decode_latency_s": 0.06053736197645776, "kv_live_bytes": 462274560, "latency_s": 1.9380205422639847, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73023488, "prompt_kind": "short", "t_relative_s": 12121.460709000006}, {"decode_latency_s": 0.06826668097346555, "kv_live_bytes": 462274560, "latency_s": 2.1862141247838736, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73039872, "prompt_kind": "long", "t_relative_s": 12126.470794500317}, {"decode_latency_s": 0.06079582289385144, "kv_live_bytes": 462274560, "latency_s": 1.9464260833337903, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71991296, "prompt_kind": "short", "t_relative_s": 12131.480844791979}, {"decode_latency_s": 0.06828922005661298, "kv_live_bytes": 462274560, "latency_s": 2.187331000342965, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72400896, "prompt_kind": "long", "t_relative_s": 12136.490931042004}, {"decode_latency_s": 0.06059075391385704, "kv_live_bytes": 462274560, "latency_s": 1.939709875266999, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72613888, "prompt_kind": "short", "t_relative_s": 12141.500974291936}, {"decode_latency_s": 0.0681617395603098, "kv_live_bytes": 462274560, "latency_s": 2.1833870839327574, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "long", "t_relative_s": 12146.511050375178}, {"decode_latency_s": 0.06710160028887913, "kv_live_bytes": 462274560, "latency_s": 2.148867500014603, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73121792, "prompt_kind": "short", "t_relative_s": 12151.521126166917}, {"decode_latency_s": 0.060678911468130536, "kv_live_bytes": 462274560, "latency_s": 1.9431546670384705, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73039872, "prompt_kind": "long", "t_relative_s": 12156.53116104193}, {"decode_latency_s": 0.06015461587230675, "kv_live_bytes": 462274560, "latency_s": 1.9258967908099294, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71827456, "prompt_kind": "short", "t_relative_s": 12161.541224417277}, {"decode_latency_s": 0.06076595053309575, "kv_live_bytes": 462274560, "latency_s": 1.945755708962679, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70795264, "prompt_kind": "long", "t_relative_s": 12166.551308458205}, {"decode_latency_s": 0.0665575924795121, "kv_live_bytes": 462274560, "latency_s": 2.131628416944295, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71565312, "prompt_kind": "short", "t_relative_s": 12171.561357667204}, {"decode_latency_s": 0.06804114974511322, "kv_live_bytes": 462274560, "latency_s": 2.179458625148982, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72237056, "prompt_kind": "long", "t_relative_s": 12176.571476125158}, {"decode_latency_s": 0.06055552343605086, "kv_live_bytes": 462274560, "latency_s": 1.9386054170317948, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72794112, "prompt_kind": "short", "t_relative_s": 12181.581581417006}, {"decode_latency_s": 0.06802313671505544, "kv_live_bytes": 462274560, "latency_s": 2.179129040800035, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72794112, "prompt_kind": "long", "t_relative_s": 12186.591637917329}, {"decode_latency_s": 0.06203959896811284, "kv_live_bytes": 462274560, "latency_s": 1.9861859171651304, "n_emitted": 32, "ok": true, "process_footprint_bytes": 82821120, "prompt_kind": "short", "t_relative_s": 12191.601719500031}, {"decode_latency_s": 0.06817565493111033, "kv_live_bytes": 462274560, "latency_s": 2.183706416748464, "n_emitted": 32, "ok": true, "process_footprint_bytes": 82788352, "prompt_kind": "long", "t_relative_s": 12196.611761583015}, {"decode_latency_s": 0.06056633462139871, "kv_live_bytes": 462274560, "latency_s": 1.9390190420672297, "n_emitted": 32, "ok": true, "process_footprint_bytes": 83361792, "prompt_kind": "short", "t_relative_s": 12201.621844167355}, {"decode_latency_s": 0.06792688541463576, "kv_live_bytes": 462274560, "latency_s": 2.175219625234604, "n_emitted": 32, "ok": true, "process_footprint_bytes": 82624512, "prompt_kind": "long", "t_relative_s": 12206.631925249938}, {"decode_latency_s": 0.06029224609665107, "kv_live_bytes": 462274560, "latency_s": 1.9302652077749372, "n_emitted": 32, "ok": true, "process_footprint_bytes": 83099648, "prompt_kind": "short", "t_relative_s": 12211.641968333162}, {"decode_latency_s": 0.06833311196533032, "kv_live_bytes": 462274560, "latency_s": 2.188750416971743, "n_emitted": 32, "ok": true, "process_footprint_bytes": 83853312, "prompt_kind": "long", "t_relative_s": 12216.652045125142}, {"decode_latency_s": 0.060515218749060296, "kv_live_bytes": 462274560, "latency_s": 1.9372894582338631, "n_emitted": 32, "ok": true, "process_footprint_bytes": 84017152, "prompt_kind": "short", "t_relative_s": 12221.662111333106}, {"decode_latency_s": 0.0614861289068358, "kv_live_bytes": 462274560, "latency_s": 1.9685254590585828, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79036416, "prompt_kind": "long", "t_relative_s": 12226.67218691716}, {"decode_latency_s": 0.060385403645341285, "kv_live_bytes": 462274560, "latency_s": 1.9332340830005705, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68698112, "prompt_kind": "short", "t_relative_s": 12231.68226583302}, {"decode_latency_s": 0.060266608066740446, "kv_live_bytes": 462274560, "latency_s": 1.9294495838694274, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69795840, "prompt_kind": "long", "t_relative_s": 12236.692406458315}, {"decode_latency_s": 0.06073337109410204, "kv_live_bytes": 462274560, "latency_s": 1.9444221672601998, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70418432, "prompt_kind": "short", "t_relative_s": 12241.702451958321}, {"decode_latency_s": 0.06805600911320653, "kv_live_bytes": 462274560, "latency_s": 2.180113833397627, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73023488, "prompt_kind": "long", "t_relative_s": 12246.71254483331}, {"decode_latency_s": 0.06645940493035596, "kv_live_bytes": 462274560, "latency_s": 2.1277047088369727, "n_emitted": 32, "ok": true, "process_footprint_bytes": 40058880, "prompt_kind": "short", "t_relative_s": 12251.722641333006}, {"decode_latency_s": 0.0683059778821189, "kv_live_bytes": 462274560, "latency_s": 2.1879805829375982, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71270400, "prompt_kind": "long", "t_relative_s": 12256.732736125123}, {"decode_latency_s": 0.06726237760449294, "kv_live_bytes": 462274560, "latency_s": 2.154225541744381, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72531968, "prompt_kind": "short", "t_relative_s": 12261.742789958138}, {"decode_latency_s": 0.06781838934693951, "kv_live_bytes": 462274560, "latency_s": 2.1725820833817124, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "long", "t_relative_s": 12266.752908250317}, {"decode_latency_s": 0.06063890234509017, "kv_live_bytes": 462274560, "latency_s": 1.9412767910398543, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73351168, "prompt_kind": "short", "t_relative_s": 12271.76297695795}, {"decode_latency_s": 0.06802449087263085, "kv_live_bytes": 462274560, "latency_s": 2.1792213330045342, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73318400, "prompt_kind": "long", "t_relative_s": 12276.77302316716}, {"decode_latency_s": 0.06301358202472329, "kv_live_bytes": 462274560, "latency_s": 2.017794458195567, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72204288, "prompt_kind": "short", "t_relative_s": 12281.78308650013}, {"decode_latency_s": 0.0665631458396092, "kv_live_bytes": 462274560, "latency_s": 2.1320356247015297, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72761344, "prompt_kind": "long", "t_relative_s": 12286.793161833193}, {"decode_latency_s": 0.06699499087699223, "kv_live_bytes": 462274560, "latency_s": 2.145975916646421, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72859648, "prompt_kind": "short", "t_relative_s": 12291.803226875141}, {"decode_latency_s": 0.06803354296425823, "kv_live_bytes": 462274560, "latency_s": 2.178874040953815, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73482240, "prompt_kind": "long", "t_relative_s": 12296.806971250102}, {"decode_latency_s": 0.060237921876250766, "kv_live_bytes": 462274560, "latency_s": 1.9285459169186652, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72925184, "prompt_kind": "short", "t_relative_s": 12301.817049417179}, {"decode_latency_s": 0.06823134116712026, "kv_live_bytes": 462274560, "latency_s": 2.185677208006382, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73416704, "prompt_kind": "long", "t_relative_s": 12306.827105167322}, {"decode_latency_s": 0.06053378515935037, "kv_live_bytes": 462274560, "latency_s": 1.9378919582813978, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73482240, "prompt_kind": "short", "t_relative_s": 12311.83718600031}, {"decode_latency_s": 0.06848526561225299, "kv_live_bytes": 462274560, "latency_s": 2.193434542044997, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73236480, "prompt_kind": "long", "t_relative_s": 12316.847264000215}, {"decode_latency_s": 0.06037917059438769, "kv_live_bytes": 462274560, "latency_s": 1.933083708398044, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73302016, "prompt_kind": "short", "t_relative_s": 12321.857304542325}, {"decode_latency_s": 0.06134595703042578, "kv_live_bytes": 462274560, "latency_s": 1.9643642911687493, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79986688, "prompt_kind": "long", "t_relative_s": 12326.86737699993}, {"decode_latency_s": 0.06064971744490322, "kv_live_bytes": 462274560, "latency_s": 1.9416121668182313, "n_emitted": 32, "ok": true, "process_footprint_bytes": 80625664, "prompt_kind": "short", "t_relative_s": 12331.877434582915}, {"decode_latency_s": 0.06804838932293933, "kv_live_bytes": 462274560, "latency_s": 2.1796928751282394, "n_emitted": 32, "ok": true, "process_footprint_bytes": 80642048, "prompt_kind": "long", "t_relative_s": 12336.887492833193}, {"decode_latency_s": 0.06314783202833496, "kv_live_bytes": 462274560, "latency_s": 2.0216832919977605, "n_emitted": 32, "ok": true, "process_footprint_bytes": 77414400, "prompt_kind": "short", "t_relative_s": 12341.89757854212}, {"decode_latency_s": 0.0683806119923247, "kv_live_bytes": 462274560, "latency_s": 2.190392999909818, "n_emitted": 32, "ok": true, "process_footprint_bytes": 78036992, "prompt_kind": "long", "t_relative_s": 12346.90764695825}, {"decode_latency_s": 0.06587107031373307, "kv_live_bytes": 462274560, "latency_s": 2.1092764157801867, "n_emitted": 32, "ok": true, "process_footprint_bytes": 79429632, "prompt_kind": "short", "t_relative_s": 12351.917745000217}, {"decode_latency_s": 0.06332341537927277, "kv_live_bytes": 462274560, "latency_s": 2.0273245410062373, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73416704, "prompt_kind": "long", "t_relative_s": 12356.927866375074}, {"decode_latency_s": 0.0657087265572045, "kv_live_bytes": 462274560, "latency_s": 2.103538792114705, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66682880, "prompt_kind": "short", "t_relative_s": 12361.937920500059}, {"decode_latency_s": 0.06160232680849731, "kv_live_bytes": 462274560, "latency_s": 1.9723236253485084, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69435392, "prompt_kind": "long", "t_relative_s": 12366.948025042191}, {"decode_latency_s": 0.06055927996931132, "kv_live_bytes": 462274560, "latency_s": 1.9388695838861167, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68648960, "prompt_kind": "short", "t_relative_s": 12371.958112500142}, {"decode_latency_s": 0.06804214842850342, "kv_live_bytes": 462274560, "latency_s": 2.179209374822676, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71188480, "prompt_kind": "long", "t_relative_s": 12376.968189542182}, {"decode_latency_s": 0.06705392186995596, "kv_live_bytes": 462274560, "latency_s": 2.1474569169804454, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71303168, "prompt_kind": "short", "t_relative_s": 12381.978245417122}, {"decode_latency_s": 0.06847968359943479, "kv_live_bytes": 462274560, "latency_s": 2.194163749925792, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71368704, "prompt_kind": "long", "t_relative_s": 12386.988349042367}, {"decode_latency_s": 0.06120491924230009, "kv_live_bytes": 462274560, "latency_s": 1.959353500045836, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71843840, "prompt_kind": "short", "t_relative_s": 12391.998425000347}, {"decode_latency_s": 0.07000991146196611, "kv_live_bytes": 462274560, "latency_s": 2.2419300000183284, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70828032, "prompt_kind": "long", "t_relative_s": 12397.008472917136}, {"decode_latency_s": 0.060683378906105645, "kv_live_bytes": 462274560, "latency_s": 1.9427917920984328, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68009984, "prompt_kind": "short", "t_relative_s": 12402.018532750197}, {"decode_latency_s": 0.0675717968842946, "kv_live_bytes": 462274560, "latency_s": 2.1653196248225868, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69959680, "prompt_kind": "long", "t_relative_s": 12407.02859195834}, {"decode_latency_s": 0.060525971348397434, "kv_live_bytes": 462274560, "latency_s": 1.9378149593248963, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70025216, "prompt_kind": "short", "t_relative_s": 12412.038496000227}, {"decode_latency_s": 0.060492770848213695, "kv_live_bytes": 462274560, "latency_s": 1.9367706668563187, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71106560, "prompt_kind": "long", "t_relative_s": 12417.048552083317}, {"decode_latency_s": 0.060592832043766975, "kv_live_bytes": 462274560, "latency_s": 1.9397217920050025, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71761920, "prompt_kind": "short", "t_relative_s": 12422.058679167181}, {"decode_latency_s": 0.06818072784517426, "kv_live_bytes": 462274560, "latency_s": 2.1834432086907327, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71499776, "prompt_kind": "long", "t_relative_s": 12427.068765250035}, {"decode_latency_s": 0.06041372397157829, "kv_live_bytes": 462274560, "latency_s": 1.9341088752262294, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70975488, "prompt_kind": "short", "t_relative_s": 12432.078827542253}, {"decode_latency_s": 0.0678368072694866, "kv_live_bytes": 462274560, "latency_s": 2.1732739582657814, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71090176, "prompt_kind": "long", "t_relative_s": 12437.083540417254}, {"decode_latency_s": 0.060394764310331084, "kv_live_bytes": 462274560, "latency_s": 1.9334860830567777, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71106560, "prompt_kind": "short", "t_relative_s": 12442.09362262534}, {"decode_latency_s": 0.06801666534738615, "kv_live_bytes": 462274560, "latency_s": 2.1782588749192655, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71270400, "prompt_kind": "long", "t_relative_s": 12447.103744583204}, {"decode_latency_s": 0.06058164453133941, "kv_live_bytes": 462274560, "latency_s": 1.9395700409077108, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71860224, "prompt_kind": "short", "t_relative_s": 12452.113787250128}, {"decode_latency_s": 0.06791738409083337, "kv_live_bytes": 462274560, "latency_s": 2.175918749999255, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71909376, "prompt_kind": "long", "t_relative_s": 12457.123858999927}, {"decode_latency_s": 0.060788664064602926, "kv_live_bytes": 462274560, "latency_s": 1.9460438750684261, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74186752, "prompt_kind": "short", "t_relative_s": 12462.133949249983}, {"decode_latency_s": 0.06066256119811442, "kv_live_bytes": 462274560, "latency_s": 1.9425432910211384, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71761920, "prompt_kind": "long", "t_relative_s": 12467.14401533315}, {"decode_latency_s": 0.06042421094025485, "kv_live_bytes": 462274560, "latency_s": 1.9344650409184396, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71761920, "prompt_kind": "short", "t_relative_s": 12472.154091042}, {"decode_latency_s": 0.06802586068806704, "kv_live_bytes": 462274560, "latency_s": 2.1786280423402786, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72810496, "prompt_kind": "long", "t_relative_s": 12477.164187000133}, {"decode_latency_s": 0.060673162748571485, "kv_live_bytes": 462274560, "latency_s": 1.942486499901861, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73498624, "prompt_kind": "short", "t_relative_s": 12482.174243125133}, {"decode_latency_s": 0.06032848305767402, "kv_live_bytes": 462274560, "latency_s": 1.9315682090818882, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73465856, "prompt_kind": "long", "t_relative_s": 12487.18432179233}, {"decode_latency_s": 0.0664334804750979, "kv_live_bytes": 462274560, "latency_s": 2.127881250344217, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73580544, "prompt_kind": "short", "t_relative_s": 12492.1944007501}, {"decode_latency_s": 0.09556336195964832, "kv_live_bytes": 462274560, "latency_s": 3.0589225003495812, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69599232, "prompt_kind": "long", "t_relative_s": 12497.204446833115}, {"decode_latency_s": 0.06151049218897242, "kv_live_bytes": 462274560, "latency_s": 1.9691338748671114, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70352896, "prompt_kind": "short", "t_relative_s": 12504.418915000279}, {"decode_latency_s": 0.06050996744306758, "kv_live_bytes": 462274560, "latency_s": 1.9376341253519058, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70893568, "prompt_kind": "long", "t_relative_s": 12509.42896229215}, {"decode_latency_s": 0.06038380468089599, "kv_live_bytes": 462274560, "latency_s": 1.9331583338789642, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69222400, "prompt_kind": "short", "t_relative_s": 12514.439010708127}, {"decode_latency_s": 0.06855121352418792, "kv_live_bytes": 462274560, "latency_s": 2.195669916924089, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69304320, "prompt_kind": "long", "t_relative_s": 12519.449082750361}, {"decode_latency_s": 0.06662582680291962, "kv_live_bytes": 462274560, "latency_s": 2.133633750025183, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69402624, "prompt_kind": "short", "t_relative_s": 12524.459149500355}, {"decode_latency_s": 0.06058233724616002, "kv_live_bytes": 462274560, "latency_s": 1.9398237499408424, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69369856, "prompt_kind": "long", "t_relative_s": 12529.469205208123}, {"decode_latency_s": 0.06072704818507191, "kv_live_bytes": 462274560, "latency_s": 1.9440426668152213, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70074368, "prompt_kind": "short", "t_relative_s": 12534.479293291923}, {"decode_latency_s": 0.06798729037109297, "kv_live_bytes": 462274560, "latency_s": 2.1775752496905625, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69353472, "prompt_kind": "long", "t_relative_s": 12539.489339125343}, {"decode_latency_s": 0.06672069399792235, "kv_live_bytes": 462274560, "latency_s": 2.136847708374262, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69337088, "prompt_kind": "short", "t_relative_s": 12544.499414708}, {"decode_latency_s": 0.0678510937577812, "kv_live_bytes": 462274560, "latency_s": 2.173056958243251, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70025216, "prompt_kind": "long", "t_relative_s": 12549.50946279196}, {"decode_latency_s": 0.06034032293246128, "kv_live_bytes": 462274560, "latency_s": 1.9317574589513242, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70008832, "prompt_kind": "short", "t_relative_s": 12554.519584625028}, {"decode_latency_s": 0.06822050259506796, "kv_live_bytes": 462274560, "latency_s": 2.1849387916736305, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69877760, "prompt_kind": "long", "t_relative_s": 12559.52965804236}, {"decode_latency_s": 0.06678687890234869, "kv_live_bytes": 462274560, "latency_s": 2.138633292168379, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70565888, "prompt_kind": "short", "t_relative_s": 12564.539702500217}, {"decode_latency_s": 0.06767161327297799, "kv_live_bytes": 462274560, "latency_s": 2.167852624785155, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70598656, "prompt_kind": "long", "t_relative_s": 12569.549780750182}, {"decode_latency_s": 0.0668764817819465, "kv_live_bytes": 462274560, "latency_s": 2.141805708874017, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70664192, "prompt_kind": "short", "t_relative_s": 12574.559883708134}, {"decode_latency_s": 0.06756966015382204, "kv_live_bytes": 462274560, "latency_s": 2.1645514997653663, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70615040, "prompt_kind": "long", "t_relative_s": 12579.569934708066}, {"decode_latency_s": 0.06030339976132382, "kv_live_bytes": 462274560, "latency_s": 1.9306007497943938, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70729728, "prompt_kind": "short", "t_relative_s": 12584.575063792057}, {"decode_latency_s": 0.06637801302713342, "kv_live_bytes": 462274560, "latency_s": 2.125746500212699, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69910528, "prompt_kind": "long", "t_relative_s": 12589.585207541939}, {"decode_latency_s": 0.0610824492177926, "kv_live_bytes": 462274560, "latency_s": 1.9559603752568364, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69435392, "prompt_kind": "short", "t_relative_s": 12594.595241250005}, {"decode_latency_s": 0.06353088282048702, "kv_live_bytes": 462274560, "latency_s": 2.0346877919510007, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71729152, "prompt_kind": "long", "t_relative_s": 12599.605297124945}, {"decode_latency_s": 0.06686354297562502, "kv_live_bytes": 462274560, "latency_s": 2.141300041228533, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72351744, "prompt_kind": "short", "t_relative_s": 12604.615396792069}, {"decode_latency_s": 0.0677087434742134, "kv_live_bytes": 462274560, "latency_s": 2.1692739170975983, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72482816, "prompt_kind": "long", "t_relative_s": 12609.625439583324}, {"decode_latency_s": 0.06057444789621513, "kv_live_bytes": 462274560, "latency_s": 1.9392180829308927, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71385088, "prompt_kind": "short", "t_relative_s": 12614.635529292282}, {"decode_latency_s": 0.060836315125925466, "kv_live_bytes": 462274560, "latency_s": 1.948036041110754, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71843840, "prompt_kind": "long", "t_relative_s": 12619.64562162524}, {"decode_latency_s": 0.06043762889748905, "kv_live_bytes": 462274560, "latency_s": 1.9349272497929633, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72433664, "prompt_kind": "short", "t_relative_s": 12624.655661917292}, {"decode_latency_s": 0.06846043231780641, "kv_live_bytes": 462274560, "latency_s": 2.1926879161037505, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72499200, "prompt_kind": "long", "t_relative_s": 12629.665754917078}, {"decode_latency_s": 0.06686473306035623, "kv_live_bytes": 462274560, "latency_s": 2.141379290726036, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73039872, "prompt_kind": "short", "t_relative_s": 12634.673539374955}, {"decode_latency_s": 0.06786630468559451, "kv_live_bytes": 462274560, "latency_s": 2.1741765001788735, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72941568, "prompt_kind": "long", "t_relative_s": 12639.68359050015}, {"decode_latency_s": 0.06061113539908547, "kv_live_bytes": 462274560, "latency_s": 1.9403442088514566, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72663040, "prompt_kind": "short", "t_relative_s": 12644.693679625168}, {"decode_latency_s": 0.06751523046114016, "kv_live_bytes": 462274560, "latency_s": 2.1624902500770986, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72663040, "prompt_kind": "long", "t_relative_s": 12649.703735332936}, {"decode_latency_s": 0.060592563822865486, "kv_live_bytes": 462274560, "latency_s": 1.9398444998078048, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "short", "t_relative_s": 12654.713799792342}, {"decode_latency_s": 0.06811096222372726, "kv_live_bytes": 462274560, "latency_s": 2.1820968752726912, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73072640, "prompt_kind": "long", "t_relative_s": 12659.723831708077}, {"decode_latency_s": 0.06052062238450162, "kv_live_bytes": 462274560, "latency_s": 1.9376163752749562, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71860224, "prompt_kind": "short", "t_relative_s": 12664.733951500151}, {"decode_latency_s": 0.060582822901778854, "kv_live_bytes": 462274560, "latency_s": 1.9400875843130052, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70467584, "prompt_kind": "long", "t_relative_s": 12669.744026625063}, {"decode_latency_s": 0.06090637498709839, "kv_live_bytes": 462274560, "latency_s": 1.9498642077669501, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70189056, "prompt_kind": "short", "t_relative_s": 12674.754124208353}, {"decode_latency_s": 0.06775191925407853, "kv_live_bytes": 462274560, "latency_s": 2.170211249962449, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71090176, "prompt_kind": "long", "t_relative_s": 12679.764173833188}, {"decode_latency_s": 0.06037860414653551, "kv_live_bytes": 462274560, "latency_s": 1.93302454194054, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71073792, "prompt_kind": "short", "t_relative_s": 12684.774250833318}, {"decode_latency_s": 0.06767369659792166, "kv_live_bytes": 462274560, "latency_s": 2.1673330417834222, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71761920, "prompt_kind": "long", "t_relative_s": 12689.784335208125}, {"decode_latency_s": 0.06666170702374075, "kv_live_bytes": 462274560, "latency_s": 2.1347525003366172, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72335360, "prompt_kind": "short", "t_relative_s": 12694.79438512493}, {"decode_latency_s": 0.06802615756168962, "kv_live_bytes": 462274560, "latency_s": 2.17944779060781, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72400896, "prompt_kind": "long", "t_relative_s": 12699.80446608318}, {"decode_latency_s": 0.060697029941366054, "kv_live_bytes": 462274560, "latency_s": 1.9431094168685377, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71860224, "prompt_kind": "short", "t_relative_s": 12704.814550833311}, {"decode_latency_s": 0.06093204428907484, "kv_live_bytes": 462274560, "latency_s": 1.950738999992609, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70828032, "prompt_kind": "long", "t_relative_s": 12709.824602667242}, {"decode_latency_s": 0.06091399482102133, "kv_live_bytes": 462274560, "latency_s": 1.9500075839459896, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71368704, "prompt_kind": "short", "t_relative_s": 12714.83465562528}, {"decode_latency_s": 0.060591736968490295, "kv_live_bytes": 462274560, "latency_s": 1.9401217079721391, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71483392, "prompt_kind": "long", "t_relative_s": 12719.844726958312}, {"decode_latency_s": 0.06051379686687142, "kv_live_bytes": 462274560, "latency_s": 1.9372537923045456, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72007680, "prompt_kind": "short", "t_relative_s": 12724.85480708303}, {"decode_latency_s": 0.06778858724283054, "kv_live_bytes": 462274560, "latency_s": 2.171304459217936, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71974912, "prompt_kind": "long", "t_relative_s": 12729.864857167006}, {"decode_latency_s": 0.06421019011759199, "kv_live_bytes": 462274560, "latency_s": 2.055574500001967, "n_emitted": 32, "ok": true, "process_footprint_bytes": 63848448, "prompt_kind": "short", "t_relative_s": 12734.874945749994}, {"decode_latency_s": 0.06737270051962696, "kv_live_bytes": 462274560, "latency_s": 2.1585252499207854, "n_emitted": 32, "ok": true, "process_footprint_bytes": 86016000, "prompt_kind": "long", "t_relative_s": 12739.88499129191}, {"decode_latency_s": 0.0605725755012827, "kv_live_bytes": 462274560, "latency_s": 1.9391442500054836, "n_emitted": 32, "ok": true, "process_footprint_bytes": 86654976, "prompt_kind": "short", "t_relative_s": 12744.895076707937}, {"decode_latency_s": 0.06901502342952881, "kv_live_bytes": 462274560, "latency_s": 2.2103301668539643, "n_emitted": 32, "ok": true, "process_footprint_bytes": 85835776, "prompt_kind": "long", "t_relative_s": 12749.905130667146}, {"decode_latency_s": 0.06048633070895448, "kv_live_bytes": 462274560, "latency_s": 1.9364865408279002, "n_emitted": 32, "ok": true, "process_footprint_bytes": 86605824, "prompt_kind": "short", "t_relative_s": 12754.915211291984}, {"decode_latency_s": 0.06847165364888497, "kv_live_bytes": 462274560, "latency_s": 2.193290624767542, "n_emitted": 32, "ok": true, "process_footprint_bytes": 87048192, "prompt_kind": "long", "t_relative_s": 12759.925298375078}, {"decode_latency_s": 0.06702564972511027, "kv_live_bytes": 462274560, "latency_s": 2.1464442922733724, "n_emitted": 32, "ok": true, "process_footprint_bytes": 87097344, "prompt_kind": "short", "t_relative_s": 12764.935369832907}, {"decode_latency_s": 0.06781489193963353, "kv_live_bytes": 462274560, "latency_s": 2.172777791041881, "n_emitted": 32, "ok": true, "process_footprint_bytes": 87097344, "prompt_kind": "long", "t_relative_s": 12769.945435666945}, {"decode_latency_s": 0.0606259531195974, "kv_live_bytes": 462274560, "latency_s": 1.9408390000462532, "n_emitted": 32, "ok": true, "process_footprint_bytes": 87785472, "prompt_kind": "short", "t_relative_s": 12774.954695458058}, {"decode_latency_s": 0.06799674087960739, "kv_live_bytes": 462274560, "latency_s": 2.1779312500730157, "n_emitted": 32, "ok": true, "process_footprint_bytes": 87228416, "prompt_kind": "long", "t_relative_s": 12779.96474645799}, {"decode_latency_s": 0.060591152345295995, "kv_live_bytes": 462274560, "latency_s": 1.9396640420891345, "n_emitted": 32, "ok": true, "process_footprint_bytes": 87212032, "prompt_kind": "short", "t_relative_s": 12784.974843667354}, {"decode_latency_s": 0.06748383202648256, "kv_live_bytes": 462274560, "latency_s": 2.1619081660173833, "n_emitted": 32, "ok": true, "process_footprint_bytes": 87769088, "prompt_kind": "long", "t_relative_s": 12789.984901708085}, {"decode_latency_s": 0.0603771432506619, "kv_live_bytes": 462274560, "latency_s": 1.9329254999756813, "n_emitted": 32, "ok": true, "process_footprint_bytes": 87670784, "prompt_kind": "short", "t_relative_s": 12794.994977457915}, {"decode_latency_s": 0.06795380597759504, "kv_live_bytes": 462274560, "latency_s": 2.1761570000089705, "n_emitted": 32, "ok": true, "process_footprint_bytes": 87687168, "prompt_kind": "long", "t_relative_s": 12800.005065958016}, {"decode_latency_s": 0.060665027340292, "kv_live_bytes": 462274560, "latency_s": 1.9422607501037419, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88309760, "prompt_kind": "short", "t_relative_s": 12805.015132083092}, {"decode_latency_s": 0.0687276614626171, "kv_live_bytes": 462274560, "latency_s": 2.2021284578368068, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88375296, "prompt_kind": "long", "t_relative_s": 12810.025213791989}, {"decode_latency_s": 0.060559067729627714, "kv_live_bytes": 462274560, "latency_s": 1.9387327921576798, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88424448, "prompt_kind": "short", "t_relative_s": 12815.03530554194}, {"decode_latency_s": 0.0678450130217243, "kv_live_bytes": 462274560, "latency_s": 2.1730440841056406, "n_emitted": 32, "ok": true, "process_footprint_bytes": 88391680, "prompt_kind": "long", "t_relative_s": 12820.045355082955}, {"decode_latency_s": 0.06045430731319357, "kv_live_bytes": 462274560, "latency_s": 1.9354394171386957, "n_emitted": 32, "ok": true, "process_footprint_bytes": 87113728, "prompt_kind": "short", "t_relative_s": 12825.055429417174}, {"decode_latency_s": 0.06787135025660973, "kv_live_bytes": 462274560, "latency_s": 2.173744291998446, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72712192, "prompt_kind": "long", "t_relative_s": 12830.06550737517}, {"decode_latency_s": 0.06049020444334019, "kv_live_bytes": 462274560, "latency_s": 1.9365677502937615, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72892416, "prompt_kind": "short", "t_relative_s": 12835.075549917296}, {"decode_latency_s": 0.06077203124004882, "kv_live_bytes": 462274560, "latency_s": 1.9458558750338852, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72925184, "prompt_kind": "long", "t_relative_s": 12840.08562262496}, {"decode_latency_s": 0.060522227868204936, "kv_live_bytes": 462274560, "latency_s": 1.9374513337388635, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "short", "t_relative_s": 12845.09570762515}, {"decode_latency_s": 0.06716876302380115, "kv_live_bytes": 462274560, "latency_s": 2.1517697921954095, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73564160, "prompt_kind": "long", "t_relative_s": 12850.10576274991}, {"decode_latency_s": 0.060330545558827, "kv_live_bytes": 462274560, "latency_s": 1.931458625011146, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73056256, "prompt_kind": "short", "t_relative_s": 12855.115825625136}, {"decode_latency_s": 0.06773342836822849, "kv_live_bytes": 462274560, "latency_s": 2.169765332713723, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71090176, "prompt_kind": "long", "t_relative_s": 12860.125913457945}, {"decode_latency_s": 0.06033552606822923, "kv_live_bytes": 462274560, "latency_s": 1.9316654172725976, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72859648, "prompt_kind": "short", "t_relative_s": 12865.13596837502}, {"decode_latency_s": 0.07807435546419583, "kv_live_bytes": 462274560, "latency_s": 2.4998329584486783, "n_emitted": 32, "ok": true, "process_footprint_bytes": 161841152, "prompt_kind": "long", "t_relative_s": 12870.14604908321}, {"decode_latency_s": 0.06055687631305773, "kv_live_bytes": 462274560, "latency_s": 1.938626834191382, "n_emitted": 32, "ok": true, "process_footprint_bytes": 162496512, "prompt_kind": "short", "t_relative_s": 12875.156119625084}, {"decode_latency_s": 0.06792177734314464, "kv_live_bytes": 462274560, "latency_s": 2.175840583164245, "n_emitted": 32, "ok": true, "process_footprint_bytes": 161988608, "prompt_kind": "long", "t_relative_s": 12880.166165833361}, {"decode_latency_s": 0.06034234375692904, "kv_live_bytes": 462274560, "latency_s": 1.931848666165024, "n_emitted": 32, "ok": true, "process_footprint_bytes": 135905280, "prompt_kind": "short", "t_relative_s": 12885.176240125205}, {"decode_latency_s": 0.06427817710209638, "kv_live_bytes": 462274560, "latency_s": 2.0589282917790115, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72073216, "prompt_kind": "long", "t_relative_s": 12890.186340500135}, {"decode_latency_s": 0.06060832811635919, "kv_live_bytes": 462274560, "latency_s": 1.9402456250973046, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72826880, "prompt_kind": "short", "t_relative_s": 12895.196409583092}, {"decode_latency_s": 0.06774494658748154, "kv_live_bytes": 462274560, "latency_s": 2.1706103747710586, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72826880, "prompt_kind": "long", "t_relative_s": 12900.206475958228}, {"decode_latency_s": 0.060474122408777475, "kv_live_bytes": 462274560, "latency_s": 1.9360447502695024, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72859648, "prompt_kind": "short", "t_relative_s": 12905.216559207998}, {"decode_latency_s": 0.06763907552522141, "kv_live_bytes": 462274560, "latency_s": 2.166478833183646, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73367552, "prompt_kind": "long", "t_relative_s": 12910.226625167299}, {"decode_latency_s": 0.06063489321968518, "kv_live_bytes": 462274560, "latency_s": 1.9412282924167812, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72712192, "prompt_kind": "short", "t_relative_s": 12915.236675917171}, {"decode_latency_s": 0.06043409115227405, "kv_live_bytes": 462274560, "latency_s": 1.9348486671224236, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "long", "t_relative_s": 12920.246725500096}, {"decode_latency_s": 0.060302852842141874, "kv_live_bytes": 462274560, "latency_s": 1.9305790001526475, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73547776, "prompt_kind": "short", "t_relative_s": 12925.256813958287}, {"decode_latency_s": 0.06827020439959597, "kv_live_bytes": 462274560, "latency_s": 2.1865135827101767, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72908800, "prompt_kind": "long", "t_relative_s": 12930.266893499997}, {"decode_latency_s": 0.060473492194432765, "kv_live_bytes": 462274560, "latency_s": 1.9360456657595932, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73564160, "prompt_kind": "short", "t_relative_s": 12935.276966625359}, {"decode_latency_s": 0.06788657550350763, "kv_live_bytes": 462274560, "latency_s": 2.174618124961853, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "long", "t_relative_s": 12940.28703270806}, {"decode_latency_s": 0.06708126043668017, "kv_live_bytes": 462274560, "latency_s": 2.1485397503711283, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72105984, "prompt_kind": "short", "t_relative_s": 12945.288252166938}, {"decode_latency_s": 0.06062224609195255, "kv_live_bytes": 462274560, "latency_s": 1.9408700419589877, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73203712, "prompt_kind": "long", "t_relative_s": 12950.29830520833}, {"decode_latency_s": 0.06273550259356853, "kv_live_bytes": 462274560, "latency_s": 2.0090939588844776, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72695808, "prompt_kind": "short", "t_relative_s": 12955.305431208108}, {"decode_latency_s": 0.06843810025020503, "kv_live_bytes": 462274560, "latency_s": 2.1916692908853292, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72531968, "prompt_kind": "long", "t_relative_s": 12960.315479082987}, {"decode_latency_s": 0.060388678379240446, "kv_live_bytes": 462274560, "latency_s": 1.93334695789963, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73154560, "prompt_kind": "short", "t_relative_s": 12965.325531250332}, {"decode_latency_s": 0.06719903384509962, "kv_live_bytes": 462274560, "latency_s": 2.152428791858256, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73744384, "prompt_kind": "long", "t_relative_s": 12970.335598333273}, {"decode_latency_s": 0.060804385429946706, "kv_live_bytes": 462274560, "latency_s": 1.9465377503074706, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73842688, "prompt_kind": "short", "t_relative_s": 12975.345636500046}, {"decode_latency_s": 0.06879642838612199, "kv_live_bytes": 462274560, "latency_s": 2.203975958749652, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72712192, "prompt_kind": "long", "t_relative_s": 12980.355833792128}, {"decode_latency_s": 0.06070422787161078, "kv_live_bytes": 462274560, "latency_s": 1.9433449157513678, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72761344, "prompt_kind": "short", "t_relative_s": 12985.365904417355}, {"decode_latency_s": 0.06829987891251221, "kv_live_bytes": 462274560, "latency_s": 2.1876917500048876, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72761344, "prompt_kind": "long", "t_relative_s": 12990.373242083006}, {"decode_latency_s": 0.06068709504324943, "kv_live_bytes": 462274560, "latency_s": 1.9429439157247543, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72876032, "prompt_kind": "short", "t_relative_s": 12995.383325292263}, {"decode_latency_s": 0.06744751821679529, "kv_live_bytes": 462274560, "latency_s": 2.1603085827082396, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72335360, "prompt_kind": "long", "t_relative_s": 13000.39338662522}, {"decode_latency_s": 0.07752461846393999, "kv_live_bytes": 462274560, "latency_s": 2.4816520418971777, "n_emitted": 32, "ok": true, "process_footprint_bytes": 369082368, "prompt_kind": "short", "t_relative_s": 13005.403439000249}, {"decode_latency_s": 0.06837092056230176, "kv_live_bytes": 462274560, "latency_s": 2.1899695000611246, "n_emitted": 32, "ok": true, "process_footprint_bytes": 369754112, "prompt_kind": "long", "t_relative_s": 13010.413508875296}, {"decode_latency_s": 0.06700592706329189, "kv_live_bytes": 462274560, "latency_s": 2.145571666304022, "n_emitted": 32, "ok": true, "process_footprint_bytes": 370638848, "prompt_kind": "short", "t_relative_s": 13015.423576083034}, {"decode_latency_s": 0.06794668622023892, "kv_live_bytes": 462274560, "latency_s": 2.1764519168064, "n_emitted": 32, "ok": true, "process_footprint_bytes": 370671616, "prompt_kind": "long", "t_relative_s": 13020.43364199996}, {"decode_latency_s": 0.060884291684487835, "kv_live_bytes": 462274560, "latency_s": 1.9491459587588906, "n_emitted": 32, "ok": true, "process_footprint_bytes": 371245056, "prompt_kind": "short", "t_relative_s": 13025.443705458194}, {"decode_latency_s": 0.06771327213209588, "kv_live_bytes": 462274560, "latency_s": 2.169195707887411, "n_emitted": 32, "ok": true, "process_footprint_bytes": 370393088, "prompt_kind": "long", "t_relative_s": 13030.453761416953}, {"decode_latency_s": 0.06112135809962638, "kv_live_bytes": 462274560, "latency_s": 1.9569397079758346, "n_emitted": 32, "ok": true, "process_footprint_bytes": 133791744, "prompt_kind": "short", "t_relative_s": 13035.457929375116}, {"decode_latency_s": 0.06770039063121658, "kv_live_bytes": 462274560, "latency_s": 2.168245708104223, "n_emitted": 32, "ok": true, "process_footprint_bytes": 137527296, "prompt_kind": "long", "t_relative_s": 13040.468009083066}, {"decode_latency_s": 0.060468506504548714, "kv_live_bytes": 462274560, "latency_s": 1.9358442081138492, "n_emitted": 32, "ok": true, "process_footprint_bytes": 137592832, "prompt_kind": "short", "t_relative_s": 13045.478080042172}, {"decode_latency_s": 0.06781785808561835, "kv_live_bytes": 462274560, "latency_s": 2.1727734999731183, "n_emitted": 32, "ok": true, "process_footprint_bytes": 137625600, "prompt_kind": "long", "t_relative_s": 13050.48816516716}, {"decode_latency_s": 0.0607548385305563, "kv_live_bytes": 462274560, "latency_s": 1.9449351248331368, "n_emitted": 32, "ok": true, "process_footprint_bytes": 138199040, "prompt_kind": "short", "t_relative_s": 13055.498260917142}, {"decode_latency_s": 0.06845624087145552, "kv_live_bytes": 462274560, "latency_s": 2.1930207079276443, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75546624, "prompt_kind": "long", "t_relative_s": 13060.508312625345}, {"decode_latency_s": 0.06032545049674809, "kv_live_bytes": 462274560, "latency_s": 1.9313397924415767, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73564160, "prompt_kind": "short", "t_relative_s": 13065.518375874963}, {"decode_latency_s": 0.07024892450135667, "kv_live_bytes": 462274560, "latency_s": 2.2503557498566806, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71303168, "prompt_kind": "long", "t_relative_s": 13070.528467332944}, {"decode_latency_s": 0.06459480337798595, "kv_live_bytes": 462274560, "latency_s": 2.0686736246570945, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67502080, "prompt_kind": "short", "t_relative_s": 13075.538553458173}, {"decode_latency_s": 0.0686629895499209, "kv_live_bytes": 462274560, "latency_s": 2.1995620410889387, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71630848, "prompt_kind": "long", "t_relative_s": 13080.548636124935}, {"decode_latency_s": 0.06106135545996949, "kv_live_bytes": 462274560, "latency_s": 1.9548945422284305, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72499200, "prompt_kind": "short", "t_relative_s": 13085.55871587526}, {"decode_latency_s": 0.0678782747418154, "kv_live_bytes": 462274560, "latency_s": 2.17377462470904, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73711616, "prompt_kind": "long", "t_relative_s": 13090.568826333154}, {"decode_latency_s": 0.060610473970882595, "kv_live_bytes": 462274560, "latency_s": 1.9403874170966446, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72515584, "prompt_kind": "short", "t_relative_s": 13095.578881542198}, {"decode_latency_s": 0.06039452212280594, "kv_live_bytes": 462274560, "latency_s": 1.9336641668342054, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73220096, "prompt_kind": "long", "t_relative_s": 13100.588940167334}, {"decode_latency_s": 0.06027887237723917, "kv_live_bytes": 462274560, "latency_s": 1.9298136252909899, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71696384, "prompt_kind": "short", "t_relative_s": 13105.599014250096}, {"decode_latency_s": 0.06817038281587884, "kv_live_bytes": 462274560, "latency_s": 2.1831988748162985, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73416704, "prompt_kind": "long", "t_relative_s": 13110.609135458246}, {"decode_latency_s": 0.060709695317200385, "kv_live_bytes": 462274560, "latency_s": 1.9436309589073062, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74055680, "prompt_kind": "short", "t_relative_s": 13115.619192375336}, {"decode_latency_s": 0.06834313541185111, "kv_live_bytes": 462274560, "latency_s": 2.1894141668453813, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74088448, "prompt_kind": "long", "t_relative_s": 13120.629282125272}, {"decode_latency_s": 0.060960315095144324, "kv_live_bytes": 462274560, "latency_s": 1.951992165762931, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69353472, "prompt_kind": "short", "t_relative_s": 13125.639371875208}, {"decode_latency_s": 0.06816426433215383, "kv_live_bytes": 462274560, "latency_s": 2.1839180001989007, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72531968, "prompt_kind": "long", "t_relative_s": 13130.649413333274}, {"decode_latency_s": 0.06060037630959414, "kv_live_bytes": 462274560, "latency_s": 1.9404917089268565, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73170944, "prompt_kind": "short", "t_relative_s": 13135.650595291983}, {"decode_latency_s": 0.06838927212811541, "kv_live_bytes": 462274560, "latency_s": 2.189988249912858, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72204288, "prompt_kind": "long", "t_relative_s": 13140.660665583331}, {"decode_latency_s": 0.06062749218835961, "kv_live_bytes": 462274560, "latency_s": 1.9410836668685079, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72876032, "prompt_kind": "short", "t_relative_s": 13145.670746208169}, {"decode_latency_s": 0.06508869530807715, "kv_live_bytes": 462274560, "latency_s": 2.083789417054504, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70205440, "prompt_kind": "long", "t_relative_s": 13150.680844082963}, {"decode_latency_s": 0.06313315361330751, "kv_live_bytes": 462274560, "latency_s": 2.021301709115505, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67993600, "prompt_kind": "short", "t_relative_s": 13155.690906083211}, {"decode_latency_s": 0.0808047630416695, "kv_live_bytes": 462274560, "latency_s": 2.5873335828073323, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68304896, "prompt_kind": "long", "t_relative_s": 13160.70099579217}, {"decode_latency_s": 0.060509528644615784, "kv_live_bytes": 462274560, "latency_s": 1.9370786249637604, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68452352, "prompt_kind": "short", "t_relative_s": 13166.976198417135}, {"decode_latency_s": 0.06839648699678946, "kv_live_bytes": 462274560, "latency_s": 2.191176749765873, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69812224, "prompt_kind": "long", "t_relative_s": 13171.986247667111}, {"decode_latency_s": 0.06047572265379131, "kv_live_bytes": 462274560, "latency_s": 1.9361036247573793, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70205440, "prompt_kind": "short", "t_relative_s": 13176.996327083092}, {"decode_latency_s": 0.06893590102845337, "kv_live_bytes": 462274560, "latency_s": 2.2080909996293485, "n_emitted": 32, "ok": true, "process_footprint_bytes": 64700416, "prompt_kind": "long", "t_relative_s": 13182.006427125074}, {"decode_latency_s": 0.061041484368615784, "kv_live_bytes": 462274560, "latency_s": 1.9542019157670438, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66158592, "prompt_kind": "short", "t_relative_s": 13187.016527249943}, {"decode_latency_s": 0.06834437632642221, "kv_live_bytes": 462274560, "latency_s": 2.188585708849132, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68616192, "prompt_kind": "long", "t_relative_s": 13192.026584208012}, {"decode_latency_s": 0.06034713018743787, "kv_live_bytes": 462274560, "latency_s": 1.932048958260566, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69779456, "prompt_kind": "short", "t_relative_s": 13197.03665400017}, {"decode_latency_s": 0.06367287760076579, "kv_live_bytes": 462274560, "latency_s": 2.0392832499928772, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70631424, "prompt_kind": "long", "t_relative_s": 13202.046711042058}, {"decode_latency_s": 0.0608373854047386, "kv_live_bytes": 462274560, "latency_s": 1.947643875144422, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70713344, "prompt_kind": "short", "t_relative_s": 13207.0567871253}, {"decode_latency_s": 0.06868080858839676, "kv_live_bytes": 462274560, "latency_s": 2.1998633327893913, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70369280, "prompt_kind": "long", "t_relative_s": 13212.066871250048}, {"decode_latency_s": 0.06727582159510348, "kv_live_bytes": 462274560, "latency_s": 2.154875624924898, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69533696, "prompt_kind": "short", "t_relative_s": 13217.076924417168}, {"decode_latency_s": 0.06830955078476109, "kv_live_bytes": 462274560, "latency_s": 2.1881591249257326, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70746112, "prompt_kind": "long", "t_relative_s": 13222.086976458319}, {"decode_latency_s": 0.061402053412166424, "kv_live_bytes": 462274560, "latency_s": 1.9657944589853287, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70139904, "prompt_kind": "short", "t_relative_s": 13227.097065958194}, {"decode_latency_s": 0.06876401694898959, "kv_live_bytes": 462274560, "latency_s": 2.202897499781102, "n_emitted": 32, "ok": true, "process_footprint_bytes": 60964864, "prompt_kind": "long", "t_relative_s": 13232.10717533296}, {"decode_latency_s": 0.06150485416583251, "kv_live_bytes": 462274560, "latency_s": 1.9690705831162632, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67305472, "prompt_kind": "short", "t_relative_s": 13237.117252292112}, {"decode_latency_s": 0.06340702863235492, "kv_live_bytes": 462274560, "latency_s": 2.0302725001238286, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67420160, "prompt_kind": "long", "t_relative_s": 13242.121431500185}, {"decode_latency_s": 0.061551479157060385, "kv_live_bytes": 462274560, "latency_s": 1.9706897079013288, "n_emitted": 32, "ok": true, "process_footprint_bytes": 65667072, "prompt_kind": "short", "t_relative_s": 13247.131489207968}, {"decode_latency_s": 0.15072389971464872, "kv_live_bytes": 462274560, "latency_s": 4.933303042314947, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66453504, "prompt_kind": "long", "t_relative_s": 13252.141570042353}, {"decode_latency_s": 0.060514816403156146, "kv_live_bytes": 462274560, "latency_s": 1.9372880421578884, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67485696, "prompt_kind": "short", "t_relative_s": 13262.399750542361}, {"decode_latency_s": 0.0680208580888575, "kv_live_bytes": 462274560, "latency_s": 2.1795350410975516, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68616192, "prompt_kind": "long", "t_relative_s": 13267.409806250129}, {"decode_latency_s": 0.06043172006320674, "kv_live_bytes": 462274560, "latency_s": 1.934680209029466, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69206016, "prompt_kind": "short", "t_relative_s": 13272.419869417325}, {"decode_latency_s": 0.06840775390446652, "kv_live_bytes": 462274560, "latency_s": 2.1911447909660637, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69320704, "prompt_kind": "long", "t_relative_s": 13277.429934666958}, {"decode_latency_s": 0.06091985806415323, "kv_live_bytes": 462274560, "latency_s": 1.9502418329939246, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69435392, "prompt_kind": "short", "t_relative_s": 13282.439996832982}, {"decode_latency_s": 0.06831155989493709, "kv_live_bytes": 462274560, "latency_s": 2.188370666000992, "n_emitted": 32, "ok": true, "process_footprint_bytes": 68911104, "prompt_kind": "long", "t_relative_s": 13287.450106667355}, {"decode_latency_s": 0.060635338537395, "kv_live_bytes": 462274560, "latency_s": 1.9411270003765821, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69468160, "prompt_kind": "short", "t_relative_s": 13292.46020400012}, {"decode_latency_s": 0.06051694921916351, "kv_live_bytes": 462274560, "latency_s": 1.9376977919600904, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69566464, "prompt_kind": "long", "t_relative_s": 13297.470258875284}, {"decode_latency_s": 0.06070329947397113, "kv_live_bytes": 462274560, "latency_s": 1.9437972498126328, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70189056, "prompt_kind": "short", "t_relative_s": 13302.475327958353}, {"decode_latency_s": 0.06787632685154676, "kv_live_bytes": 462274560, "latency_s": 2.1745457500219345, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70320128, "prompt_kind": "long", "t_relative_s": 13307.485409750137}, {"decode_latency_s": 0.06694727734429762, "kv_live_bytes": 462274560, "latency_s": 2.1448355419561267, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70549504, "prompt_kind": "short", "t_relative_s": 13312.495461833198}, {"decode_latency_s": 0.06739726039813831, "kv_live_bytes": 462274560, "latency_s": 2.1594184590503573, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70615040, "prompt_kind": "long", "t_relative_s": 13317.505561542232}, {"decode_latency_s": 0.060846638021757826, "kv_live_bytes": 462274560, "latency_s": 1.9486625422723591, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71188480, "prompt_kind": "short", "t_relative_s": 13322.515645000152}, {"decode_latency_s": 0.06807456380920485, "kv_live_bytes": 462274560, "latency_s": 2.181027582846582, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70631424, "prompt_kind": "long", "t_relative_s": 13327.525696583092}, {"decode_latency_s": 0.06031286455981899, "kv_live_bytes": 462274560, "latency_s": 1.9309257920831442, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71188480, "prompt_kind": "short", "t_relative_s": 13332.53575362498}, {"decode_latency_s": 0.06831378905917518, "kv_live_bytes": 462274560, "latency_s": 2.1883934582583606, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71303168, "prompt_kind": "long", "t_relative_s": 13337.545845542103}, {"decode_latency_s": 0.06072850522468798, "kv_live_bytes": 462274560, "latency_s": 1.944171333219856, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71335936, "prompt_kind": "short", "t_relative_s": 13342.555904916953}, {"decode_latency_s": 0.06855861586518586, "kv_live_bytes": 462274560, "latency_s": 2.1963787502609193, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71385088, "prompt_kind": "long", "t_relative_s": 13347.565960583277}, {"decode_latency_s": 0.060570610701688565, "kv_live_bytes": 462274560, "latency_s": 1.939016207586974, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71925760, "prompt_kind": "short", "t_relative_s": 13352.576037333347}, {"decode_latency_s": 0.06884485547197983, "kv_live_bytes": 462274560, "latency_s": 2.2051807092502713, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69894144, "prompt_kind": "long", "t_relative_s": 13357.586091875099}, {"decode_latency_s": 0.0605033775937045, "kv_live_bytes": 462274560, "latency_s": 1.9370726249180734, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70483968, "prompt_kind": "short", "t_relative_s": 13362.596165542025}, {"decode_latency_s": 0.06834649867960252, "kv_live_bytes": 462274560, "latency_s": 2.1888320832513273, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71286784, "prompt_kind": "long", "t_relative_s": 13367.606236916967}, {"decode_latency_s": 0.06061874871375039, "kv_live_bytes": 462274560, "latency_s": 1.9405922079458833, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71319552, "prompt_kind": "short", "t_relative_s": 13372.612152083311}, {"decode_latency_s": 0.06788903252163436, "kv_live_bytes": 462274560, "latency_s": 2.174688957631588, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71467008, "prompt_kind": "long", "t_relative_s": 13377.622206000146}, {"decode_latency_s": 0.06059355598699767, "kv_live_bytes": 462274560, "latency_s": 1.9398132497444749, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72024064, "prompt_kind": "short", "t_relative_s": 13382.63228479214}, {"decode_latency_s": 0.06821729037619662, "kv_live_bytes": 462274560, "latency_s": 2.1849212497472763, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71942144, "prompt_kind": "long", "t_relative_s": 13387.642344791908}, {"decode_latency_s": 0.060535033873748034, "kv_live_bytes": 462274560, "latency_s": 1.9380284999497235, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71237632, "prompt_kind": "short", "t_relative_s": 13392.652410792187}, {"decode_latency_s": 0.06818375518196262, "kv_live_bytes": 462274560, "latency_s": 2.1835214579477906, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71909376, "prompt_kind": "long", "t_relative_s": 13397.662498916965}, {"decode_latency_s": 0.060515139310155064, "kv_live_bytes": 462274560, "latency_s": 1.9373204591684043, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71974912, "prompt_kind": "short", "t_relative_s": 13402.672567958012}, {"decode_latency_s": 0.06812706771597732, "kv_live_bytes": 462274560, "latency_s": 2.1824351670220494, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72122368, "prompt_kind": "long", "t_relative_s": 13407.682661625091}, {"decode_latency_s": 0.060455350248957984, "kv_live_bytes": 462274560, "latency_s": 1.9353781249374151, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72630272, "prompt_kind": "short", "t_relative_s": 13412.6927424171}, {"decode_latency_s": 0.06807487628248055, "kv_live_bytes": 462274560, "latency_s": 2.1802332089282572, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72744960, "prompt_kind": "long", "t_relative_s": 13417.702794750221}, {"decode_latency_s": 0.06072395050432533, "kv_live_bytes": 462274560, "latency_s": 1.944072250276804, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72220672, "prompt_kind": "short", "t_relative_s": 13422.712847582996}, {"decode_latency_s": 0.06885069921554532, "kv_live_bytes": 462274560, "latency_s": 2.206698624882847, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72810496, "prompt_kind": "long", "t_relative_s": 13427.722929083277}, {"decode_latency_s": 0.06666032294742763, "kv_live_bytes": 462274560, "latency_s": 2.135297750122845, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72237056, "prompt_kind": "short", "t_relative_s": 13432.732974792365}, {"decode_latency_s": 0.06809702735336032, "kv_live_bytes": 462274560, "latency_s": 2.1817728341557086, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72269824, "prompt_kind": "long", "t_relative_s": 13437.743032750208}, {"decode_latency_s": 0.06418879037664738, "kv_live_bytes": 462274560, "latency_s": 2.055903166066855, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72712192, "prompt_kind": "short", "t_relative_s": 13442.753135167062}, {"decode_latency_s": 0.0679862304677954, "kv_live_bytes": 462274560, "latency_s": 2.1776399998925626, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72761344, "prompt_kind": "long", "t_relative_s": 13447.763169958256}, {"decode_latency_s": 0.060286947918939404, "kv_live_bytes": 462274560, "latency_s": 1.9301109169609845, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73089024, "prompt_kind": "short", "t_relative_s": 13452.773211667314}, {"decode_latency_s": 0.06767002865672112, "kv_live_bytes": 462274560, "latency_s": 2.1679591671563685, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73154560, "prompt_kind": "long", "t_relative_s": 13457.783291208092}, {"decode_latency_s": 0.06078774741035886, "kv_live_bytes": 462274560, "latency_s": 1.946037500165403, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73203712, "prompt_kind": "short", "t_relative_s": 13462.793361958116}, {"decode_latency_s": 0.06777730341127608, "kv_live_bytes": 462274560, "latency_s": 2.171828791964799, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73121792, "prompt_kind": "long", "t_relative_s": 13467.80340812495}, {"decode_latency_s": 0.06064391665859148, "kv_live_bytes": 462274560, "latency_s": 1.9414180838502944, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73760768, "prompt_kind": "short", "t_relative_s": 13472.813473875169}, {"decode_latency_s": 0.06774003646569327, "kv_live_bytes": 462274560, "latency_s": 2.169948250055313, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73252864, "prompt_kind": "long", "t_relative_s": 13477.823513166979}, {"decode_latency_s": 0.06444701952568721, "kv_live_bytes": 462274560, "latency_s": 2.063535540830344, "n_emitted": 32, "ok": true, "process_footprint_bytes": 62013440, "prompt_kind": "short", "t_relative_s": 13482.833564500324}, {"decode_latency_s": 0.06048873958934564, "kv_live_bytes": 462274560, "latency_s": 1.9369863751344383, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69730304, "prompt_kind": "long", "t_relative_s": 13487.843679457903}, {"decode_latency_s": 0.06092832422291394, "kv_live_bytes": 462274560, "latency_s": 1.950518083292991, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71892992, "prompt_kind": "short", "t_relative_s": 13492.853759250138}, {"decode_latency_s": 0.07563639713043813, "kv_live_bytes": 462274560, "latency_s": 2.421140917111188, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69779456, "prompt_kind": "long", "t_relative_s": 13497.863797375001}, {"decode_latency_s": 0.06025573566148523, "kv_live_bytes": 462274560, "latency_s": 1.929055250249803, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70385664, "prompt_kind": "short", "t_relative_s": 13504.988724207971}, {"decode_latency_s": 0.06743561718030833, "kv_live_bytes": 462274560, "latency_s": 2.160733124706894, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70926336, "prompt_kind": "long", "t_relative_s": 13509.998802792281}, {"decode_latency_s": 0.06740822138090152, "kv_live_bytes": 462274560, "latency_s": 2.1586557500995696, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71499776, "prompt_kind": "short", "t_relative_s": 13515.008874417283}, {"decode_latency_s": 0.06821707294147927, "kv_live_bytes": 462274560, "latency_s": 2.185502375010401, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71565312, "prompt_kind": "long", "t_relative_s": 13520.018932458013}, {"decode_latency_s": 0.0608258163993014, "kv_live_bytes": 462274560, "latency_s": 1.9473236673511565, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71139328, "prompt_kind": "short", "t_relative_s": 13525.02898195805}, {"decode_latency_s": 0.06808383984025568, "kv_live_bytes": 462274560, "latency_s": 2.1805908330716193, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71106560, "prompt_kind": "long", "t_relative_s": 13530.039062667172}, {"decode_latency_s": 0.060720321591361426, "kv_live_bytes": 462274560, "latency_s": 1.9439695831388235, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71745536, "prompt_kind": "short", "t_relative_s": 13535.049113166984}, {"decode_latency_s": 0.06758938799612224, "kv_live_bytes": 462274560, "latency_s": 2.165010791271925, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71778304, "prompt_kind": "long", "t_relative_s": 13540.05919912504}, {"decode_latency_s": 0.06058278256386984, "kv_live_bytes": 462274560, "latency_s": 1.939411458093673, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72286208, "prompt_kind": "short", "t_relative_s": 13545.069297125097}, {"decode_latency_s": 0.06764643489441369, "kv_live_bytes": 462274560, "latency_s": 2.1669144169427454, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72204288, "prompt_kind": "long", "t_relative_s": 13550.079347542021}, {"decode_latency_s": 0.06073447788367048, "kv_live_bytes": 462274560, "latency_s": 1.9443652504123747, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72286208, "prompt_kind": "short", "t_relative_s": 13555.089431458153}, {"decode_latency_s": 0.06769679946592078, "kv_live_bytes": 462274560, "latency_s": 2.1683890828862786, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71778304, "prompt_kind": "long", "t_relative_s": 13560.099519958254}, {"decode_latency_s": 0.0604198906221427, "kv_live_bytes": 462274560, "latency_s": 1.9344026669859886, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72433664, "prompt_kind": "short", "t_relative_s": 13565.10961775016}, {"decode_latency_s": 0.06770146875351202, "kv_live_bytes": 462274560, "latency_s": 2.168960167095065, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72450048, "prompt_kind": "long", "t_relative_s": 13570.119701832999}, {"decode_latency_s": 0.060524898435687646, "kv_live_bytes": 462274560, "latency_s": 1.9375513335689902, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72974336, "prompt_kind": "short", "t_relative_s": 13575.125117749907}, {"decode_latency_s": 0.06756424740888178, "kv_live_bytes": 462274560, "latency_s": 2.164104249794036, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72892416, "prompt_kind": "long", "t_relative_s": 13580.135163708124}, {"decode_latency_s": 0.06035254820017144, "kv_live_bytes": 462274560, "latency_s": 1.932146499864757, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72974336, "prompt_kind": "short", "t_relative_s": 13585.145233792253}, {"decode_latency_s": 0.06757920053496491, "kv_live_bytes": 462274560, "latency_s": 2.1643073330633342, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72187904, "prompt_kind": "long", "t_relative_s": 13590.155304000247}, {"decode_latency_s": 0.060335776026477106, "kv_live_bytes": 462274560, "latency_s": 1.9316463340073824, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72859648, "prompt_kind": "short", "t_relative_s": 13595.165355958045}, {"decode_latency_s": 0.0679352356819436, "kv_live_bytes": 462274560, "latency_s": 2.176263333763927, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72892416, "prompt_kind": "long", "t_relative_s": 13600.175445500296}, {"decode_latency_s": 0.06059872396872379, "kv_live_bytes": 462274560, "latency_s": 1.939953625202179, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73482240, "prompt_kind": "short", "t_relative_s": 13605.18554437533}, {"decode_latency_s": 0.06820099998731166, "kv_live_bytes": 462274560, "latency_s": 2.1848720000125468, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73400320, "prompt_kind": "long", "t_relative_s": 13610.195588333067}, {"decode_latency_s": 0.06031697134312708, "kv_live_bytes": 462274560, "latency_s": 1.9310281672514975, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73515008, "prompt_kind": "short", "t_relative_s": 13615.205658792052}, {"decode_latency_s": 0.06798730859009083, "kv_live_bytes": 462274560, "latency_s": 2.1774609577842057, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73465856, "prompt_kind": "long", "t_relative_s": 13620.215740292333}, {"decode_latency_s": 0.0603341080568498, "kv_live_bytes": 462274560, "latency_s": 1.9316010419279337, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73564160, "prompt_kind": "short", "t_relative_s": 13625.225795292296}, {"decode_latency_s": 0.06744127735146321, "kv_live_bytes": 462274560, "latency_s": 2.160275917034596, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73629696, "prompt_kind": "long", "t_relative_s": 13630.235873917118}, {"decode_latency_s": 0.060624568999628536, "kv_live_bytes": 462274560, "latency_s": 1.940745749976486, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73629696, "prompt_kind": "short", "t_relative_s": 13635.24596487498}, {"decode_latency_s": 0.06839512371516321, "kv_live_bytes": 462274560, "latency_s": 2.1905703339725733, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73531392, "prompt_kind": "long", "t_relative_s": 13640.25600916706}, {"decode_latency_s": 0.060452906240243465, "kv_live_bytes": 462274560, "latency_s": 1.9353815829381347, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73744384, "prompt_kind": "short", "t_relative_s": 13645.266088000033}, {"decode_latency_s": 0.06810787109134253, "kv_live_bytes": 462274560, "latency_s": 2.1812980421818793, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73777152, "prompt_kind": "long", "t_relative_s": 13650.276156499982}, {"decode_latency_s": 0.060625914062256925, "kv_live_bytes": 462274560, "latency_s": 1.9408958749845624, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73809920, "prompt_kind": "short", "t_relative_s": 13655.286202125251}, {"decode_latency_s": 0.06800227606436238, "kv_live_bytes": 462274560, "latency_s": 2.178674125112593, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73875456, "prompt_kind": "long", "t_relative_s": 13660.296285250224}, {"decode_latency_s": 0.06064756769046653, "kv_live_bytes": 462274560, "latency_s": 1.9415174159221351, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73875456, "prompt_kind": "short", "t_relative_s": 13665.306351708248}, {"decode_latency_s": 0.06789521615428384, "kv_live_bytes": 462274560, "latency_s": 2.174719417002052, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73891840, "prompt_kind": "long", "t_relative_s": 13670.316400958225}, {"decode_latency_s": 0.060443217429565266, "kv_live_bytes": 462274560, "latency_s": 1.935079500079155, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73777152, "prompt_kind": "short", "t_relative_s": 13675.326465375256}, {"decode_latency_s": 0.06840068490419071, "kv_live_bytes": 462274560, "latency_s": 2.190851209219545, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73875456, "prompt_kind": "long", "t_relative_s": 13680.33653987525}, {"decode_latency_s": 0.06061215103545692, "kv_live_bytes": 462274560, "latency_s": 1.9404091672040522, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74383360, "prompt_kind": "short", "t_relative_s": 13685.346611124929}, {"decode_latency_s": 0.06750149479194079, "kv_live_bytes": 462274560, "latency_s": 2.1624837503768504, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74514432, "prompt_kind": "long", "t_relative_s": 13690.356717917137}, {"decode_latency_s": 0.06060152084683068, "kv_live_bytes": 462274560, "latency_s": 1.9400392496027052, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74514432, "prompt_kind": "short", "t_relative_s": 13695.366797875147}, {"decode_latency_s": 0.06798668490955606, "kv_live_bytes": 462274560, "latency_s": 2.177874334156513, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74448896, "prompt_kind": "long", "t_relative_s": 13700.37685341714}, {"decode_latency_s": 0.06033330337959342, "kv_live_bytes": 462274560, "latency_s": 1.931554083712399, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74465280, "prompt_kind": "short", "t_relative_s": 13705.386900499929}, {"decode_latency_s": 0.06746480597939808, "kv_live_bytes": 462274560, "latency_s": 2.1605225834064186, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74514432, "prompt_kind": "long", "t_relative_s": 13710.396983667277}, {"decode_latency_s": 0.060710145815392025, "kv_live_bytes": 462274560, "latency_s": 1.9435334159061313, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74596352, "prompt_kind": "short", "t_relative_s": 13715.407015624922}, {"decode_latency_s": 0.06752490362850949, "kv_live_bytes": 462274560, "latency_s": 2.163277416024357, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74645504, "prompt_kind": "long", "t_relative_s": 13720.417076291982}, {"decode_latency_s": 0.06080925000424031, "kv_live_bytes": 462274560, "latency_s": 1.946700416970998, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74645504, "prompt_kind": "short", "t_relative_s": 13725.427143041976}, {"decode_latency_s": 0.06813137499557342, "kv_live_bytes": 462274560, "latency_s": 2.1820510840043426, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74481664, "prompt_kind": "long", "t_relative_s": 13730.437194125261}, {"decode_latency_s": 0.060474677055026405, "kv_live_bytes": 462274560, "latency_s": 1.9360984582453966, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74498048, "prompt_kind": "short", "t_relative_s": 13735.447253542021}, {"decode_latency_s": 0.06794856381020509, "kv_live_bytes": 462274560, "latency_s": 2.176333792041987, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74547200, "prompt_kind": "long", "t_relative_s": 13740.45734583307}, {"decode_latency_s": 0.06084742055099923, "kv_live_bytes": 462274560, "latency_s": 1.9479643749073148, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75071488, "prompt_kind": "short", "t_relative_s": 13745.467411458027}, {"decode_latency_s": 0.06782408725121059, "kv_live_bytes": 462274560, "latency_s": 2.172793624922633, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75186176, "prompt_kind": "long", "t_relative_s": 13750.477445542347}, {"decode_latency_s": 0.06076531381404493, "kv_live_bytes": 462274560, "latency_s": 1.9453078326769173, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75087872, "prompt_kind": "short", "t_relative_s": 13755.487532875035}, {"decode_latency_s": 0.0683043463359354, "kv_live_bytes": 462274560, "latency_s": 2.1882740827277303, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74301440, "prompt_kind": "long", "t_relative_s": 13760.49758854229}, {"decode_latency_s": 0.06035394403443206, "kv_live_bytes": 462274560, "latency_s": 1.9322002921253443, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73596928, "prompt_kind": "short", "t_relative_s": 13765.507656875066}, {"decode_latency_s": 0.06762645444541704, "kv_live_bytes": 462274560, "latency_s": 2.1655843751505017, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74448896, "prompt_kind": "long", "t_relative_s": 13770.517740167212}, {"decode_latency_s": 0.0608459427166963, "kv_live_bytes": 462274560, "latency_s": 1.9479233329184353, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74481664, "prompt_kind": "short", "t_relative_s": 13775.527777832933}, {"decode_latency_s": 0.06727701562340371, "kv_live_bytes": 462274560, "latency_s": 2.155203207861632, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74514432, "prompt_kind": "long", "t_relative_s": 13780.537833500188}, {"decode_latency_s": 0.06325515753997024, "kv_live_bytes": 462274560, "latency_s": 2.0251493747346103, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72630272, "prompt_kind": "short", "t_relative_s": 13785.547886332963}, {"decode_latency_s": 0.06758977471326943, "kv_live_bytes": 462274560, "latency_s": 2.164970957674086, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73449472, "prompt_kind": "long", "t_relative_s": 13790.557942500338}, {"decode_latency_s": 0.06037636849214323, "kv_live_bytes": 462274560, "latency_s": 1.932940624654293, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73170944, "prompt_kind": "short", "t_relative_s": 13795.568019500002}, {"decode_latency_s": 0.06773624350898899, "kv_live_bytes": 462274560, "latency_s": 2.1692174579948187, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73318400, "prompt_kind": "long", "t_relative_s": 13800.578091125004}, {"decode_latency_s": 0.06073764062603004, "kv_live_bytes": 462274560, "latency_s": 1.9444977920502424, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73908224, "prompt_kind": "short", "t_relative_s": 13805.58812637534}, {"decode_latency_s": 0.0676922747225035, "kv_live_bytes": 462274560, "latency_s": 2.1688627498224378, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73973760, "prompt_kind": "long", "t_relative_s": 13810.598214332946}, {"decode_latency_s": 0.060915759138879366, "kv_live_bytes": 462274560, "latency_s": 1.9501343332231045, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74530816, "prompt_kind": "short", "t_relative_s": 13815.608322083019}, {"decode_latency_s": 0.06054839324497152, "kv_live_bytes": 462274560, "latency_s": 1.9384458749555051, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74039296, "prompt_kind": "long", "t_relative_s": 13820.618364125025}, {"decode_latency_s": 0.060534274714882486, "kv_live_bytes": 462274560, "latency_s": 1.9379958328790963, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74039296, "prompt_kind": "short", "t_relative_s": 13825.62843129225}, {"decode_latency_s": 0.06803252472309396, "kv_live_bytes": 462274560, "latency_s": 2.178990458138287, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74055680, "prompt_kind": "long", "t_relative_s": 13830.638512125239}, {"decode_latency_s": 0.06051016537821852, "kv_live_bytes": 462274560, "latency_s": 1.9373002499341965, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74121216, "prompt_kind": "short", "t_relative_s": 13835.648552624974}, {"decode_latency_s": 0.06831459766544867, "kv_live_bytes": 462274560, "latency_s": 2.188310916069895, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74285056, "prompt_kind": "long", "t_relative_s": 13840.658633917104}, {"decode_latency_s": 0.061100677092326805, "kv_live_bytes": 462274560, "latency_s": 1.956015749834478, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74776576, "prompt_kind": "short", "t_relative_s": 13845.668726083357}, {"decode_latency_s": 0.0668927656224696, "kv_live_bytes": 462274560, "latency_s": 2.143161749932915, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74907648, "prompt_kind": "long", "t_relative_s": 13850.678779750131}, {"decode_latency_s": 0.060593295565922745, "kv_live_bytes": 462274560, "latency_s": 1.9398996662348509, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74891264, "prompt_kind": "short", "t_relative_s": 13855.68138604192}, {"decode_latency_s": 0.06039118880289607, "kv_live_bytes": 462274560, "latency_s": 1.9334867079742253, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73875456, "prompt_kind": "long", "t_relative_s": 13860.691468417179}, {"decode_latency_s": 0.060546546868863516, "kv_live_bytes": 462274560, "latency_s": 1.9385100421495736, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74104832, "prompt_kind": "short", "t_relative_s": 13865.701517499983}, {"decode_latency_s": 0.06746539581217803, "kv_live_bytes": 462274560, "latency_s": 2.161473708227277, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74514432, "prompt_kind": "long", "t_relative_s": 13870.711602708325}, {"decode_latency_s": 0.060716644540661946, "kv_live_bytes": 462274560, "latency_s": 1.9437311249785125, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74907648, "prompt_kind": "short", "t_relative_s": 13875.721682291944}, {"decode_latency_s": 0.06043939452501945, "kv_live_bytes": 462274560, "latency_s": 1.9353271671570837, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75038720, "prompt_kind": "long", "t_relative_s": 13880.731736292131}, {"decode_latency_s": 0.06068432684696745, "kv_live_bytes": 462274560, "latency_s": 1.942689208779484, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74514432, "prompt_kind": "short", "t_relative_s": 13885.741817499977}, {"decode_latency_s": 0.06783286199788563, "kv_live_bytes": 462274560, "latency_s": 2.1726511251181364, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73433088, "prompt_kind": "long", "t_relative_s": 13890.751900041942}, {"decode_latency_s": 0.0606395976501517, "kv_live_bytes": 462274560, "latency_s": 1.9413813748396933, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73580544, "prompt_kind": "short", "t_relative_s": 13895.761956417002}, {"decode_latency_s": 0.06802989062271081, "kv_live_bytes": 462274560, "latency_s": 2.179140332620591, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74153984, "prompt_kind": "long", "t_relative_s": 13900.772040083073}, {"decode_latency_s": 0.06127884116722271, "kv_live_bytes": 462274560, "latency_s": 1.96175791695714, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74137600, "prompt_kind": "short", "t_relative_s": 13905.782103333157}, {"decode_latency_s": 0.06055851040582638, "kv_live_bytes": 462274560, "latency_s": 1.9387291660532355, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74842112, "prompt_kind": "long", "t_relative_s": 13910.792181750294}, {"decode_latency_s": 0.060611464839894325, "kv_live_bytes": 462274560, "latency_s": 1.9403987089172006, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74285056, "prompt_kind": "short", "t_relative_s": 13915.802248999942}, {"decode_latency_s": 0.06786541144538205, "kv_live_bytes": 462274560, "latency_s": 2.174290666822344, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74285056, "prompt_kind": "long", "t_relative_s": 13920.812296333257}, {"decode_latency_s": 0.060468643219792284, "kv_live_bytes": 462274560, "latency_s": 1.9358862908557057, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73695232, "prompt_kind": "short", "t_relative_s": 13925.822358750273}, {"decode_latency_s": 0.06819551302760374, "kv_live_bytes": 462274560, "latency_s": 2.1837873328477144, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74366976, "prompt_kind": "long", "t_relative_s": 13930.832434375305}, {"decode_latency_s": 0.06070769269717857, "kv_live_bytes": 462274560, "latency_s": 1.943481083959341, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74842112, "prompt_kind": "short", "t_relative_s": 13935.842487333342}, {"decode_latency_s": 0.06070476431341376, "kv_live_bytes": 462274560, "latency_s": 1.9435748751275241, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74956800, "prompt_kind": "long", "t_relative_s": 13940.852571375202}, {"decode_latency_s": 0.060562945320270956, "kv_live_bytes": 462274560, "latency_s": 1.938785333186388, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74858496, "prompt_kind": "short", "t_relative_s": 13945.862707500346}, {"decode_latency_s": 0.06823991014971398, "kv_live_bytes": 462274560, "latency_s": 2.1860742093995214, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74874880, "prompt_kind": "long", "t_relative_s": 13950.87275737524}, {"decode_latency_s": 0.060576084622880444, "kv_live_bytes": 462274560, "latency_s": 1.9393225843086839, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73875456, "prompt_kind": "short", "t_relative_s": 13955.882849541958}, {"decode_latency_s": 0.06746470181678887, "kv_live_bytes": 462274560, "latency_s": 2.161239583976567, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74514432, "prompt_kind": "long", "t_relative_s": 13960.892907375004}, {"decode_latency_s": 0.060330178384901956, "kv_live_bytes": 462274560, "latency_s": 1.9314064583741128, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75055104, "prompt_kind": "short", "t_relative_s": 13965.90299412515}, {"decode_latency_s": 0.06810980990121607, "kv_live_bytes": 462274560, "latency_s": 2.1811715411022305, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74481664, "prompt_kind": "long", "t_relative_s": 13970.913067458197}, {"decode_latency_s": 0.06023952865507454, "kv_live_bytes": 462274560, "latency_s": 1.928567708004266, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74645504, "prompt_kind": "short", "t_relative_s": 13975.923116333317}, {"decode_latency_s": 0.06812970053579193, "kv_live_bytes": 462274560, "latency_s": 2.182219082955271, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74678272, "prompt_kind": "long", "t_relative_s": 13980.933193000033}, {"decode_latency_s": 0.06084578904847149, "kv_live_bytes": 462274560, "latency_s": 1.947895917110145, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74711040, "prompt_kind": "short", "t_relative_s": 13985.943304332905}, {"decode_latency_s": 0.06053563543537166, "kv_live_bytes": 462274560, "latency_s": 1.9384592501446605, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74743808, "prompt_kind": "long", "t_relative_s": 13990.953357083257}, {"decode_latency_s": 0.06044978775025811, "kv_live_bytes": 462274560, "latency_s": 1.9352167500182986, "n_emitted": 32, "ok": true, "process_footprint_bytes": 75284480, "prompt_kind": "short", "t_relative_s": 13995.963452208322}, {"decode_latency_s": 0.06816899219120387, "kv_live_bytes": 462274560, "latency_s": 2.183446791023016, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74399744, "prompt_kind": "long", "t_relative_s": 14000.973533208016}, {"decode_latency_s": 0.06032330858579371, "kv_live_bytes": 462274560, "latency_s": 1.9312290837988257, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74678272, "prompt_kind": "short", "t_relative_s": 14005.98357616691}, {"decode_latency_s": 0.06780986981175374, "kv_live_bytes": 462274560, "latency_s": 2.1722359163686633, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74694656, "prompt_kind": "long", "t_relative_s": 14010.993663208093}, {"decode_latency_s": 0.06086561587289907, "kv_live_bytes": 462274560, "latency_s": 1.948492584284395, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74711040, "prompt_kind": "short", "t_relative_s": 14016.003771833144}, {"decode_latency_s": 0.06744768749922514, "kv_live_bytes": 462274560, "latency_s": 2.160916999913752, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74825728, "prompt_kind": "long", "t_relative_s": 14021.013830000069}, {"decode_latency_s": 0.06035879427508917, "kv_live_bytes": 462274560, "latency_s": 1.9323807079344988, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74596352, "prompt_kind": "short", "t_relative_s": 14026.023908625357}, {"decode_latency_s": 0.06944196484982967, "kv_live_bytes": 462274560, "latency_s": 2.223787957802415, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72630272, "prompt_kind": "long", "t_relative_s": 14031.03397837514}, {"decode_latency_s": 0.060251898437854834, "kv_live_bytes": 462274560, "latency_s": 1.9289319589734077, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74498048, "prompt_kind": "short", "t_relative_s": 14036.044042250142}, {"decode_latency_s": 0.07119676303409506, "kv_live_bytes": 462274560, "latency_s": 2.2805724581703544, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73482240, "prompt_kind": "long", "t_relative_s": 14041.054113958031}, {"decode_latency_s": 0.060626497404882684, "kv_live_bytes": 462274560, "latency_s": 1.9408632921986282, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73236480, "prompt_kind": "short", "t_relative_s": 14046.064189542085}, {"decode_latency_s": 0.06811642840330023, "kv_live_bytes": 462274560, "latency_s": 2.1818767921067774, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73728000, "prompt_kind": "long", "t_relative_s": 14051.074266416952}, {"decode_latency_s": 0.06041889716289006, "kv_live_bytes": 462274560, "latency_s": 1.9342926666140556, "n_emitted": 32, "ok": true, "process_footprint_bytes": 69435392, "prompt_kind": "short", "t_relative_s": 14056.08431962505}, {"decode_latency_s": 0.06817281249095686, "kv_live_bytes": 462274560, "latency_s": 2.1838462497107685, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73891840, "prompt_kind": "long", "t_relative_s": 14061.094358708244}, {"decode_latency_s": 0.06026770702737849, "kv_live_bytes": 462274560, "latency_s": 1.9294798751361668, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73973760, "prompt_kind": "short", "t_relative_s": 14066.10443341732}, {"decode_latency_s": 0.0677700755186379, "kv_live_bytes": 462274560, "latency_s": 2.170136125292629, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74203136, "prompt_kind": "long", "t_relative_s": 14071.114556458313}, {"decode_latency_s": 0.06086703775508795, "kv_live_bytes": 462274560, "latency_s": 1.948732334189117, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74285056, "prompt_kind": "short", "t_relative_s": 14076.12461400032}, {"decode_latency_s": 0.06828672137635294, "kv_live_bytes": 462274560, "latency_s": 2.1875518327578902, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74383360, "prompt_kind": "long", "t_relative_s": 14081.134701167233}, {"decode_latency_s": 0.060965717435465194, "kv_live_bytes": 462274560, "latency_s": 1.9517026250250638, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74268672, "prompt_kind": "short", "t_relative_s": 14086.144822957925}, {"decode_latency_s": 0.06830729034845717, "kv_live_bytes": 462274560, "latency_s": 2.188245415687561, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74874880, "prompt_kind": "long", "t_relative_s": 14091.15487591736}, {"decode_latency_s": 0.060381115865311585, "kv_live_bytes": 462274560, "latency_s": 1.9331602090969682, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74956800, "prompt_kind": "short", "t_relative_s": 14096.16496241698}, {"decode_latency_s": 0.0680011666554492, "kv_live_bytes": 462274560, "latency_s": 2.1777805001474917, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74219520, "prompt_kind": "long", "t_relative_s": 14101.175030750223}, {"decode_latency_s": 0.06035796743526589, "kv_live_bytes": 462274560, "latency_s": 1.9323762501589954, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74481664, "prompt_kind": "short", "t_relative_s": 14106.185069125146}, {"decode_latency_s": 0.06054552344721742, "kv_live_bytes": 462274560, "latency_s": 1.9384057503193617, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74645504, "prompt_kind": "long", "t_relative_s": 14111.19515166711}, {"decode_latency_s": 0.06343681509315502, "kv_live_bytes": 462274560, "latency_s": 2.0310006248764694, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66764800, "prompt_kind": "short", "t_relative_s": 14116.205223750323}, {"decode_latency_s": 0.0674204726528842, "kv_live_bytes": 462274560, "latency_s": 2.1597415832802653, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73613312, "prompt_kind": "long", "t_relative_s": 14121.215301292017}, {"decode_latency_s": 0.06049582554260269, "kv_live_bytes": 462274560, "latency_s": 1.9367585419677198, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73678848, "prompt_kind": "short", "t_relative_s": 14126.225344750099}, {"decode_latency_s": 0.06857640753150918, "kv_live_bytes": 462274560, "latency_s": 2.196833458263427, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74088448, "prompt_kind": "long", "t_relative_s": 14131.235415792093}, {"decode_latency_s": 0.06308974609419238, "kv_live_bytes": 462274560, "latency_s": 2.019810250028968, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72646656, "prompt_kind": "short", "t_relative_s": 14136.245464042295}, {"decode_latency_s": 0.06980350390949752, "kv_live_bytes": 462274560, "latency_s": 2.2360777501016855, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70680576, "prompt_kind": "long", "t_relative_s": 14141.25554145826}, {"decode_latency_s": 0.06088421356980689, "kv_live_bytes": 462274560, "latency_s": 1.9491889169439673, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72761344, "prompt_kind": "short", "t_relative_s": 14146.265609750059}, {"decode_latency_s": 0.06799307030451018, "kv_live_bytes": 462274560, "latency_s": 2.1775334998965263, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72777728, "prompt_kind": "long", "t_relative_s": 14151.275685042143}, {"decode_latency_s": 0.060616083355853334, "kv_live_bytes": 462274560, "latency_s": 1.940571874845773, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72957952, "prompt_kind": "short", "t_relative_s": 14156.28576291725}, {"decode_latency_s": 0.06753648960147984, "kv_live_bytes": 462274560, "latency_s": 2.1631582910194993, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73990144, "prompt_kind": "long", "t_relative_s": 14161.295822250191}, {"decode_latency_s": 0.060718488282873295, "kv_live_bytes": 462274560, "latency_s": 1.943889459129423, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73957376, "prompt_kind": "short", "t_relative_s": 14166.305892582983}, {"decode_latency_s": 0.06246530340285972, "kv_live_bytes": 462274560, "latency_s": 2.00039237504825, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74055680, "prompt_kind": "long", "t_relative_s": 14171.315940208267}, {"decode_latency_s": 0.06049003906082362, "kv_live_bytes": 462274560, "latency_s": 1.936455124989152, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73891840, "prompt_kind": "short", "t_relative_s": 14176.326021333225}, {"decode_latency_s": 0.0679346184770111, "kv_live_bytes": 462274560, "latency_s": 2.1761719998903573, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73924608, "prompt_kind": "long", "t_relative_s": 14181.336065292358}, {"decode_latency_s": 0.060469472664408386, "kv_live_bytes": 462274560, "latency_s": 1.9359376667998731, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73334784, "prompt_kind": "short", "t_relative_s": 14186.34612029232}, {"decode_latency_s": 0.06809582031564787, "kv_live_bytes": 462274560, "latency_s": 2.181184458080679, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74203136, "prompt_kind": "long", "t_relative_s": 14191.356157083064}, {"decode_latency_s": 0.06044383204425685, "kv_live_bytes": 462274560, "latency_s": 1.9350832919590175, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74235904, "prompt_kind": "short", "t_relative_s": 14196.366237208247}, {"decode_latency_s": 0.06802668877935503, "kv_live_bytes": 462274560, "latency_s": 2.1784034171141684, "n_emitted": 32, "ok": true, "process_footprint_bytes": 52379648, "prompt_kind": "long", "t_relative_s": 14201.376340582967}, {"decode_latency_s": 0.06680991147004534, "kv_live_bytes": 462274560, "latency_s": 2.1398110832087696, "n_emitted": 32, "ok": true, "process_footprint_bytes": 72892416, "prompt_kind": "short", "t_relative_s": 14206.38642504206}, {"decode_latency_s": 0.06791849480941892, "kv_live_bytes": 462274560, "latency_s": 2.175632707774639, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73990144, "prompt_kind": "long", "t_relative_s": 14211.396472417284}, {"decode_latency_s": 0.06029459374258295, "kv_live_bytes": 462274560, "latency_s": 1.930324333254248, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73613312, "prompt_kind": "short", "t_relative_s": 14216.406539458316}, {"decode_latency_s": 0.06750260418630205, "kv_live_bytes": 462274560, "latency_s": 2.1616426669061184, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74006528, "prompt_kind": "long", "t_relative_s": 14221.416630708147}, {"decode_latency_s": 0.06710641668178141, "kv_live_bytes": 462274560, "latency_s": 2.1495372499339283, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74235904, "prompt_kind": "short", "t_relative_s": 14226.426683542319}, {"decode_latency_s": 0.067162050792831, "kv_live_bytes": 462274560, "latency_s": 2.1515570003539324, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74285056, "prompt_kind": "long", "t_relative_s": 14231.436754208058}, {"decode_latency_s": 0.0680847656331025, "kv_live_bytes": 462274560, "latency_s": 2.180059750098735, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73564160, "prompt_kind": "short", "t_relative_s": 14236.440743542276}, {"decode_latency_s": 0.06765415887639392, "kv_live_bytes": 462274560, "latency_s": 2.166797166224569, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74317824, "prompt_kind": "long", "t_relative_s": 14241.450839417055}, {"decode_latency_s": 0.06061476953618694, "kv_live_bytes": 462274560, "latency_s": 1.940455457661301, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74285056, "prompt_kind": "short", "t_relative_s": 14246.46093075024}, {"decode_latency_s": 0.061673687494476326, "kv_live_bytes": 462274560, "latency_s": 1.9749687500298023, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74334208, "prompt_kind": "long", "t_relative_s": 14251.470976583194}, {"decode_latency_s": 0.060504864566610195, "kv_live_bytes": 462274560, "latency_s": 1.9369191252626479, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74301440, "prompt_kind": "short", "t_relative_s": 14256.481064124964}, {"decode_latency_s": 0.0681957864726428, "kv_live_bytes": 462274560, "latency_s": 2.184253416955471, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73662464, "prompt_kind": "long", "t_relative_s": 14261.491149958223}, {"decode_latency_s": 0.06064315883850213, "kv_live_bytes": 462274560, "latency_s": 1.9414277919568121, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74350592, "prompt_kind": "short", "t_relative_s": 14266.497880042065}, {"decode_latency_s": 0.06788216016138904, "kv_live_bytes": 462274560, "latency_s": 2.174694749992341, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74858496, "prompt_kind": "long", "t_relative_s": 14271.507929500192}, {"decode_latency_s": 0.060296165363979526, "kv_live_bytes": 462274560, "latency_s": 1.9303388753905892, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74301440, "prompt_kind": "short", "t_relative_s": 14276.517981375102}, {"decode_latency_s": 0.0691175950341858, "kv_live_bytes": 462274560, "latency_s": 2.213015957735479, "n_emitted": 32, "ok": true, "process_footprint_bytes": 67452928, "prompt_kind": "long", "t_relative_s": 14281.528063333128}, {"decode_latency_s": 0.06091132944857236, "kv_live_bytes": 462274560, "latency_s": 1.9499901672825217, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71090176, "prompt_kind": "short", "t_relative_s": 14286.536630624905}, {"decode_latency_s": 0.06141983463021461, "kv_live_bytes": 462274560, "latency_s": 1.9668934172950685, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73793536, "prompt_kind": "long", "t_relative_s": 14291.54669004213}, {"decode_latency_s": 0.06252245443465654, "kv_live_bytes": 462274560, "latency_s": 2.0018027503974736, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73777152, "prompt_kind": "short", "t_relative_s": 14296.556764750276}, {"decode_latency_s": 0.06810366015997715, "kv_live_bytes": 462274560, "latency_s": 2.1816262919455767, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74416128, "prompt_kind": "long", "t_relative_s": 14301.56686212495}, {"decode_latency_s": 0.06040512632171158, "kv_live_bytes": 462274560, "latency_s": 1.9337249579839408, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74596352, "prompt_kind": "short", "t_relative_s": 14306.576935457997}, {"decode_latency_s": 0.06863992057333235, "kv_live_bytes": 462274560, "latency_s": 2.198324624914676, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74694656, "prompt_kind": "long", "t_relative_s": 14311.586987667251}, {"decode_latency_s": 0.060367010402842425, "kv_live_bytes": 462274560, "latency_s": 1.9327349998056889, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74235904, "prompt_kind": "short", "t_relative_s": 14316.597048792057}, {"decode_latency_s": 0.06807138804288115, "kv_live_bytes": 462274560, "latency_s": 2.1799823748879135, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74317824, "prompt_kind": "long", "t_relative_s": 14321.607132958248}, {"decode_latency_s": 0.06046865103417076, "kv_live_bytes": 462274560, "latency_s": 1.9359830422326922, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74891264, "prompt_kind": "short", "t_relative_s": 14326.617182417307}, {"decode_latency_s": 0.06782307944376953, "kv_live_bytes": 462274560, "latency_s": 2.1727071669884026, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74760192, "prompt_kind": "long", "t_relative_s": 14331.62726774998}, {"decode_latency_s": 0.060606263039517216, "kv_live_bytes": 462274560, "latency_s": 1.9401857918128371, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74940416, "prompt_kind": "short", "t_relative_s": 14336.637343292125}, {"decode_latency_s": 0.06800741536426358, "kv_live_bytes": 462274560, "latency_s": 2.178320167120546, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73564160, "prompt_kind": "long", "t_relative_s": 14341.64739412535}, {"decode_latency_s": 0.06044995444244705, "kv_live_bytes": 462274560, "latency_s": 1.9353296672925353, "n_emitted": 32, "ok": true, "process_footprint_bytes": 73908224, "prompt_kind": "short", "t_relative_s": 14346.657453583088}, {"decode_latency_s": 0.06922685283643659, "kv_live_bytes": 462274560, "latency_s": 2.2174283326603472, "n_emitted": 32, "ok": true, "process_footprint_bytes": 71680000, "prompt_kind": "long", "t_relative_s": 14351.667539375369}, {"decode_latency_s": 0.06040965497959405, "kv_live_bytes": 462274560, "latency_s": 1.9341657501645386, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74072064, "prompt_kind": "short", "t_relative_s": 14356.67759337509}, {"decode_latency_s": 0.06812563799030613, "kv_live_bytes": 462274560, "latency_s": 2.182012958917767, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74776576, "prompt_kind": "long", "t_relative_s": 14361.68766937498}, {"decode_latency_s": 0.06080827084952034, "kv_live_bytes": 462274560, "latency_s": 1.946680957917124, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74940416, "prompt_kind": "short", "t_relative_s": 14366.693770792335}, {"decode_latency_s": 0.06837962109420914, "kv_live_bytes": 462274560, "latency_s": 2.1907313750125468, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74465280, "prompt_kind": "long", "t_relative_s": 14371.703821083065}, {"decode_latency_s": 0.06065113803197164, "kv_live_bytes": 462274560, "latency_s": 1.94162887474522, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74481664, "prompt_kind": "short", "t_relative_s": 14376.713920041919}, {"decode_latency_s": 0.06867097919166554, "kv_live_bytes": 462274560, "latency_s": 2.199203500058502, "n_emitted": 32, "ok": true, "process_footprint_bytes": 66781184, "prompt_kind": "long", "t_relative_s": 14381.723968000151}, {"decode_latency_s": 0.06084484895109199, "kv_live_bytes": 462274560, "latency_s": 1.9479982499033213, "n_emitted": 32, "ok": true, "process_footprint_bytes": 70991872, "prompt_kind": "short", "t_relative_s": 14386.734054542147}, {"decode_latency_s": 0.06767613021656871, "kv_live_bytes": 462274560, "latency_s": 2.1681315419264138, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74072064, "prompt_kind": "long", "t_relative_s": 14391.744112625252}, {"decode_latency_s": 0.06035307289857883, "kv_live_bytes": 462274560, "latency_s": 1.9322260841727257, "n_emitted": 32, "ok": true, "process_footprint_bytes": 74203136, "prompt_kind": "short", "t_relative_s": 14396.754187125247}]}, "gate": "mixed_prompt_endurance", "metrics": {"aggregate": {"buckets_10min": [{"bucket_index": 0, "mean_kv_live_bytes": 399360999.0243902, "n_turns": 41, "p50_latency_s": 2.341742292046547, "p95_latency_s": 52.34493345860392}, {"bucket_index": 1, "mean_kv_live_bytes": 462274560, "n_turns": 15, "p50_latency_s": 3.7953444998711348, "p95_latency_s": 148.82869383310862}, {"bucket_index": 2, "mean_kv_live_bytes": 462274560, "n_turns": 75, "p50_latency_s": 2.1362737081944942, "p95_latency_s": 2.7404998499434394}, {"bucket_index": 3, "mean_kv_live_bytes": 462274560, "n_turns": 120, "p50_latency_s": 2.119702583178878, "p95_latency_s": 2.1934768794570116}, {"bucket_index": 4, "mean_kv_live_bytes": 462274560, "n_turns": 119, "p50_latency_s": 2.1287325830198824, "p95_latency_s": 2.1952770628035068}, {"bucket_index": 5, "mean_kv_live_bytes": 462274560, "n_turns": 119, "p50_latency_s": 2.0929697919636965, "p95_latency_s": 2.198374154837802}, {"bucket_index": 6, "mean_kv_live_bytes": 462274560, "n_turns": 120, "p50_latency_s": 2.0267889585811645, "p95_latency_s": 2.190342754404992}, {"bucket_index": 7, "mean_kv_live_bytes": 462274560, "n_turns": 115, "p50_latency_s": 2.030197416432202, "p95_latency_s": 2.2249790540430694}, {"bucket_index": 8, "mean_kv_live_bytes": 462274560, "n_turns": 119, "p50_latency_s": 1.9814810841344297, "p95_latency_s": 2.186367407720536}, {"bucket_index": 9, "mean_kv_live_bytes": 462274560, "n_turns": 119, "p50_latency_s": 2.133393708150834, "p95_latency_s": 2.196558258682489}, {"bucket_index": 10, "mean_kv_live_bytes": 462274560, "n_turns": 119, "p50_latency_s": 2.1431248327717185, "p95_latency_s": 2.187859428534284}, {"bucket_index": 11, "mean_kv_live_bytes": 462274560, "n_turns": 119, "p50_latency_s": 2.1319570830091834, "p95_latency_s": 2.1923822838347404}, {"bucket_index": 12, "mean_kv_live_bytes": 462274560, "n_turns": 119, "p50_latency_s": 1.9491943749599159, "p95_latency_s": 2.1891607338096946}, {"bucket_index": 13, "mean_kv_live_bytes": 462274560, "n_turns": 118, "p50_latency_s": 1.9672603332437575, "p95_latency_s": 2.192311875009909}, {"bucket_index": 14, "mean_kv_live_bytes": 462274560, "n_turns": 118, "p50_latency_s": 1.9978569999802858, "p95_latency_s": 2.194456239347346}, {"bucket_index": 15, "mean_kv_live_bytes": 462274560, "n_turns": 120, "p50_latency_s": 2.139613520586863, "p95_latency_s": 2.187065088143572}, {"bucket_index": 16, "mean_kv_live_bytes": 462274560, "n_turns": 120, "p50_latency_s": 2.1221560414414853, "p95_latency_s": 2.1909621228231115}, {"bucket_index": 17, "mean_kv_live_bytes": 462274560, "n_turns": 119, "p50_latency_s": 1.948968917131424, "p95_latency_s": 2.194352866336703}, {"bucket_index": 18, "mean_kv_live_bytes": 462274560, "n_turns": 119, "p50_latency_s": 2.1212582918815315, "p95_latency_s": 2.192575104115531}, {"bucket_index": 19, "mean_kv_live_bytes": 462274560, "n_turns": 119, "p50_latency_s": 2.0085462909191847, "p95_latency_s": 2.1986037076916545}, {"bucket_index": 20, "mean_kv_live_bytes": 462274560, "n_turns": 120, "p50_latency_s": 2.127229542005807, "p95_latency_s": 2.193445692001842}, {"bucket_index": 21, "mean_kv_live_bytes": 462274560, "n_turns": 119, "p50_latency_s": 2.055574500001967, "p95_latency_s": 2.2043874628376217}, {"bucket_index": 22, "mean_kv_live_bytes": 462274560, "n_turns": 118, "p50_latency_s": 2.0994166454765946, "p95_latency_s": 2.1969014376401903}, {"bucket_index": 23, "mean_kv_live_bytes": 462274560, "n_turns": 120, "p50_latency_s": 1.958886833395809, "p95_latency_s": 2.1983685686718673}], "duration_s": 14401.82984066708, "kv_bounded": false, "latency_drift_p50_s": 0.3687449167482555, "max_kv_live_bytes": 462274560, "mean_kv_live_bytes": 461293405.1882845, "min_kv_live_bytes": 9011200, "n_errors": 0, "n_turns": 2629, "p50_latency_s": 2.086497541051358, "p95_latency_s": 2.1988716083578765, "prefill_bounded": true}, "duration_s": 14401.82984066708, "memory_bucket_medians_bytes": [70516736.0, 71139328.0, 71909376.0, 73891840.0, 73285632.0, 73760768.0, 73023488.0, 73588736.0], "monotonic_memory_growth": false, "prompt_kinds": ["long", "short"]}, "status": "passed", "summary": "14401.8/14400.0s; errors=0; monotonic_memory=False; p50_drift_s=0.3687449167482555", "thresholds": {"max_errors": 0, "max_p50_drift_s": 5.0, "minimum_monotonic_growth_bytes": 67108864, "required_prompt_kinds": ["short", "long"], "target_duration_s": 14400.0}}{"evidence": {}, "gate": "disconnect_cancellation", "metrics": {"active_generations": 0, "active_sessions": 0, "cancellation_elapsed_s": 0.5925894170068204}, "status": "passed", "summary": "settled in 0.593s; active_generations=0; active_sessions=0", "thresholds": {"max_cancellation_elapsed_s": 5.0, "required_active_generations": 0, "required_active_sessions": 0}}{"evidence": {}, "gate": "injected_hang_recycle", "metrics": {"injection_accepted": true, "recycle_elapsed_s": 111.7483552498743, "restart_count_after": 1, "restart_count_before": 0, "worker_pid_after": 17254, "worker_pid_before": 54542}, "status": "passed", "summary": "accepted=True; recycled=True; elapsed=111.748s", "thresholds": {"max_recycle_elapsed_s": 120.0}}{"evidence": {}, "gate": "kv_restore_parity", "metrics": {"baseline_first_token_id": 107, "baseline_logits_sha256": "dc1085c5ae016ae2e81a717c78065a630bdd44a2f5f84e10fdb2515822686b2b", "logits_match": true, "restore_source": "allens_kv+proof_checkpoint", "restored_first_token_id": 107, "restored_logits_sha256": "dc1085c5ae016ae2e81a717c78065a630bdd44a2f5f84e10fdb2515822686b2b", "token_match": true}, "status": "passed", "summary": "token_match=True; logits_match=True; restore_source=allens_kv+proof_checkpoint", "thresholds": {"require_logits_sha256_match": true, "require_token_match": true, "required_restore_source": "allens_kv+proof_checkpoint"}}{"evidence": {}, "gate": "decode_latency_regression", "metrics": {"baseline_p50_decode_latency_s": 0.04455711807046706, "baseline_p95_decode_latency_s": 0.04455711807046706, "baseline_source": "bench_mlx_verifier_mean_per_token", "long_run_p50_drift_s": 0.3687449167482555, "p50_decode_latency_s": 0.03768789650348481, "p50_regression_fraction": -0.1541666486624782, "p95_decode_latency_s": 0.03850383528406383, "p95_regression_fraction": -0.13585445038949717, "sample_count": 20}, "status": "passed", "summary": "p50 regression=-15.42%; p95 regression=-13.59%; long-run drift=0.369s", "thresholds": {"max_long_run_p50_drift_s": 5.0, "max_regression_fraction": 0.15}} diff --git a/schemas/primary_decode_acceptance.schema.json b/schemas/primary_decode_acceptance.schema.json new file mode 100644 index 00000000..7a132432 --- /dev/null +++ b/schemas/primary_decode_acceptance.schema.json @@ -0,0 +1,119 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://kakeya.dev/schemas/primary-decode-acceptance-v1.json", + "title": "Primary Decode Mac Acceptance Result", + "type": "object", + "additionalProperties": false, + "required": [ + "$schema", + "schema_version", + "kind", + "status", + "started_at", + "finished_at", + "duration_s", + "config", + "environment", + "git", + "gates" + ], + "properties": { + "$schema": { + "const": "https://kakeya.dev/schemas/primary-decode-acceptance-v1.json" + }, + "schema_version": { + "const": 1 + }, + "kind": { + "const": "primary_decode_mac_acceptance" + }, + "status": { + "enum": ["passed", "failed", "incomplete"] + }, + "started_at": { + "type": "string", + "format": "date-time" + }, + "finished_at": { + "type": "string", + "format": "date-time" + }, + "duration_s": { + "type": "number", + "minimum": 0 + }, + "config": { + "type": "object" + }, + "environment": { + "type": "object" + }, + "git": { + "type": "object" + }, + "latency": { + "type": "object", + "additionalProperties": false, + "required": [ + "p50_decode_latency_s", + "p95_decode_latency_s", + "sample_count" + ], + "properties": { + "p50_decode_latency_s": { + "type": "number", + "minimum": 0 + }, + "p95_decode_latency_s": { + "type": "number", + "minimum": 0 + }, + "sample_count": { + "type": "integer", + "minimum": 1 + } + } + }, + "gates": { + "type": "array", + "items": { + "$ref": "#/$defs/gate" + } + } + }, + "$defs": { + "gate": { + "type": "object", + "additionalProperties": false, + "required": [ + "gate", + "status", + "summary", + "metrics", + "thresholds", + "evidence" + ], + "properties": { + "gate": { + "type": "string", + "minLength": 1 + }, + "status": { + "enum": ["passed", "failed", "error", "skipped"] + }, + "summary": { + "type": "string" + }, + "metrics": { + "type": "object" + }, + "thresholds": { + "type": "object" + }, + "evidence": { + "type": "object" + } + } + } + } +} diff --git a/scripts/bench_agentic/decode_worker_acceptance_adapter.py b/scripts/bench_agentic/decode_worker_acceptance_adapter.py new file mode 100644 index 00000000..fc34f332 --- /dev/null +++ b/scripts/bench_agentic/decode_worker_acceptance_adapter.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Bridge the acceptance harness stdin protocol to the runtime control UDS.""" + +from __future__ import annotations + +import argparse +import json +import socket +import sys + +from inference_engine.backends.mlx.decode_worker import _recv_frame, _send_frame +from inference_engine.backends.mlx.decode_worker_acceptance import ( + normalize_control_socket_path, +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--socket", required=True) + parser.add_argument("--timeout-s", type=float, default=240.0) + args = parser.parse_args() + request = json.load(sys.stdin) + operation = str(request.get("operation", "")) + connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connection.settimeout(args.timeout_s) + try: + connection.connect(normalize_control_socket_path(args.socket)) + _send_frame( + connection, + { + "schema_version": int(request.get("schema_version", -1)), + "operation": operation, + "payload": dict(request.get("payload", {})), + }, + ) + response, _ = _recv_frame(connection) + except (OSError, ValueError, EOFError, json.JSONDecodeError) as exc: + print(f"acceptance control {operation!r} failed: {exc}", file=sys.stderr) + return 1 + finally: + connection.close() + json.dump(response, sys.stdout, sort_keys=True) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/bench_agentic/primary_decode_acceptance.py b/scripts/bench_agentic/primary_decode_acceptance.py new file mode 100644 index 00000000..c9515d6b --- /dev/null +++ b/scripts/bench_agentic/primary_decode_acceptance.py @@ -0,0 +1,532 @@ +#!/usr/bin/env python3 +"""Standalone Mac acceptance runner for the Primary decode worker. + +This runner intentionally drives public gRPC sessions for workload traffic. +Worker-only observations and fault injection are delegated to +``--worker-control-command`` using the JSON protocol documented in +``docs/primary_decode_acceptance.md``. + +The full endurance mode defaults to four hours. It is never started +implicitly: ``--mode`` is required. +""" + +from __future__ import annotations + +import argparse +import json +import platform +import shlex +import subprocess +import sys +import threading +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +from inference_engine.bench.primary_decode_acceptance import ( + GateResult, + atomic_write_json, + build_report, + disconnect_gate, + endurance_gate, + error_gate, + footprint_gate, + hang_recycle_gate, + kv_restore_gate, + latency_baseline_from_report, + latency_gate, + render_junit, +) + + +SHORT_PROMPTS = [ + "Define a KV cache in one sentence.", + "Name one benefit of speculative decoding.", + "What does p95 latency mean?", +] +LONG_PROMPTS = [ + ( + "Analyze a local inference service that uses a bounded sink-window KV " + "cache. Explain cancellation, worker recycling, checkpoint recovery, " + "and memory-pressure failure modes. Give concrete operator checks and " + "keep the answer below six paragraphs. " + ) + * 8, + ( + "Compare process isolation and thread isolation for an Apple Silicon " + "decode runtime, including hangs, unified memory, IPC overhead, state " + "restoration, and observability tradeoffs. " + ) + * 10, +] + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +class WorkerControl: + """Stateless JSON-over-stdin adapter to decode-worker-only controls.""" + + def __init__(self, command: str, timeout_s: float = 10.0) -> None: + self._argv = shlex.split(command) + if not self._argv: + raise ValueError("worker control command must not be empty") + self._timeout_s = timeout_s + + def call(self, operation: str, **payload: Any) -> dict[str, Any]: + request = { + "schema_version": 1, + "operation": operation, + "payload": payload, + } + completed = subprocess.run( + self._argv, + input=json.dumps(request), + text=True, + capture_output=True, + timeout=self._timeout_s, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + f"worker control {operation!r} exited {completed.returncode}: " + f"{completed.stderr.strip()}" + ) + try: + response = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"worker control {operation!r} returned invalid JSON" + ) from exc + if ( + response.get("schema_version") != 1 + or response.get("operation") != operation + ): + raise RuntimeError(f"worker control {operation!r} response mismatch") + if not response.get("ok"): + raise RuntimeError( + f"worker control {operation!r} failed: {response.get('error')}" + ) + data = response.get("data") + if not isinstance(data, dict): + raise RuntimeError(f"worker control {operation!r} data must be an object") + return data + + def snapshot(self) -> dict[str, Any]: + data = self.call("snapshot") + required = { + "runtime_pid", + "worker_pid", + "worker_restart_count", + "process_footprint_bytes", + "active_sessions", + "active_generations", + } + missing = required.difference(data) + if missing: + raise RuntimeError(f"worker snapshot missing fields: {sorted(missing)}") + return data + + +class Harness: + def __init__(self, args: argparse.Namespace) -> None: + from kakeya import Client + from transformers import AutoTokenizer + + self.args = args + self.Client = Client + self.tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_id) + eos = self.tokenizer.eos_token_id + self.eos_ids = [] if eos is None else [int(eos)] + self.control = WorkerControl( + args.worker_control_command, + timeout_s=args.control_timeout_s, + ) + + def _tokens(self, prompt: str) -> list[int]: + return list( + self.tokenizer.encode(prompt, add_special_tokens=False) + ) + + def _one_session( + self, + prompt: str, + *, + max_tokens: int, + ) -> dict[str, Any]: + with self.Client(self.args.grpc_address) as client: + with client.create_session(eos_token_ids=self.eos_ids) as session: + session.append(self._tokens(prompt)) + started = time.perf_counter() + emitted = list(session.generate(max_tokens=max_tokens)) + wall_s = time.perf_counter() - started + total_s = float(session.last_total_duration_seconds or wall_s) + info = session.info() + return { + "emitted": emitted, + "wall_s": wall_s, + "total_s": total_s, + "decode_latency_s": total_s / max(len(emitted), 1), + "kv_live_bytes": info.kv_live_bytes, + } + + def footprint(self) -> GateResult: + fresh = self._wait_for( + lambda snapshot: ( + int(snapshot["active_sessions"]) == 0 + and int(snapshot["active_generations"]) == 0 + ), + timeout_s=self.args.settle_timeout_s, + ) + completed = 0 + for index in range(self.args.session_count): + self._one_session( + SHORT_PROMPTS[index % len(SHORT_PROMPTS)], + max_tokens=self.args.session_max_tokens, + ) + completed += 1 + final = self._wait_for( + lambda snapshot: int(snapshot["active_sessions"]) == 0, + timeout_s=self.args.settle_timeout_s, + ) + if self.args.footprint_settle_s > 0: + time.sleep(self.args.footprint_settle_s) + final = self.control.snapshot() + return footprint_gate( + fresh_bytes=int(fresh["process_footprint_bytes"]), + final_bytes=int(final["process_footprint_bytes"]), + sessions_completed=completed, + required_sessions=100, + max_growth_bytes=2 << 30, + active_sessions=int(final["active_sessions"]), + ) + + def endurance(self) -> GateResult: + records: list[dict[str, Any]] = [] + started = time.perf_counter() + with self.Client(self.args.grpc_address) as client: + with client.create_session(eos_token_ids=self.eos_ids) as session: + turn = 0 + while time.perf_counter() - started < self.args.endurance_duration_s: + prompt_kind = "short" if turn % 2 == 0 else "long" + prompts = SHORT_PROMPTS if prompt_kind == "short" else LONG_PROMPTS + relative_s = time.perf_counter() - started + try: + session.append(self._tokens(prompts[turn % len(prompts)])) + call_started = time.perf_counter() + emitted = list( + session.generate(max_tokens=self.args.endurance_max_tokens) + ) + latency_s = time.perf_counter() - call_started + snapshot = self.control.snapshot() + records.append( + { + "ok": True, + "t_relative_s": relative_s, + "latency_s": latency_s, + "kv_live_bytes": session.info().kv_live_bytes, + "process_footprint_bytes": int( + snapshot["process_footprint_bytes"] + ), + "prompt_kind": prompt_kind, + "n_emitted": len(emitted), + "decode_latency_s": float( + session.last_total_duration_seconds or latency_s + ) + / max(len(emitted), 1), + } + ) + except Exception as exc: # noqa: BLE001 - report every failure + records.append( + { + "ok": False, + "t_relative_s": relative_s, + "prompt_kind": prompt_kind, + "error_class": type(exc).__name__, + "error": str(exc), + } + ) + turn += 1 + sleep_s = self.args.endurance_turn_spacing_s - ( + time.perf_counter() - started - relative_s + ) + if sleep_s > 0: + time.sleep(sleep_s) + duration_s = time.perf_counter() - started + return endurance_gate( + records, + duration_s=duration_s, + target_duration_s=14400.0, + max_p50_drift_s=5.0, + ) + + def disconnect(self) -> GateResult: + client = self.Client(self.args.grpc_address) + session = client.create_session(eos_token_ids=self.eos_ids) + session.append(self._tokens(LONG_PROMPTS[0])) + finished = threading.Event() + + def consume() -> None: + try: + for _ in session.generate(max_tokens=self.args.disconnect_max_tokens): + pass + except Exception: # noqa: BLE001 - channel closure is expected + pass + finally: + finished.set() + + thread = threading.Thread(target=consume, daemon=True) + thread.start() + self._wait_for( + lambda snapshot: int(snapshot["active_generations"]) > 0, + timeout_s=self.args.probe_start_timeout_s, + ) + started = time.perf_counter() + client.close() + settled = self._wait_for( + lambda snapshot: ( + int(snapshot["active_generations"]) == 0 + and int(snapshot["active_sessions"]) == 0 + ), + timeout_s=self.args.disconnect_limit_s, + ) + elapsed = time.perf_counter() - started + finished.wait(timeout=1.0) + return disconnect_gate( + cancellation_elapsed_s=elapsed, + active_generations=int(settled["active_generations"]), + active_sessions=int(settled["active_sessions"]), + limit_s=5.0, + ) + + def hang(self) -> GateResult: + with self.Client(self.args.grpc_address) as client: + session = client.create_session(eos_token_ids=self.eos_ids) + session.append(self._tokens(SHORT_PROMPTS[0])) + before = self.control.snapshot() + injected = self.control.call( + "inject_hang", + phase="next_forward", + expected_worker_pid=int(before["worker_pid"]), + ) + thread = threading.Thread( + target=lambda: list( + session.generate( + max_tokens=2, + inter_token_timeout_s=self.args.hang_limit_s + 15, + ) + ), + daemon=True, + ) + started = time.perf_counter() + thread.start() + after = self._wait_for( + lambda snapshot: ( + int(snapshot["worker_pid"]) != int(before["worker_pid"]) + and int(snapshot["worker_restart_count"]) + > int(before["worker_restart_count"]) + ), + timeout_s=self.args.hang_limit_s, + ) + elapsed = time.perf_counter() - started + return hang_recycle_gate( + injection_accepted=bool(injected.get("accepted")), + recycle_elapsed_s=elapsed, + worker_pid_before=int(before["worker_pid"]), + worker_pid_after=int(after["worker_pid"]), + restart_count_before=int(before["worker_restart_count"]), + restart_count_after=int(after["worker_restart_count"]), + limit_s=120.0, + ) + + def kv_restore(self) -> GateResult: + data = self.control.call( + "kv_restore_parity", + prompt_token_ids=self._tokens(LONG_PROMPTS[1]), + ) + return kv_restore_gate( + baseline_first_token_id=int(data["baseline_first_token_id"]), + restored_first_token_id=int(data["restored_first_token_id"]), + baseline_logits_sha256=str(data["baseline_logits_sha256"]), + restored_logits_sha256=str(data["restored_logits_sha256"]), + restore_source=str(data["restore_source"]), + ) + + def latency(self, long_run_p50_drift_s: float) -> GateResult: + baseline_payload = json.loads(Path(self.args.latency_baseline).read_text()) + baseline = latency_baseline_from_report(baseline_payload) + samples = [ + self._one_session( + SHORT_PROMPTS[index % len(SHORT_PROMPTS)], + max_tokens=self.args.latency_max_tokens, + )["decode_latency_s"] + for index in range(self.args.latency_samples) + ] + return latency_gate( + samples, + baseline=baseline, + long_run_p50_drift_s=long_run_p50_drift_s, + max_regression_fraction=0.15, + max_long_run_p50_drift_s=5.0, + ) + + def _wait_for( + self, + predicate: Callable[[dict[str, Any]], bool], + *, + timeout_s: float, + ) -> dict[str, Any]: + deadline = time.perf_counter() + timeout_s + last: dict[str, Any] = {} + while time.perf_counter() < deadline: + last = self.control.snapshot() + if predicate(last): + return last + time.sleep(self.args.poll_interval_s) + raise TimeoutError(f"worker state did not settle in {timeout_s}s; last={last}") + + +def _git_metadata() -> dict[str, Any]: + def run(*args: str) -> str: + return subprocess.check_output(["git", *args], text=True).strip() + + try: + return { + "commit": run("rev-parse", "HEAD"), + "branch": run("branch", "--show-current"), + "dirty": bool(run("status", "--porcelain")), + } + except (OSError, subprocess.SubprocessError): + return {"commit": "unknown", "branch": "unknown", "dirty": None} + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--mode", + required=True, + choices=["all", "footprint", "endurance", "disconnect", "hang", "kv-restore", "latency"], + ) + parser.add_argument("--grpc-address", default="127.0.0.1:50051") + parser.add_argument("--tokenizer-id", default="Qwen/Qwen3-0.6B") + parser.add_argument("--worker-control-command", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--junit-output", required=True) + parser.add_argument("--latency-baseline") + parser.add_argument("--long-run-report") + parser.add_argument("--session-count", type=int, default=100) + parser.add_argument("--session-max-tokens", type=int, default=8) + parser.add_argument("--endurance-duration-s", type=float, default=14400.0) + parser.add_argument("--endurance-turn-spacing-s", type=float, default=5.0) + parser.add_argument("--endurance-max-tokens", type=int, default=32) + parser.add_argument("--disconnect-max-tokens", type=int, default=4096) + parser.add_argument("--latency-samples", type=int, default=20) + parser.add_argument("--latency-max-tokens", type=int, default=32) + parser.add_argument("--disconnect-limit-s", type=float, default=5.0) + parser.add_argument("--hang-limit-s", type=float, default=120.0) + parser.add_argument("--settle-timeout-s", type=float, default=30.0) + parser.add_argument("--footprint-settle-s", type=float, default=5.0) + parser.add_argument("--probe-start-timeout-s", type=float, default=15.0) + parser.add_argument("--control-timeout-s", type=float, default=240.0) + parser.add_argument("--poll-interval-s", type=float, default=0.25) + return parser + + +def _drift_from_report(path: str) -> float: + payload = json.loads(Path(path).read_text()) + for gate in payload["gates"]: + if gate["gate"] == "mixed_prompt_endurance": + drift = gate["metrics"]["aggregate"]["latency_drift_p50_s"] + if drift is None: + raise ValueError("endurance report has no p50 drift") + return float(drift) + raise ValueError("endurance report is missing mixed_prompt_endurance gate") + + +def main() -> int: + args = _parser().parse_args() + if platform.system() != "Darwin" or platform.machine() != "arm64": + print("Primary decode acceptance requires Apple Silicon macOS", file=sys.stderr) + return 2 + if args.mode in {"all", "latency"} and not args.latency_baseline: + print("--latency-baseline is required for latency mode", file=sys.stderr) + return 2 + if args.mode == "latency" and not args.long_run_report: + print("--long-run-report is required for standalone latency mode", file=sys.stderr) + return 2 + + started_at = _utc_now() + started = time.perf_counter() + gates: list[GateResult] = [] + harness = Harness(args) + operations = { + "footprint": harness.footprint, + "endurance": harness.endurance, + "disconnect": harness.disconnect, + "hang": harness.hang, + "kv-restore": harness.kv_restore, + } + selected = ( + ["footprint", "endurance", "disconnect", "hang", "kv-restore"] + if args.mode == "all" + else [args.mode] + ) + endurance_result: GateResult | None = None + for name in selected: + if name == "latency": + continue + try: + result = operations[name]() + except Exception as exc: # noqa: BLE001 - preserve report on hardware failure + result = error_gate(name, exc) + gates.append(result) + if name == "endurance": + endurance_result = result + + if args.mode in {"all", "latency"}: + try: + if endurance_result is not None: + drift = endurance_result.metrics["aggregate"]["latency_drift_p50_s"] + if drift is None: + raise ValueError("endurance run has no p50 drift") + long_run_drift = float(drift) + else: + long_run_drift = _drift_from_report(args.long_run_report) + gates.append(harness.latency(long_run_drift)) + except Exception as exc: # noqa: BLE001 + gates.append(error_gate("decode_latency_regression", exc)) + + duration_s = time.perf_counter() - started + report = build_report( + gates=gates, + started_at=started_at, + finished_at=_utc_now(), + duration_s=duration_s, + config={ + key: value + for key, value in vars(args).items() + if key != "worker_control_command" + }, + environment={ + "platform": platform.platform(), + "machine": platform.machine(), + "python": platform.python_version(), + }, + git=_git_metadata(), + ) + atomic_write_json(Path(args.output), report) + junit_path = Path(args.junit_output) + junit_path.parent.mkdir(parents=True, exist_ok=True) + junit_path.write_text(render_junit(report) + "\n") + print( + f"[acceptance] status={report['status']} gates={len(gates)} " + f"json={args.output} junit={args.junit_output}", + flush=True, + ) + return 0 if report["status"] == "passed" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/decode_watchdog.py b/scripts/decode_watchdog.py new file mode 100755 index 00000000..00c9c399 --- /dev/null +++ b/scripts/decode_watchdog.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""One-shot external watchdog for the Primary decode LaunchAgent.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import time +from pathlib import Path + + +def _read(path: Path) -> dict: + try: + return json.loads(path.read_text()) + except (FileNotFoundError, json.JSONDecodeError, OSError): + return {} + + +def evaluate( + liveness: dict, + previous: dict, + *, + now: float, + stall_seconds: float, +) -> tuple[dict, bool]: + """Require two observations of the same stale decode token.""" + stale = ( + liveness.get("phase") == "decode" + and now - float(liveness.get("updated_at_unix", now)) >= stall_seconds + ) + identity = [ + liveness.get("pid"), + liveness.get("session_id"), + liveness.get("token_index"), + ] + count = ( + int(previous.get("consecutive", 0)) + 1 + if stale and previous.get("identity") == identity + else (1 if stale else 0) + ) + state = {"identity": identity, "consecutive": count, "checked_at_unix": now} + return state, count >= 2 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--liveness-file", required=True) + parser.add_argument("--state-file", required=True) + parser.add_argument("--unhealthy-file", required=True) + parser.add_argument("--runtime-label", required=True) + parser.add_argument("--stall-seconds", type=float, default=120.0) + args = parser.parse_args() + + liveness_path = Path(args.liveness_file).expanduser() + state_path = Path(args.state_file).expanduser() + unhealthy_path = Path(args.unhealthy_file).expanduser() + liveness = _read(liveness_path) + previous = _read(state_path) + state, stalled = evaluate( + liveness, + previous, + now=time.time(), + stall_seconds=args.stall_seconds, + ) + memory_unhealthy = bool(_read(unhealthy_path)) + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text(json.dumps(state, sort_keys=True)) + if not stalled and not memory_unhealthy: + return 0 + + reason = "decode_stall" if stalled else "memory" + unhealthy_path.parent.mkdir(parents=True, exist_ok=True) + unhealthy_path.write_text(json.dumps({ + "reason": reason, + "observed": liveness, + "updated_at_unix": time.time(), + }, sort_keys=True)) + subprocess.run( + [ + "launchctl", + "kickstart", + "-k", + f"gui/{os.getuid()}/{args.runtime_label}", + ], + check=True, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/start_grpc_runtime_server.py b/scripts/start_grpc_runtime_server.py index 12818257..e453ef8d 100755 --- a/scripts/start_grpc_runtime_server.py +++ b/scripts/start_grpc_runtime_server.py @@ -34,10 +34,13 @@ import argparse import asyncio +import json import threading import logging +import os import signal import sys +from pathlib import Path from typing import Tuple import torch @@ -49,6 +52,13 @@ GrpcServerConfig, create_grpc_server, ) +from inference_engine.server.runtime_health import ( + DecodeLiveness, + PrimaryMemoryGovernor, + create_runtime_health_app, + mlx_memory_bytes, + process_footprint_bytes, +) from inference_engine.session.coordinator import AppendTokensCoordinator from inference_engine.session.generator import GenerationCoordinator from inference_engine.session.store import SessionStore @@ -354,19 +364,43 @@ async def _serve(args: argparse.Namespace) -> int: print(str(exc), file=sys.stderr) return 2 - _LOG.info( - "loading verifier backend=%s id=%s sink=%d window=%d", - args.backend, args.verifier_id, args.sink, args.window, - ) - verifier = _build_verifier( - backend=args.backend, verifier_id=args.verifier_id, - sink=args.sink, window=args.window, - drafter_id=args.drafter_id, f_theta_dir=args.f_theta_dir, - s5_exact_full_attn=not args.no_s5_exact_full_attn, - device=args.device, - ) - - num_layers, num_kv_heads, head_dim = _resolve_kv_dims(verifier) + worker_client = None + if args.decode_worker: + if args.backend != "mlx": + raise SystemExit("--decode-worker requires --backend mlx") + from inference_engine.backends.mlx.decode_worker import ( + DecodeWorkerClient, + DecodeWorkerConfig, + ) + _LOG.info( + "starting isolated MLX decode worker id=%s sink=%d window=%d", + args.verifier_id, args.sink, args.window, + ) + worker_client = DecodeWorkerClient(DecodeWorkerConfig( + model_id=args.verifier_id, + sink_size=args.sink, + window_size=args.window, + request_timeout_s=args.decode_worker_timeout_s, + startup_timeout_s=args.decode_worker_startup_timeout_s, + socket_path=args.decode_worker_socket, + )) + # This object is a transport/process owner, not a model. The router + # process has not imported the MLX verifier module or loaded weights. + verifier = worker_client + num_layers, num_kv_heads, head_dim = worker_client.dimensions + else: + _LOG.info( + "loading verifier backend=%s id=%s sink=%d window=%d", + args.backend, args.verifier_id, args.sink, args.window, + ) + verifier = _build_verifier( + backend=args.backend, verifier_id=args.verifier_id, + sink=args.sink, window=args.window, + drafter_id=args.drafter_id, f_theta_dir=args.f_theta_dir, + s5_exact_full_attn=not args.no_s5_exact_full_attn, + device=args.device, + ) + num_layers, num_kv_heads, head_dim = _resolve_kv_dims(verifier) _LOG.info( "verifier dims: layers=%d kv_heads=%d head_dim=%d capacity=%d", num_layers, num_kv_heads, head_dim, args.sink + args.window, @@ -470,9 +504,8 @@ async def _serve(args: argparse.Namespace) -> int: # PR-A3c: per-session binding for true multi-tenant serving. Each session # gets its own verifier adapter (isolated KV) sharing the model weights via # the adapter's spawn(); the registry doubles as cache_inspector + resolver. - registry = None - on_session_close = None - if args.multi_tenant: + session_registry = None + if args.multi_tenant and worker_client is None: if not hasattr(verifier, "spawn"): raise SystemExit( f"--multi-tenant requires a verifier that supports per-session " @@ -480,16 +513,57 @@ async def _serve(args: argparse.Namespace) -> int: from inference_engine.session.verifier_registry import ( PerSessionVerifierRegistry, ) - registry = PerSessionVerifierRegistry(factory=verifier.spawn) - on_session_close = registry.remove + session_registry = PerSessionVerifierRegistry(factory=verifier.spawn) _LOG.info("multi-tenant per-session binding ENABLED (PR-A3c)") + liveness = DecodeLiveness( + args.decode_liveness_file, + unhealthy_path=args.runtime_unhealthy_file, + ) + governor_holder = [] + + def on_session_removed(session_id: str, reason: str) -> None: + if session_registry is not None: + session_registry.remove(session_id) + if worker_client is not None: + worker_client.close_session(session_id) + if governor_holder: + governor_holder[0].on_session_removed(session_id, reason) + _LOG.info("session removed id=%s reason=%s", session_id, reason) + store = SessionStore( capacity=args.capacity, - cache_inspector=registry if registry is not None else verifier, + cache_inspector=( + session_registry if session_registry is not None else verifier + ), slab_pool=pool, + on_session_removed=on_session_removed, + ) + governor = PrimaryMemoryGovernor( + store, + verifier, + warning_bytes=int(args.memory_warning_gb * (1 << 30)), + drain_bytes=int(args.memory_drain_gb * (1 << 30)), + unhealthy_bytes=int(args.memory_unhealthy_gb * (1 << 30)), + memory_provider=( + worker_client.memory_bytes if worker_client is not None else None + ) or mlx_memory_bytes, + footprint_provider=( + ( + lambda: ( + process_footprint_bytes() + + process_footprint_bytes(worker_client.pid) + ) + ) + if worker_client is not None else process_footprint_bytes + ), + ) + governor_holder.append(governor) + resolver = ( + worker_client.get + if worker_client is not None + else (session_registry.get if session_registry is not None else None) ) - resolver = registry.get if registry is not None else None cache_fill_capture = None if args.cache_fill_capture_size: from inference_engine.distributed.cache_fill import CacheFillCapture @@ -514,13 +588,45 @@ async def _serve(args: argparse.Namespace) -> int: ) if cache_fill_capture is not None else None ), + liveness=liveness, ) gen_coord = GenerationCoordinator( store, verifier, resolver=resolver, on_tokens=_build_token_telemetry_callback(args), + liveness=liveness, ) + acceptance_server = None + if args.decode_worker_acceptance_socket: + if worker_client is None: + raise SystemExit( + "--decode-worker-acceptance-socket requires --decode-worker" + ) + from inference_engine.backends.mlx.decode_worker_acceptance import ( + DecodeWorkerAcceptanceServer, + ) + + acceptance_server = DecodeWorkerAcceptanceServer( + socket_path=args.decode_worker_acceptance_socket, + worker=worker_client, + runtime_pid=os.getpid(), + active_sessions=lambda: store.active_count, + active_generations=lambda: gen_coord.active_count, + process_footprint=lambda: ( + process_footprint_bytes() + + ( + process_footprint_bytes(worker_client.pid) + if worker_client.pid is not None + else 0 + ) + ), + ) + acceptance_server.start() + _LOG.warning( + "local decode acceptance control enabled at %s", + acceptance_server.socket_path, + ) # Multi-host capability plane (ADR 0009): only constructed when # the operator opts in via --enable-capability-exchange (implied @@ -554,7 +660,8 @@ async def _serve(args: argparse.Namespace) -> int: append_coordinator=append_coord, generation_coordinator=gen_coord, config=config, - on_session_close=on_session_close, + memory_governor=governor, + liveness=liveness, capability_registry=registry, proposers=proposers, prefill_cache_store=prefill_store, @@ -607,6 +714,47 @@ async def _serve(args: argparse.Namespace) -> int: args.network_http_port, ) + health_server = None + health_thread = None + if args.runtime_health_port: + import uvicorn + health_server = uvicorn.Server(uvicorn.Config( + create_runtime_health_app(liveness, governor), + host=args.runtime_health_host, + port=args.runtime_health_port, + log_level=args.log_level.lower(), + )) + health_thread = threading.Thread( + target=health_server.run, + name="kakeya-runtime-health", + daemon=True, + ) + health_thread.start() + _LOG.info( + "runtime health listening on http://%s:%d", + args.runtime_health_host, + args.runtime_health_port, + ) + + async def monitor_memory() -> None: + last_level = "" + unhealthy_path = Path(args.runtime_unhealthy_file).expanduser() + while True: + snapshot = await asyncio.to_thread(governor.sample) + if snapshot.level != last_level: + _LOG.warning("Primary memory level changed: %s", snapshot) + last_level = snapshot.level + if snapshot.level == "unhealthy": + unhealthy_path.parent.mkdir(parents=True, exist_ok=True) + unhealthy_path.write_text(json.dumps({ + "reason": "memory", + "process_footprint_bytes": snapshot.process_footprint_bytes, + "updated_at_unix": snapshot.updated_at_unix, + })) + await asyncio.sleep(args.memory_poll_s) + + memory_task = asyncio.create_task(monitor_memory()) + exchange_task = None if registry is not None and args.peer: exchange_task = asyncio.create_task( @@ -639,6 +787,11 @@ def _on_signal(sig: int) -> None: pass await stop_event.wait() + memory_task.cancel() + try: + await memory_task + except asyncio.CancelledError: + pass if exchange_task is not None: exchange_task.cancel() try: @@ -647,11 +800,19 @@ def _on_signal(sig: int) -> None: pass if http_server is not None: http_server.should_exit = True + if health_server is not None: + health_server.should_exit = True if http_thread is not None: await asyncio.to_thread(http_thread.join, args.shutdown_grace_s) + if health_thread is not None: + await asyncio.to_thread(health_thread.join, args.shutdown_grace_s) if prefill_hook is not None: prefill_hook.close() await server.stop(grace=args.shutdown_grace_s) + if acceptance_server is not None: + acceptance_server.close() + if worker_client is not None: + worker_client.close() _LOG.info("kakeya gRPC RuntimeService stopped cleanly") return 0 @@ -692,9 +853,42 @@ def main() -> int: "gets its own isolated KV cache (sharing model " "weights). Requires backend=restored (spawn()). " "Without it, v0.3 single-tenant (one shared cache).") + ap.add_argument( + "--decode-worker", + action="store_true", + help="Run MLX model/KV in an isolated local UDS worker process.", + ) + ap.add_argument( + "--decode-worker-socket", + default="", + help="UDS path for --decode-worker; defaults to a private temp path.", + ) + ap.add_argument("--decode-worker-timeout-s", type=float, default=120.0) + ap.add_argument( + "--decode-worker-startup-timeout-s", type=float, default=180.0, + ) + ap.add_argument( + "--decode-worker-acceptance-socket", + default="", + help=( + "Enable mode-0600 local acceptance controls on this UDS. " + "Never expose this test-only interface on a network listener." + ), + ) ap.add_argument("--shutdown-grace-s", type=float, default=5.0, help="Seconds to give in-flight RPCs to finish on " "SIGTERM/SIGINT before hard-aborting.") + ap.add_argument("--decode-liveness-file", + default="~/.kakeya/primary-decode-liveness.json") + ap.add_argument("--runtime-unhealthy-file", + default="~/.kakeya/primary-runtime-unhealthy.json") + ap.add_argument("--runtime-health-host", default="127.0.0.1") + ap.add_argument("--runtime-health-port", type=int, default=8091, + help="Read-only liveness/memory HTTP port; 0 disables.") + ap.add_argument("--memory-warning-gb", type=float, default=18.0) + ap.add_argument("--memory-drain-gb", type=float, default=20.0) + ap.add_argument("--memory-unhealthy-gb", type=float, default=21.5) + ap.add_argument("--memory-poll-s", type=float, default=5.0) ap.add_argument("--log-level", default="INFO", choices=["DEBUG", "INFO", "WARNING", "ERROR"]) # --- Multi-host capability plane (ADR 0009, v0.5-M1) ------------- diff --git a/sdks/python/kakeya/__init__.py b/sdks/python/kakeya/__init__.py index 098d4e80..737ac284 100644 --- a/sdks/python/kakeya/__init__.py +++ b/sdks/python/kakeya/__init__.py @@ -30,6 +30,7 @@ from kakeya.client import DEFAULT_ADDRESS, Client from kakeya.errors import ( + InterTokenTimeoutError, InvalidArgumentError, InvariantViolationError, KakeyaError, @@ -45,6 +46,7 @@ "Client", "DEFAULT_ADDRESS", "InvalidArgumentError", + "InterTokenTimeoutError", "InvariantViolationError", "KakeyaError", "ResourceExhaustedError", diff --git a/sdks/python/kakeya/errors.py b/sdks/python/kakeya/errors.py index bc6c7042..5f7375f8 100644 --- a/sdks/python/kakeya/errors.py +++ b/sdks/python/kakeya/errors.py @@ -101,6 +101,10 @@ class RpcCancelledError(KakeyaError): """ +class InterTokenTimeoutError(KakeyaError): + """Client-side notification that no stream frame arrived in time.""" + + class SessionClosedError(KakeyaError): """Raised by the SDK itself — never crosses the wire — when a method is called on a :class:`~kakeya.session.Session` whose diff --git a/sdks/python/kakeya/session.py b/sdks/python/kakeya/session.py index 7fd02dd0..efa6c2aa 100644 --- a/sdks/python/kakeya/session.py +++ b/sdks/python/kakeya/session.py @@ -18,6 +18,8 @@ from __future__ import annotations +import queue +import threading from typing import Iterable, Iterator, Optional, TYPE_CHECKING import grpc @@ -27,6 +29,7 @@ runtime_pb2_grpc, ) from kakeya.errors import ( + InterTokenTimeoutError, SessionClosedError, _wrap_grpc_error, ) @@ -165,6 +168,7 @@ def generate( temperature: Optional[float] = None, top_p: Optional[float] = None, top_k: Optional[int] = None, + inter_token_timeout_s: Optional[float] = None, ) -> Iterator[int]: """Stream generated token ids. @@ -178,8 +182,15 @@ def generate( than the greedy no-op default raises :class:`~kakeya.errors.InvalidArgumentError` from the runtime (per ADR 0008 §2.10 "no graceful degradation"). + + ``inter_token_timeout_s`` is a client-side failure notification. + On expiry the SDK cancels the stream and raises + :class:`InterTokenTimeoutError`; the external runtime watchdog owns + hard recovery. """ self._check_open() + if inter_token_timeout_s is not None and inter_token_timeout_s <= 0: + raise ValueError("inter_token_timeout_s must be > 0") # Reset metadata from any prior call before this one starts. self._last_stop_reason = None self._last_generated_token_count = 0 @@ -200,8 +211,44 @@ def generate( if top_k is not None: request.top_k = top_k + call = self._stub.Generate(request) + + def responses(): + if inter_token_timeout_s is None: + yield from call + return + items: queue.Queue = queue.Queue(maxsize=1) + + def consume() -> None: + try: + for item in call: + items.put((item, None)) + items.put((None, StopIteration())) + except BaseException as exc: + items.put((None, exc)) + + threading.Thread( + target=consume, + name="kakeya-sdk-generate", + daemon=True, + ).start() + while True: + try: + response, error = items.get(timeout=inter_token_timeout_s) + except queue.Empty as exc: + call.cancel() + raise InterTokenTimeoutError( + f"no Generate frame received for " + f"{inter_token_timeout_s:.3f}s" + ) from exc + if isinstance(error, StopIteration): + return + if error is not None: + raise error + yield response + try: - for response in self._stub.Generate(request): + for response in responses(): payload = response.WhichOneof("payload") if payload == "token_id": yield response.token_id @@ -224,6 +271,9 @@ def generate( return except grpc.RpcError as exc: raise _wrap_grpc_error(exc) from exc + finally: + if inter_token_timeout_s is not None: + call.cancel() def info(self) -> SessionInfo: """Return a snapshot of the session's server-side state. diff --git a/tests/__init__.py b/tests/__init__.py index e69de29b..595b42a4 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Test support package for spawn-safe subprocess fixtures.""" diff --git a/tests/backends/mlx/test_decode_worker.py b/tests/backends/mlx/test_decode_worker.py new file mode 100644 index 00000000..569053be --- /dev/null +++ b/tests/backends/mlx/test_decode_worker.py @@ -0,0 +1,261 @@ +"""Linux-runnable protocol/recovery tests for the isolated MLX worker.""" + +from __future__ import annotations + +import hashlib +import json +import os +import socket +import threading +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from inference_engine.backends.mlx.decode_worker import ( + PROTOCOL_VERSION, + DecodeWorkerClient, + DecodeWorkerConfig, + _recv_frame, + _send_frame, +) +from inference_engine.distributed.capability import CacheCompatibility +from inference_engine.session.coordinator import OperationCancelledError + + +class FakeDecodeVerifier: + dimensions = (3, 2, 8) + + def __init__(self, model_id: str = "fake") -> None: + self.model_id = model_id + self.model = SimpleNamespace() + self.stats = SimpleNamespace(weight_bytes=1234) + self.cached_token_sequence: list[int] = [] + self.next_global_position = 0 + + def spawn(self): + return type(self)(self.model_id) + + def prefill(self, tokens): + self.cached_token_sequence = list(tokens)[-8:] + self.next_global_position = len(tokens) + + def append_accepted_tokens(self, tokens): + if -999 in tokens: + time.sleep(5) + self.cached_token_sequence = (self.cached_token_sequence + list(tokens))[-8:] + self.next_global_position += len(tokens) + marker = self.model_id.removeprefix("crash-after-append:") + if marker != self.model_id and not Path(marker).exists(): + Path(marker).write_text("crashed") + os._exit(17) + + def greedy_next_token_id(self): + return ((self.cached_token_sequence or [0])[-1] + 1) % 1000 + + def live_kv_bytes(self): + return len(self.cached_token_sequence) * 32 + + def import_snapshot(self, payload, _compatibility): + tokens = json.loads(payload)["tokens"] + self.prefill(tokens) + + def export_snapshot(self, _compatibility): + return json.dumps({"tokens": self.cached_token_sequence}).encode() + + def logits_sha256(self): + next_token = ((self.cached_token_sequence or [0])[-1] + 1) % 1000 + return hashlib.sha256(f"fake-int64:[1]:{next_token}".encode()).hexdigest() + + +def build_fake_verifier(config): + return FakeDecodeVerifier(str(config["model_id"])) + + +def _client(tmp_path, *, model_id="fake", timeout=2.0): + return DecodeWorkerClient(DecodeWorkerConfig( + model_id=model_id, + sink_size=2, + window_size=6, + request_timeout_s=timeout, + startup_timeout_s=10.0, + socket_path=str(tmp_path / "decode.sock"), + verifier_factory=( + "tests.backends.mlx.test_decode_worker:build_fake_verifier" + ), + )) + + +def test_protocol_frame_preserves_binary_snapshot_without_pickle(): + left, right = socket.socketpair() + try: + payload = b"\x00KPKV1\xffsnapshot" + _send_frame(left, { + "version": PROTOCOL_VERSION, + "request_id": "request-1", + "operation": "ImportSnapshot", + }, payload) + header, received = _recv_frame(right) + assert header == { + "operation": "ImportSnapshot", + "request_id": "request-1", + "version": PROTOCOL_VERSION, + } + assert received == payload + finally: + left.close() + right.close() + + +def test_protocol_health_and_all_operations(tmp_path): + client = _client(tmp_path) + try: + health = client.health() + assert health["protocol_version"] == PROTOCOL_VERSION + assert client.dimensions == (3, 2, 8) + + session = client.get("s1") + session.prefill([1, 2, 3]) + session.append_accepted_tokens([4]) + assert session.generate_step() == 5 + assert session.cached_token_sequence == [1, 2, 3, 4, 5] + + compatibility = CacheCompatibility(model_id="fake") + session.import_snapshot( + json.dumps({"tokens": [10, 20]}).encode(), + compatibility, + ) + assert session.next_global_position == 2 + assert session.generate_step() == 21 + client.close_session("s1") + assert client.health()["session_count"] == 0 + finally: + client.close() + + +def test_worker_matches_in_process_greedy_parity(tmp_path): + expected = FakeDecodeVerifier() + expected.prefill([7, 8, 9]) + baseline = [] + for _ in range(6): + token = expected.greedy_next_token_id() + expected.append_accepted_tokens([token]) + baseline.append(token) + + client = _client(tmp_path) + try: + remote = client.get("parity") + remote.prefill([7, 8, 9]) + assert [remote.generate_step() for _ in range(6)] == baseline + finally: + client.close() + + +def test_cancellation_hard_kills_worker_and_preserves_checkpoint(tmp_path): + client = _client(tmp_path, timeout=10.0) + try: + session = client.get("cancel") + session.prefill([1, 2]) + old_pid = client.pid + cancel = threading.Event() + timer = threading.Timer(0.15, cancel.set) + timer.start() + with pytest.raises(OperationCancelledError): + session.append_accepted_tokens([-999], cancel_event=cancel) + timer.cancel() + + # A later request restarts and restores only acknowledged tokens. + assert session.generate_step() == 3 + assert client.pid != old_pid + assert session.cached_token_sequence == [1, 2, 3] + finally: + client.close() + + +def test_crash_recovers_checkpoint_and_retries_current_step(tmp_path): + marker = tmp_path / "crashed" + client = _client(tmp_path, model_id=f"crash-after-append:{marker}") + try: + session = client.get("recover") + session.prefill([40, 41]) + old_pid = client.pid + assert session.generate_step() == 42 + assert marker.exists() + assert client.restart_count == 1 + assert client.pid != old_pid + assert session.cached_token_sequence == [40, 41, 42] + finally: + client.close() + + +def test_snapshot_plus_post_checkpoint_replay_survives_hard_kill(tmp_path): + client = _client(tmp_path) + try: + session = client.get("snapshot") + session.import_snapshot( + json.dumps({"tokens": [10, 11, 12]}).encode(), + CacheCompatibility(model_id="fake"), + ) + session.append_accepted_tokens([13, 14]) + old_pid = client.pid + client.hard_kill() + assert session.generate_step() == 15 + assert client.pid != old_pid + assert session.cached_token_sequence == [10, 11, 12, 13, 14, 15] + finally: + client.close() + + +def test_recycle_lazily_restores_other_sessions(tmp_path): + client = _client(tmp_path) + try: + first = client.get("first") + second = client.get("second") + first.prefill([1, 2]) + second.prefill([90, 91]) + + client.hard_kill() + assert first.generate_step() == 3 + # First's recovery restarted the process, which also lost second's KV. + # Second must notice the worker generation and restore independently. + assert second.generate_step() == 92 + assert second.cached_token_sequence == [90, 91, 92] + finally: + client.close() + + +def test_acceptance_snapshot_hang_and_kv_restore_interfaces(tmp_path): + client = _client(tmp_path, timeout=0.2) + try: + snapshot = client.acceptance_snapshot( + runtime_pid=100, + process_footprint_bytes=200, + active_sessions=3, + active_generations=1, + ) + assert snapshot == { + "runtime_pid": 100, + "worker_pid": client.pid, + "worker_restart_count": 0, + "process_footprint_bytes": 200, + "active_sessions": 3, + "active_generations": 1, + } + + old_pid = client.pid + session = client.get("hang") + session.prefill([1, 2]) + assert client.inject_hang(old_pid)["accepted"] is True + session.append_accepted_tokens([3]) + assert session.generate_step() == 4 + assert client.pid != old_pid + assert client.restart_count == 1 + + parity = client.kv_restore_parity([10, 11, 12]) + assert parity["baseline_first_token_id"] == 13 + assert parity["restored_first_token_id"] == 13 + assert parity["baseline_logits_sha256"] == parity["restored_logits_sha256"] + assert parity["restore_source"] == "allens_kv+proof_checkpoint" + finally: + client.close() diff --git a/tests/backends/mlx/test_decode_worker_acceptance.py b/tests/backends/mlx/test_decode_worker_acceptance.py new file mode 100644 index 00000000..d0235b3b --- /dev/null +++ b/tests/backends/mlx/test_decode_worker_acceptance.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import socket + +from inference_engine.backends.mlx.decode_worker import _recv_frame, _send_frame +from inference_engine.backends.mlx.decode_worker_acceptance import ( + DecodeWorkerAcceptanceServer, + normalize_control_socket_path, +) + + +class FakeWorker: + pid = 222 + restart_count = 3 + + def acceptance_snapshot(self, **values): + return { + **values, + "worker_pid": self.pid, + "worker_restart_count": self.restart_count, + } + + def inject_hang(self, expected_worker_pid): + return { + "accepted": expected_worker_pid == self.pid, + "worker_pid": self.pid, + } + + def kv_restore_parity(self, prompt_token_ids): + return { + "baseline_first_token_id": prompt_token_ids[-1] + 1, + "restored_first_token_id": prompt_token_ids[-1] + 1, + "baseline_logits_sha256": "abc", + "restored_logits_sha256": "abc", + "restore_source": "allens_kv+proof_checkpoint", + } + + +def _call(path: str, operation: str, payload: dict): + connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + connection.connect(path) + _send_frame( + connection, + {"schema_version": 1, "operation": operation, "payload": payload}, + ) + response, _ = _recv_frame(connection) + return response + finally: + connection.close() + + +def test_acceptance_control_server_routes_local_operations(tmp_path): + worker = FakeWorker() + server = DecodeWorkerAcceptanceServer( + socket_path=str(tmp_path / "acceptance.sock"), + worker=worker, + runtime_pid=111, + active_sessions=lambda: 2, + active_generations=lambda: 1, + process_footprint=lambda: 4096, + ) + server.start() + try: + snapshot = _call(server.socket_path, "snapshot", {}) + assert snapshot["ok"] is True + assert snapshot["data"] == { + "runtime_pid": 111, + "worker_pid": 222, + "worker_restart_count": 3, + "process_footprint_bytes": 4096, + "active_sessions": 2, + "active_generations": 1, + } + assert _call( + server.socket_path, + "inject_hang", + {"phase": "next_forward", "expected_worker_pid": 222}, + )["data"]["accepted"] is True + parity = _call( + server.socket_path, + "kv_restore_parity", + {"prompt_token_ids": [7, 8]}, + ) + assert parity["data"]["restored_first_token_id"] == 9 + error = _call(server.socket_path, "unknown", {}) + assert error["ok"] is False + assert "unknown acceptance operation" in error["error"] + finally: + server.close() + assert not (tmp_path / "acceptance.sock").exists() + + +def test_acceptance_control_validation_and_long_path(tmp_path): + worker = FakeWorker() + server = DecodeWorkerAcceptanceServer( + socket_path=str(tmp_path / "unused.sock"), + worker=worker, + runtime_pid=1, + active_sessions=lambda: 0, + active_generations=lambda: 0, + process_footprint=lambda: 0, + ) + try: + server.dispatch( + "inject_hang", + {"phase": "wrong", "expected_worker_pid": 222}, + ) + except ValueError as exc: + assert "next_forward" in str(exc) + else: + raise AssertionError("invalid phase was accepted") + long_path = "/" + "x" * 200 + assert normalize_control_socket_path(long_path).endswith(".sock") diff --git a/tests/backends/mlx/test_verifier.py b/tests/backends/mlx/test_verifier.py index 90b7b893..0c33d3e1 100644 --- a/tests/backends/mlx/test_verifier.py +++ b/tests/backends/mlx/test_verifier.py @@ -28,6 +28,10 @@ _model_weight_bytes, _map_torch_dtype_to_mx, ) +from inference_engine.backends.mlx.prefill_snapshot import ( + export_mlx_prefill_snapshot, +) +from inference_engine.distributed.capability import CacheCompatibility @pytest.fixture(scope="session") @@ -64,6 +68,23 @@ def test_default_config_loads(mlx_verifier_session: MLXSinkWindowVerifier) -> No assert v.stats.weight_bytes > 0 +def test_spawn_shares_model_but_owns_independent_session_kv( + mlx_verifier_session: MLXSinkWindowVerifier, +) -> None: + """Worker session adapters must not load or duplicate model weights.""" + template = mlx_verifier_session + first = template.spawn() + second = template.spawn() + + assert first.model is template.model + assert second.model is template.model + first.prefill([1, 2, 3]) + second.prefill([7, 8]) + assert first.cache is not second.cache + assert first.cached_token_sequence == [1, 2, 3] + assert second.cached_token_sequence == [7, 8] + + @pytest.mark.parametrize( "sink,window,err", [ @@ -216,6 +237,122 @@ def test_append_token_advances_state() -> None: assert logits.ndim == 1 +def test_on_device_argmax_exactly_matches_legacy_torch_argmax( + mlx_verifier_session: MLXSinkWindowVerifier, +) -> None: + """The scalar-only MLX reduction must select the exact same token.""" + v = mlx_verifier_session + v.prefill([1, 2, 3, 4]) + on_device = v.greedy_next_token_id() + legacy = int(torch.argmax(v.next_token_logits).item()) + assert on_device == legacy + + +def test_last_logits_path_has_exact_logits_and_kv_parity( + mlx_verifier_session: MLXSinkWindowVerifier, +) -> None: + """Optimized all-accepted append is bit-exact to full-block commit.""" + v = mlx_verifier_session + prompt = [1, 2, 3, 4] + block = [5, 6, 7] + + v.prefill(prompt) + v.append_accepted_tokens(block) + optimized_logits = v.next_token_logits.clone() + optimized_kv = [ + (layer.keys, layer.values, layer.offset) for layer in v.cache + ] + mx.eval(*[ + tensor + for keys, values, _offset in optimized_kv + for tensor in (keys, values) + ]) + optimized_tokens = list(v.cached_token_sequence) + optimized_position = v.next_global_position + compatibility = CacheCompatibility(model_id="step4-parity") + optimized_snapshot = export_mlx_prefill_snapshot( + v.cache, + token_count=optimized_position, + cached_token_ids=optimized_tokens, + compatibility=compatibility, + ) + + v.prefill(prompt) + full_logits = v.forward_block(block) + v.commit_or_truncate(forwarded=len(block), accepted=len(block)) + full_snapshot = export_mlx_prefill_snapshot( + v.cache, + token_count=v.next_global_position, + cached_token_ids=v.cached_token_sequence, + compatibility=compatibility, + ) + + assert torch.equal(optimized_logits, full_logits[-1]) + assert optimized_snapshot == full_snapshot + assert optimized_tokens == v.cached_token_sequence + assert optimized_position == v.next_global_position + for (expected_k, expected_v, expected_offset), layer in zip( + optimized_kv, v.cache, + ): + assert expected_offset == layer.offset + assert bool(mx.array_equal(expected_k, layer.keys).item()) + assert bool(mx.array_equal(expected_v, layer.values).item()) + + +def test_greedy_path_avoids_vocab_bridge_and_full_block_keeps_it( + mlx_verifier_session: MLXSinkWindowVerifier, + monkeypatch, +) -> None: + """Focused sync regression: greedy crosses zero vocabulary rows.""" + import inference_engine.backends.mlx.verifier as verifier_module + + bridged_shapes = [] + real_bridge = verifier_module.mx_to_torch + + def recording_bridge(arr): + bridged_shapes.append(tuple(arr.shape)) + return real_bridge(arr) + + monkeypatch.setattr(verifier_module, "mx_to_torch", recording_bridge) + v = mlx_verifier_session + v.prefill([1, 2, 3, 4]) + assert v._next_token_logits_torch is None + token = v.greedy_next_token_id() + v.append_accepted_tokens([token]) + assert bridged_shapes == [] + + # Speculative semantics are deliberately unchanged: full [L,V] + # crosses the compatibility boundary once. + block_logits = v.forward_block([5, 6]) + assert bridged_shapes == [tuple(block_logits.shape)] + assert block_logits.ndim == 2 and block_logits.shape[0] == 2 + + +def test_last_logits_memory_accounting_scales_as_one_vocab_row( + mlx_verifier_session: MLXSinkWindowVerifier, +) -> None: + """Peak output accounting distinguishes [V] from full [L,V].""" + v = mlx_verifier_session + v.stats.peak_activation_bytes = 0 + v.prefill([1, 2, 3, 4]) + row_bytes = v.stats.peak_activation_bytes + expected_row_bytes = ( + int(v._next_token_logits_mx.size) + * int(v._next_token_logits_mx.dtype.size) + ) + assert row_bytes == expected_row_bytes + + v.append_accepted_tokens([5, 6, 7]) + assert v.stats.peak_activation_bytes == row_bytes + + v.stats.peak_activation_bytes = 0 + full_logits = v.forward_block([8, 9, 10]) + assert v.stats.peak_activation_bytes == ( + int(full_logits.numel()) * int(full_logits.element_size()) + ) + assert v.stats.peak_activation_bytes == 3 * row_bytes + + # --------------------------------------------------------------------------- # Internals # --------------------------------------------------------------------------- diff --git a/tests/inference_engine/bench/test_primary_decode_acceptance.py b/tests/inference_engine/bench/test_primary_decode_acceptance.py new file mode 100644 index 00000000..56f7c093 --- /dev/null +++ b/tests/inference_engine/bench/test_primary_decode_acceptance.py @@ -0,0 +1,264 @@ +from __future__ import annotations + +import json +import xml.etree.ElementTree as ET + +import pytest + +from inference_engine.bench.primary_decode_acceptance import ( + GIB, + GateResult, + atomic_write_json, + build_report, + disconnect_gate, + endurance_gate, + error_gate, + footprint_gate, + hang_recycle_gate, + kv_restore_gate, + latency_baseline_from_report, + latency_gate, + latency_summary, + monotonic_memory_growth, + render_junit, +) + + +def _records(count: int = 12) -> list[dict]: + return [ + { + "ok": True, + "t_relative_s": float(index), + "latency_s": 1.0, + "kv_live_bytes": 100, + "process_footprint_bytes": 1000 + (index % 2), + "prompt_kind": "short" if index % 2 == 0 else "long", + } + for index in range(count) + ] + + +def _report(gates: list[GateResult]) -> dict: + return build_report( + gates=gates, + started_at="2026-01-01T00:00:00+00:00", + finished_at="2026-01-01T00:00:01+00:00", + duration_s=1.0, + config={"mode": "test"}, + environment={"machine": "arm64"}, + git={"commit": "abc"}, + ) + + +def test_gate_result_rejects_unknown_status(): + with pytest.raises(ValueError, match="invalid gate status"): + GateResult("x", "unknown", "bad") + + +def test_footprint_gate_passes_and_reports_failures(): + passed = footprint_gate( + fresh_bytes=10, + final_bytes=20, + sessions_completed=100, + ) + assert passed.status == "passed" + assert passed.metrics["growth_bytes"] == 10 + + failed = footprint_gate( + fresh_bytes=10, + final_bytes=10 + 2 * GIB + 1, + sessions_completed=99, + active_sessions=1, + ) + assert failed.status == "failed" + assert failed.thresholds["required_sessions"] == 100 + + +def test_monotonic_memory_growth_buckets_and_noise(): + growing, buckets = monotonic_memory_growth( + [0, 20 << 20, 40 << 20, 80 << 20], + minimum_growth_bytes=64 << 20, + ) + assert growing is True + assert len(buckets) == 4 + noisy, _ = monotonic_memory_growth([100, 99, 101, 100]) + assert noisy is False + assert monotonic_memory_growth([]) == (False, []) + + +def test_endurance_gate_reuses_long_run_aggregation(): + result = endurance_gate( + _records(), + duration_s=10, + target_duration_s=10, + ) + assert result.status == "passed" + assert result.metrics["aggregate"]["n_turns"] == 12 + assert result.metrics["prompt_kinds"] == ["long", "short"] + + +def test_endurance_gate_fails_incomplete_or_growing_run(): + records = _records() + for index, record in enumerate(records): + record["process_footprint_bytes"] = index * (64 << 20) + records[0]["ok"] = False + records[0]["error"] = "boom" + result = endurance_gate( + records, + duration_s=9, + target_duration_s=10, + minimum_memory_growth_bytes=1, + ) + assert result.status == "failed" + assert result.metrics["monotonic_memory_growth"] is True + assert result.metrics["aggregate"]["n_errors"] == 1 + + +def test_disconnect_gate_requires_time_and_zero_counters(): + assert disconnect_gate( + cancellation_elapsed_s=4.9, + active_generations=0, + active_sessions=0, + ).status == "passed" + assert disconnect_gate( + cancellation_elapsed_s=5.1, + active_generations=1, + active_sessions=1, + ).status == "failed" + + +def test_hang_gate_requires_pid_and_restart_counter_change(): + passed = hang_recycle_gate( + injection_accepted=True, + recycle_elapsed_s=119, + worker_pid_before=1, + worker_pid_after=2, + restart_count_before=3, + restart_count_after=4, + ) + assert passed.status == "passed" + failed = hang_recycle_gate( + injection_accepted=False, + recycle_elapsed_s=121, + worker_pid_before=1, + worker_pid_after=1, + restart_count_before=3, + restart_count_after=3, + ) + assert failed.status == "failed" + + +def test_kv_restore_gate_requires_token_logits_and_source(): + passed = kv_restore_gate( + baseline_first_token_id=7, + restored_first_token_id=7, + baseline_logits_sha256="abc", + restored_logits_sha256="abc", + restore_source="allens_kv+proof_checkpoint", + ) + assert passed.status == "passed" + failed = kv_restore_gate( + baseline_first_token_id=7, + restored_first_token_id=8, + baseline_logits_sha256="", + restored_logits_sha256="def", + restore_source="other", + ) + assert failed.status == "failed" + assert failed.metrics["logits_match"] is False + + +def test_latency_summary_and_baseline_formats(): + assert latency_summary([])["p50_decode_latency_s"] is None + summary = latency_summary([1, 2, 3, 4]) + assert summary == { + "sample_count": 4, + "p50_decode_latency_s": 2.5, + "p95_decode_latency_s": pytest.approx(3.85), + } + acceptance = latency_baseline_from_report( + {"latency": {"p50_decode_latency_s": 1, "p95_decode_latency_s": 2}} + ) + assert acceptance["source"] == "primary_decode_acceptance" + existing = latency_baseline_from_report( + {"mlx": {"n_tokens": 4, "generation_time_s": 2}} + ) + assert existing["p50_decode_latency_s"] == 0.5 + with pytest.raises(ValueError, match="no generated tokens"): + latency_baseline_from_report( + {"mlx": {"n_tokens": 0, "generation_time_s": 2}} + ) + with pytest.raises(ValueError, match="unsupported"): + latency_baseline_from_report({}) + + +def test_latency_gate_pass_fail_and_validation(): + baseline = { + "p50_decode_latency_s": 1.0, + "p95_decode_latency_s": 1.0, + "source": "test", + } + passed = latency_gate( + [1.0] * 10, + baseline=baseline, + long_run_p50_drift_s=4.9, + ) + assert passed.status == "passed" + assert passed.metrics["baseline_source"] == "test" + assert latency_gate( + [2.0] * 10, + baseline=baseline, + long_run_p50_drift_s=6, + ).status == "failed" + assert latency_gate( + [], + baseline=baseline, + long_run_p50_drift_s=0, + ).summary == "no decode latency samples" + with pytest.raises(ValueError, match="must be > 0"): + latency_gate( + [1], + baseline={"p50_decode_latency_s": 0, "p95_decode_latency_s": 1}, + long_run_p50_drift_s=0, + ) + + +def test_build_report_status_and_latency_summary(): + latency = latency_gate( + [1.0, 1.0], + baseline={"p50_decode_latency_s": 1, "p95_decode_latency_s": 1}, + long_run_p50_drift_s=0, + ) + passed = _report([latency]) + assert passed["status"] == "passed" + assert passed["latency"]["sample_count"] == 2 + assert _report([GateResult("x", "failed", "no")])["status"] == "failed" + assert _report([GateResult("x", "error", "boom")])["status"] == "failed" + assert _report([GateResult("x", "skipped", "later")])["status"] == "incomplete" + assert _report([])["status"] == "incomplete" + + +def test_error_gate_and_junit_rendering(): + gates = [ + GateResult("pass", "passed", "ok"), + GateResult("fail", "failed", "no"), + error_gate("error", RuntimeError("boom")), + GateResult("skip", "skipped", "later"), + ] + report = _report(gates) + xml = ET.fromstring(render_junit(report)) + assert xml.attrib["tests"] == "4" + assert xml.attrib["failures"] == "1" + assert xml.attrib["errors"] == "1" + assert xml.attrib["skipped"] == "1" + assert xml.find("./testcase[@name='fail']/failure") is not None + assert xml.find("./testcase[@name='error']/error") is not None + assert xml.find("./testcase[@name='skip']/skipped") is not None + + +def test_atomic_write_json_replaces_destination(tmp_path): + path = tmp_path / "report.json" + atomic_write_json(path, {"value": 1}) + atomic_write_json(path, {"value": 2}) + assert json.loads(path.read_text()) == {"value": 2} + assert list(tmp_path.iterdir()) == [path] diff --git a/tests/inference_engine/bridge/test_prefill_worker_launchd.py b/tests/inference_engine/bridge/test_prefill_worker_launchd.py index 0e2b7f86..b8259dd5 100644 --- a/tests/inference_engine/bridge/test_prefill_worker_launchd.py +++ b/tests/inference_engine/bridge/test_prefill_worker_launchd.py @@ -6,6 +6,7 @@ HEAD_PLIST = ROOT / "deploy" / "launchd" / "ai.kakeya.grpc-runtime-prefill.plist" PEER_PLIST = ROOT / "deploy" / "launchd" / "ai.kakeya.prefill-network-peer.plist" WORKER_PLIST = ROOT / "deploy" / "launchd" / "ai.kakeya.prefill-worker-peer.plist" +WATCHDOG_INSTALLER = ROOT / "deploy" / "install_decode_watchdog_launchd.sh" def test_worker_installer_emits_full_cache_compatibility_contract(): @@ -92,3 +93,12 @@ def test_worker_injects_explicit_retained_token_cap(): ROOT / "scripts" / "start_prefill_worker_node.py" ).read_text() assert "max_retained_tokens=args.sink + args.window" in source + + +def test_decode_watchdog_launchagent_is_external_and_restarts_primary(): + source = WATCHDOG_INSTALLER.read_text() + assert "scripts/decode_watchdog.py" in source + assert "StartInterval" in source + assert 'STALL_SECONDS="${KAKEYA_DECODE_STALL_SECONDS:-120}"' in source + assert "--runtime-label" in source + assert 'launchctl kickstart -k "$DOMAIN/$WATCHDOG_LABEL"' in source diff --git a/tests/inference_engine/distributed/test_mlx_ring.py b/tests/inference_engine/distributed/test_mlx_ring.py index d44809ee..04af48e4 100644 --- a/tests/inference_engine/distributed/test_mlx_ring.py +++ b/tests/inference_engine/distributed/test_mlx_ring.py @@ -37,6 +37,25 @@ def test_probe_never_raises_and_is_structured(): assert env.world_size == 0 +def test_probe_reports_import_failure_on_every_platform(monkeypatch): + def fail_import(_name): + raise ImportError("synthetic missing mlx") + + monkeypatch.setattr( + "inference_engine.distributed.mlx_ring.importlib.import_module", + fail_import, + ) + env = probe_ring_environment() + assert not env.is_available + assert env.backend == "" + assert env.rank == 0 + assert env.world_size == 0 + assert env.failure_reason == ( + "mlx.core.distributed import failed: " + "ImportError: synthetic missing mlx" + ) + + @pytest.mark.skipif( platform.machine() == "arm64", reason="Linux-gate branch: asserts the mlx-absent probe result", diff --git a/tests/inference_engine/distributed/test_prefill_cache_runtime_fallback.py b/tests/inference_engine/distributed/test_prefill_cache_runtime_fallback.py index b3d2e4c7..25e01258 100644 --- a/tests/inference_engine/distributed/test_prefill_cache_runtime_fallback.py +++ b/tests/inference_engine/distributed/test_prefill_cache_runtime_fallback.py @@ -283,6 +283,82 @@ def test_successful_local_import_suffix_and_on_reuse(monkeypatch): hook.close() +def test_worker_proxy_import_uses_returned_snapshot_state(): + compatibility = CacheCompatibility(model_id="m", block_size_tokens=2) + hook = DistributedPrefillCacheHook( + PrefixCacheStore( + compatibility, + max_bytes=1024, + node_id="head", + ), + compression=CompressionCodec.NONE, + ) + block_hash = chained_block_hashes([1, 2], compatibility)[0] + + class WorkerProxy(_Verifier): + def import_snapshot(self, payload, imported_compatibility): + assert payload == b"worker-snapshot" + assert imported_compatibility == compatibility + return { + "next_global_position": 2, + "cached_token_ids": [1, 2], + "block_hash": block_hash.hex(), + } + + verifier = WorkerProxy() + reused = hook._try_import( + verifier, + [1, 2], + _Hit( + "local", + "lease", + 1, + 2, + len(b"worker-snapshot"), + b"worker-snapshot", + block_hash=block_hash, + ), + ) + assert reused == 2 + assert verifier.cached_token_sequence == [1, 2] + assert verifier.next_global_position == 2 + assert hook.stats.local_hits == 1 + hook.close() + + +def test_worker_proxy_append_path_skips_snapshot_publication(monkeypatch): + compatibility = CacheCompatibility(model_id="m", block_size_tokens=2) + hook = DistributedPrefillCacheHook( + PrefixCacheStore( + compatibility, + max_bytes=1024, + node_id="head", + ), + ) + published = [] + monkeypatch.setattr( + hook, + "_publish_boundary", + lambda *args: published.append(args), + ) + + class WorkerProxy(_Verifier): + is_decode_worker_proxy = True + + def append_accepted_tokens(self, tokens): + self.cached_token_sequence.extend(tokens) + self.next_global_position += len(tokens) + + verifier = WorkerProxy() + verifier.prefill([1, 2]) + hashes = chained_block_hashes([1, 2, 3, 4], compatibility) + hook._compute_and_publish(verifier, [1, 2, 3, 4], hashes, reused=2) + assert verifier.cached_token_sequence == [1, 2, 3, 4] + assert hook.stats.tokens_computed == 2 + assert published == [] + hook.close() + + def test_import_budget_and_remote_worker_failures(monkeypatch): compatibility = CacheCompatibility(model_id="m", block_size_tokens=2) store = PrefixCacheStore(compatibility, max_bytes=1024, node_id="head") diff --git a/tests/inference_engine/server/test_grpc_app.py b/tests/inference_engine/server/test_grpc_app.py index 7036aba0..7dbd2a5c 100644 --- a/tests/inference_engine/server/test_grpc_app.py +++ b/tests/inference_engine/server/test_grpc_app.py @@ -17,6 +17,8 @@ from __future__ import annotations +import asyncio +import threading from typing import AsyncIterator import grpc @@ -29,6 +31,7 @@ DEFAULT_BIND_ADDRESS, GrpcServerConfig, RuntimeServiceServicer, + _ThreadedEventStream, create_grpc_server, ) from inference_engine.server.proto_gen.kakeya.v1 import ( @@ -195,18 +198,16 @@ async def test_close_session_double_close_returns_not_found(grpc_pair): assert exc_info.value.code() == grpc.StatusCode.NOT_FOUND -async def test_close_session_invokes_on_session_close_hook(): - """PR-A3c: CloseSession fires the optional ``on_session_close`` hook with the - session id, so a per-session verifier registry can free that session's KV. - - Built with its own server (the shared ``grpc_pair`` fixture wires no hook) to - exercise the ``self._on_session_close is not None`` branch in CloseSession. - """ - store = SessionStore(capacity=4) - seen: list[str] = [] +async def test_close_session_invokes_unified_store_removal_hook(): + """CloseSession reaches the same hook used by eviction and failures.""" + seen: list[tuple[str, str]] = [] + store = SessionStore( + capacity=4, + on_session_removed=lambda *args: seen.append(args), + ) server = grpc.aio.server() runtime_pb2_grpc.add_RuntimeServiceServicer_to_server( - RuntimeServiceServicer(store, on_session_close=seen.append), server, + RuntimeServiceServicer(store), server, ) port = server.add_insecure_port("127.0.0.1:0") await server.start() @@ -217,7 +218,7 @@ async def test_close_session_invokes_on_session_close_hook(): await stub.CloseSession( runtime_pb2.CloseSessionRequest(session_id=create_resp.session_id), ) - assert seen == [create_resp.session_id] + assert seen == [(create_resp.session_id, "explicit_close")] finally: await channel.close() await server.stop(grace=0.1) @@ -606,7 +607,7 @@ def generate(self, session_id, *, max_tokens, **kw): await server.stop(grace=0.1) -async def test_generate_cancellation_emits_cancelled_done(): +async def test_generate_cancellation_removes_session(): """Direct-invocation test: drive the Servicer's Generate with a fake gRPC context that flips ``cancelled()`` to True after the first event. The servicer must yield a CANCELLED done @@ -656,12 +657,103 @@ async def abort(self, code, details): # pragma: no cover events = [] async for resp in servicer.Generate(request, ctx): events.append(resp) - assert len(events) == 2 + assert len(events) == 1 assert events[0].WhichOneof("payload") == "token_id" - assert events[1].WhichOneof("payload") == "done" - assert events[1].done.stop_reason == \ - runtime_pb2.GenerateDone.STOP_REASON_CANCELLED - assert events[1].done.generated_token_count == 1 + from inference_engine.session import SessionNotFoundError + with pytest.raises(SessionNotFoundError): + store.get_session(sess.session_id) + + +async def test_generate_cancellation_after_done_preserves_session(): + from inference_engine.session import ( + DoneEvent, + GenerationCoordinator, + STOP_REASON_MAX_TOKENS, + ) + + class CompletedGeneration(GenerationCoordinator): + def generate(self, session_id, *, max_tokens, **kwargs): + del session_id, max_tokens, kwargs + yield DoneEvent( + stop_reason=STOP_REASON_MAX_TOKENS, + generated_token_count=0, + prefill_seconds=0.0, + total_seconds=0.0, + ) + + class Context: + def cancelled(self): + return False + + async def abort(self, code, details): + raise AssertionError((code, details)) + + store = SessionStore(capacity=1) + session = store.create_session() + servicer = RuntimeServiceServicer( + store, + generation_coordinator=CompletedGeneration(store, verifier=None), + ) + stream = servicer.Generate( + runtime_pb2.GenerateRequest( + session_id=session.session_id, + max_tokens=1, + ), + Context(), + ) + response = await anext(stream) + assert response.WhichOneof("payload") == "done" + with pytest.raises(asyncio.CancelledError): + await stream.athrow(asyncio.CancelledError()) + assert store.get_session(session.session_id) is session + + +async def test_wire_cancellation_releases_blocked_generate_session(): + """A disconnect removes session state while model thread is still blocked.""" + from inference_engine.session import GenerationCoordinator + + started = threading.Event() + release = threading.Event() + + class BlockingGeneration(GenerationCoordinator): + def generate(self, session_id, *, max_tokens, **kwargs): + del session_id, max_tokens, kwargs + started.set() + release.wait(timeout=2) + if False: + yield + + store = SessionStore(capacity=1) + coordinator = BlockingGeneration(store, verifier=None) + server = grpc.aio.server() + runtime_pb2_grpc.add_RuntimeServiceServicer_to_server( + RuntimeServiceServicer(store, generation_coordinator=coordinator), + server, + ) + port = server.add_insecure_port("127.0.0.1:0") + await server.start() + channel = grpc.aio.insecure_channel(f"127.0.0.1:{port}") + stub = runtime_pb2_grpc.RuntimeServiceStub(channel) + try: + session = store.create_session() + call = stub.Generate(runtime_pb2.GenerateRequest( + session_id=session.session_id, + max_tokens=1, + )) + read_task = asyncio.create_task(call.read()) + assert await asyncio.to_thread(started.wait, 1) + call.cancel() + with pytest.raises(asyncio.CancelledError): + await read_task + for _ in range(50): + if store.active_count == 0: + break + await asyncio.sleep(0.01) + assert store.active_count == 0 + finally: + release.set() + await channel.close() + await server.stop(grace=0.1) async def test_append_tokens_session_not_found_returns_not_found(): @@ -733,6 +825,47 @@ def append_tokens(self, session_id, token_ids): await server.stop(grace=0.1) +async def test_append_tokens_worker_does_not_block_event_loop(): + from inference_engine.session import AppendTokensCoordinator + + started = threading.Event() + release = threading.Event() + + class BlockingCoordinator(AppendTokensCoordinator): + def append_tokens(self, session_id, token_ids): + del session_id, token_ids + started.set() + release.wait(timeout=2) + return 1 + + store = SessionStore(capacity=2) + servicer = RuntimeServiceServicer( + store, + append_coordinator=BlockingCoordinator(store, verifier=None), + ) + + class Context: + def cancelled(self): + return False + + async def abort(self, code, detail): + raise AssertionError((code, detail)) + + session = store.create_session() + append_task = asyncio.create_task(servicer.AppendTokens( + runtime_pb2.AppendTokensRequest(session_id=session.session_id, token_ids=[1]), + Context(), + )) + await asyncio.to_thread(started.wait, 1) + created = await asyncio.wait_for( + servicer.CreateSession(runtime_pb2.CreateSessionRequest(), Context()), + timeout=0.2, + ) + assert created.session_id + release.set() + await append_task + + async def test_generate_yields_history_truncated_then_done(): """Generator override yields HistoryTruncatedEvent + DoneEvent → truncated frame + done frame on the wire. Covers the @@ -795,6 +928,252 @@ def generate(self, session_id, *, max_tokens, **kw): await server.stop(grace=0.1) +async def test_threaded_event_stream_stops_putting_after_cancel(): + cancel_event = threading.Event() + stream = _ThreadedEventStream((), cancel_event) + stream._queue.put(("occupied", None)) + cancel_event.set() + stream._put(("ignored", None)) + assert stream._queue.get_nowait() == ("occupied", None) + assert stream._queue.empty() + + +async def test_threaded_event_stream_breaks_after_cancelled_event(): + cancel_event = threading.Event() + + class CancelOnFirst: + def __iter__(self): + return self + + def __next__(self): + cancel_event.set() + return "first" + + stream = _ThreadedEventStream(CancelOnFirst(), cancel_event) + stream._run() + assert stream._queue.get_nowait() == ("first", None) + + +class _DirectAbort(Exception): + pass + + +class _DirectContext: + def __init__(self): + self.code = None + self.detail = "" + self.callback = None + + async def abort(self, code, detail): + self.code = code + self.detail = detail + raise _DirectAbort + + def cancelled(self): + return False + + def add_done_callback(self, callback): + self.callback = callback + + +async def test_watch_context_removes_cancelled_session(): + store = SessionStore(capacity=1) + session = store.create_session() + servicer = RuntimeServiceServicer(store) + context = _DirectContext() + cancel_event = threading.Event() + servicer._watch_context(context, cancel_event, session.session_id) + assert context.callback is not None + context.callback(type("Done", (), {"cancelled": lambda self: True})()) + assert cancel_event.is_set() + assert store.active_count == 0 + + +async def test_watch_context_preserves_logically_completed_session(): + store = SessionStore(capacity=1) + session = store.create_session() + servicer = RuntimeServiceServicer(store) + context = _DirectContext() + cancel_event = threading.Event() + completed_event = threading.Event() + servicer._watch_context( + context, + cancel_event, + session.session_id, + completed_event, + ) + completed_event.set() + assert context.callback is not None + context.callback(type("Done", (), {"cancelled": lambda self: True})()) + assert not cancel_event.is_set() + assert store.active_count == 1 + + +async def test_create_session_rejects_memory_drain(): + store = SessionStore(capacity=1) + governor = type("Governor", (), {"draining": True})() + servicer = RuntimeServiceServicer(store, memory_governor=governor) + context = _DirectContext() + with pytest.raises(_DirectAbort): + await servicer.CreateSession(runtime_pb2.CreateSessionRequest(), context) + assert context.code == grpc.StatusCode.RESOURCE_EXHAUSTED + assert store.active_count == 0 + + +async def test_append_rejects_concurrent_operation(): + store = SessionStore(capacity=1) + session = store.create_session() + coordinator = type( + "Coordinator", + (), + {"append_tokens": lambda self, session_id, token_ids: 0}, + )() + servicer = RuntimeServiceServicer(store, append_coordinator=coordinator) + assert servicer._acquire_operation(session.session_id) + context = _DirectContext() + with pytest.raises(_DirectAbort): + await servicer.AppendTokens( + runtime_pb2.AppendTokensRequest( + session_id=session.session_id, + token_ids=[1], + ), + context, + ) + assert context.code == grpc.StatusCode.ABORTED + servicer._release_operation(session.session_id) + + +async def test_append_passes_cancel_event_and_resets_liveness(): + observed = {} + + class Coordinator: + def append_tokens(self, session_id, token_ids, cancel_event): + observed["session_id"] = session_id + observed["tokens"] = token_ids + observed["cancel_event"] = cancel_event + return len(token_ids) + + class Liveness: + def update(self, phase): + observed["phase"] = phase + + store = SessionStore(capacity=1) + session = store.create_session() + servicer = RuntimeServiceServicer( + store, + append_coordinator=Coordinator(), + liveness=Liveness(), + ) + response = await servicer.AppendTokens( + runtime_pb2.AppendTokensRequest( + session_id=session.session_id, + token_ids=[4, 5], + ), + _DirectContext(), + ) + assert response.history_length == 2 + assert observed["session_id"] == session.session_id + assert observed["tokens"] == [4, 5] + assert isinstance(observed["cancel_event"], threading.Event) + assert observed["phase"] == "idle" + + +async def test_append_asyncio_cancellation_removes_session(): + class Coordinator: + def append_tokens(self, session_id, token_ids): + raise asyncio.CancelledError + + store = SessionStore(capacity=1) + session = store.create_session() + servicer = RuntimeServiceServicer(store, append_coordinator=Coordinator()) + with pytest.raises(asyncio.CancelledError): + await servicer.AppendTokens( + runtime_pb2.AppendTokensRequest( + session_id=session.session_id, + token_ids=[1], + ), + _DirectContext(), + ) + assert store.active_count == 0 + + +async def test_append_worker_cancellation_maps_cancelled(): + from inference_engine.session import OperationCancelledError + + class Coordinator: + def append_tokens(self, session_id, token_ids): + raise OperationCancelledError("worker cancelled") + + store = SessionStore(capacity=1) + session = store.create_session() + servicer = RuntimeServiceServicer(store, append_coordinator=Coordinator()) + context = _DirectContext() + with pytest.raises(_DirectAbort): + await servicer.AppendTokens( + runtime_pb2.AppendTokensRequest( + session_id=session.session_id, + token_ids=[1], + ), + context, + ) + assert context.code == grpc.StatusCode.CANCELLED + assert store.active_count == 0 + + +async def test_generate_rejects_concurrent_operation(): + class Coordinator: + def generate(self, session_id, **kwargs): + return iter(()) + + store = SessionStore(capacity=1) + session = store.create_session() + servicer = RuntimeServiceServicer( + store, + generation_coordinator=Coordinator(), + ) + assert servicer._acquire_operation(session.session_id) + context = _DirectContext() + with pytest.raises(_DirectAbort): + async for _ in servicer.Generate( + runtime_pb2.GenerateRequest( + session_id=session.session_id, + max_tokens=1, + ), + context, + ): + pass + assert context.code == grpc.StatusCode.ABORTED + servicer._release_operation(session.session_id) + + +async def test_generate_busy_error_maps_aborted(): + from inference_engine.session import SessionGenerationBusyError + + class Coordinator: + def generate(self, session_id, **kwargs): + if False: + yield + raise SessionGenerationBusyError(session_id) + + store = SessionStore(capacity=1) + session = store.create_session() + servicer = RuntimeServiceServicer( + store, + generation_coordinator=Coordinator(), + ) + context = _DirectContext() + with pytest.raises(_DirectAbort): + async for _ in servicer.Generate( + runtime_pb2.GenerateRequest( + session_id=session.session_id, + max_tokens=1, + ), + context, + ): + pass + assert context.code == grpc.StatusCode.ABORTED + + async def test_factory_wires_prefill_cache_service(): from inference_engine.distributed.capability import CacheCompatibility from inference_engine.distributed.prefill_cache import PrefixCacheStore diff --git a/tests/inference_engine/server/test_runtime_health.py b/tests/inference_engine/server/test_runtime_health.py new file mode 100644 index 00000000..035801cc --- /dev/null +++ b/tests/inference_engine/server/test_runtime_health.py @@ -0,0 +1,107 @@ +import json + +import psutil +from fastapi.testclient import TestClient + +from inference_engine.server import runtime_health +from inference_engine.server.runtime_health import ( + DecodeLiveness, + PrimaryMemoryGovernor, + create_runtime_health_app, +) +from inference_engine.session import SessionStore + + +class Verifier: + def __init__(self): + self.resets = 0 + + def reset(self): + self.resets += 1 + + +def test_process_footprint_uses_fork_free_psutil(monkeypatch): + seen = [] + + class Process: + def __init__(self, pid): + seen.append(pid) + + def memory_info(self): + return type("MemoryInfo", (), {"rss": 12345})() + + monkeypatch.setattr(runtime_health.psutil, "Process", Process) + assert runtime_health.process_footprint_bytes(42) == 12345 + assert seen == [42] + + +def test_process_footprint_returns_zero_when_process_disappears(monkeypatch): + def missing(pid): + raise psutil.NoSuchProcess(pid) + + monkeypatch.setattr(runtime_health.psutil, "Process", missing) + assert runtime_health.process_footprint_bytes(42) == 0 + + +def test_liveness_is_atomically_published(tmp_path): + path = tmp_path / "live.json" + live = DecodeLiveness(path, clock=lambda: 123.0, pid=42) + live.update("decode", "session-1", 7) + assert json.loads(path.read_text()) == { + "phase": "decode", + "session_id": "session-1", + "token_index": 7, + "updated_at_unix": 123.0, + "pid": 42, + } + + +def test_memory_governor_levels_and_idle_cleanup(): + store = SessionStore(capacity=1) + verifier = Verifier() + cleared = [] + footprint = [18] + governor = PrimaryMemoryGovernor( + store, + verifier, + warning_bytes=10, + drain_bytes=20, + unhealthy_bytes=30, + memory_provider=lambda: (3, 4, 5), + footprint_provider=lambda: footprint[0], + clear_cache=lambda: cleared.append(True), + ) + assert governor.sample().level == "warning" + session = store.create_session() + governor.on_session_removed(session.session_id, "test") + assert verifier.resets == 0 + store.close_session(session.session_id) + governor.on_session_removed(session.session_id, "test") + assert verifier.resets == 1 + assert cleared == [True] + footprint[0] = 20 + governor.sample() + assert governor.draining + footprint[0] = 30 + governor.sample() + assert governor.unhealthy + + +def test_runtime_health_endpoint_reports_unhealthy_marker(tmp_path): + unhealthy = tmp_path / "unhealthy.json" + live = DecodeLiveness(tmp_path / "live.json", unhealthy_path=unhealthy) + governor = PrimaryMemoryGovernor( + SessionStore(capacity=1), + Verifier(), + warning_bytes=10, + drain_bytes=20, + unhealthy_bytes=30, + memory_provider=lambda: (0, 0, 0), + footprint_provider=lambda: 0, + ) + client = TestClient(create_runtime_health_app(live, governor)) + assert client.get("/healthz").status_code == 200 + unhealthy.write_text("{}") + response = client.get("/healthz") + assert response.status_code == 503 + assert response.json()["status"] == "unhealthy" diff --git a/tests/inference_engine/session/test_coordinator_validation.py b/tests/inference_engine/session/test_coordinator_validation.py index cf7b7c39..9c1474fb 100644 --- a/tests/inference_engine/session/test_coordinator_validation.py +++ b/tests/inference_engine/session/test_coordinator_validation.py @@ -116,6 +116,42 @@ def kv_live_bytes(self, _session): assert observed == [("live", [1, 2])] +def test_incremental_append_prefers_last_logits_only_fast_path(): + class Verifier: + cached_token_sequence = [] + next_global_position = 0 + next_token_logits = None + fast_path_calls = [] + + def prefill(self, tokens): + self.cached_token_sequence = list(tokens) + self.next_global_position = len(tokens) + + def append_accepted_tokens(self, tokens): + self.fast_path_calls.append(list(tokens)) + self.cached_token_sequence.extend(tokens) + self.next_global_position += len(tokens) + + def forward_block(self, _tokens): + raise AssertionError("full-block path should not run") + + def k_seq_length(self, _session): + return len(self.cached_token_sequence) + + def kv_live_bytes(self, _session): + return 0 + + verifier = Verifier() + store = SessionStore(capacity=1, cache_inspector=verifier) + session = store.create_session() + coord = AppendTokensCoordinator(store, verifier) + coord.append_tokens(session.session_id, [1, 2]) + coord.append_tokens(session.session_id, [3, 4]) + + assert verifier.fast_path_calls == [[3, 4]] + assert session.history_token_ids == [1, 2, 3, 4] + + # Note: tests for the ``_sync_slab_bytes`` helper (PR-E1c addition) # live in PR-E1c's own commit. PR-N1 is branched off main; once # PR-E1c merges, a follow-up will add the helper's None-branch test diff --git a/tests/inference_engine/session/test_generator_validation.py b/tests/inference_engine/session/test_generator_validation.py index fef114ad..75472eca 100644 --- a/tests/inference_engine/session/test_generator_validation.py +++ b/tests/inference_engine/session/test_generator_validation.py @@ -22,9 +22,11 @@ from __future__ import annotations import pytest +import threading from inference_engine.session import ( GenerationCoordinator, + SessionGenerationBusyError, SessionNotFoundError, SessionStore, ) @@ -163,3 +165,79 @@ def test_constructor_stores_references_without_calling_them(self): # Check internals — neither object was poked. assert coord._store is sentinel_store assert coord._verifier is sentinel_verifier + + +def test_generate_honors_pre_set_cancellation_at_token_boundary(): + store, sess = _store_and_session_with_history() + cancelled = threading.Event() + cancelled.set() + events = list(GenerationCoordinator(store, verifier=None).generate( + sess.session_id, + max_tokens=1, + cancel_event=cancelled, + )) + assert len(events) == 1 + assert events[0].stop_reason == "cancelled" + + +def test_generate_rejects_concurrent_stream_for_same_session(): + store, sess = _store_and_session_with_history() + cancelled = threading.Event() + cancelled.set() + coordinator = GenerationCoordinator(store, verifier=None) + first = coordinator.generate( + sess.session_id, max_tokens=1, cancel_event=cancelled, + ) + assert coordinator.active_count == 0 + assert next(first).stop_reason == "cancelled" + assert coordinator.active_count == 1 + second = coordinator.generate( + sess.session_id, max_tokens=1, cancel_event=cancelled, + ) + with pytest.raises(SessionGenerationBusyError): + next(second) + first.close() + assert coordinator.active_count == 0 + + +def test_generate_prefers_on_device_argmax_and_last_logits_fast_path(): + class Verifier: + cached_token_sequence = [10] + next_global_position = 1 + next_token_logits = None + argmax_calls = 0 + append_calls = [] + + def greedy_next_token_id(self): + self.argmax_calls += 1 + return 11 + + def append_accepted_tokens(self, tokens): + self.append_calls.append(list(tokens)) + self.cached_token_sequence.extend(tokens) + self.next_global_position += len(tokens) + + def forward_block(self, _tokens): + raise AssertionError("full-block path should not run") + + def k_seq_length(self, _session): + return len(self.cached_token_sequence) + + def kv_live_bytes(self, _session): + return 0 + + verifier = Verifier() + store = SessionStore(capacity=1, cache_inspector=verifier) + session = store.create_session() + session.cached_token_sequence = [10] + store.append_tokens(session.session_id, [10]) + store.record_position_advance(session.session_id, 1) + + events = list(GenerationCoordinator(store, verifier).generate( + session.session_id, + max_tokens=1, + )) + + assert verifier.argmax_calls == 1 + assert verifier.append_calls == [[11]] + assert events[0].token_id == 11 diff --git a/tests/inference_engine/session/test_store.py b/tests/inference_engine/session/test_store.py index 019fd623..8722b797 100644 --- a/tests/inference_engine/session/test_store.py +++ b/tests/inference_engine/session/test_store.py @@ -287,6 +287,18 @@ def test_double_close_raises(self): with pytest.raises(SessionNotFoundError): store.close_session(sess.session_id) + def test_remove_if_present_is_idempotent_for_cancellation(self): + seen = [] + store = SessionStore(capacity=1, on_session_removed=lambda *args: seen.append(args)) + sess = store.create_session() + assert store.remove_session_if_present( + sess.session_id, reason="client_cancelled", + ) + assert not store.remove_session_if_present( + sess.session_id, reason="client_cancelled", + ) + assert seen == [(sess.session_id, "client_cancelled")] + # --------------------------------------------------------------------------- # LRU eviction at capacity @@ -324,6 +336,13 @@ def test_eviction_picks_least_recently_active_strictly(self): assert store.get_session(c.session_id) is c assert store.get_session(d.session_id) is d + def test_reports_lru_reason_to_unified_removal_hook(self): + seen = [] + store = SessionStore(capacity=1, on_session_removed=lambda *args: seen.append(args)) + old = store.create_session() + store.create_session() + assert seen == [(old.session_id, "lru_eviction")] + # --------------------------------------------------------------------------- # evict_idle (TTL eviction) @@ -373,6 +392,13 @@ def test_at_threshold_is_evicted(self): evicted = store.evict_idle(ttl_seconds=5.0, now=1005.0) assert evicted == [sess] + def test_reports_ttl_reason_to_unified_removal_hook(self): + seen = [] + store = SessionStore(capacity=1, on_session_removed=lambda *args: seen.append(args)) + sess = store.create_session() + store.evict_idle(ttl_seconds=0, now=sess.last_active_at) + assert seen == [(sess.session_id, "ttl_eviction")] + # --------------------------------------------------------------------------- # record_position_advance / INV-2 diff --git a/tests/scripts/test_decode_watchdog.py b/tests/scripts/test_decode_watchdog.py new file mode 100644 index 00000000..fce8eda4 --- /dev/null +++ b/tests/scripts/test_decode_watchdog.py @@ -0,0 +1,37 @@ +from scripts.decode_watchdog import evaluate + + +def test_watchdog_requires_two_identical_stale_decode_observations(): + live = { + "phase": "decode", + "pid": 10, + "session_id": "s", + "token_index": 4, + "updated_at_unix": 100.0, + } + first, restart = evaluate(live, {}, now=221.0, stall_seconds=120.0) + assert not restart + second, restart = evaluate(live, first, now=251.0, stall_seconds=120.0) + assert restart + assert second["consecutive"] == 2 + + +def test_watchdog_resets_when_token_progresses_or_phase_is_idle(): + stale = { + "phase": "decode", + "pid": 10, + "session_id": "s", + "token_index": 4, + "updated_at_unix": 100.0, + } + previous, _ = evaluate(stale, {}, now=221.0, stall_seconds=120.0) + progressed = dict(stale, token_index=5, updated_at_unix=220.0) + state, restart = evaluate( + progressed, previous, now=250.0, stall_seconds=120.0, + ) + assert not restart + assert state["consecutive"] == 0 + idle = dict(stale, phase="idle") + state, restart = evaluate(idle, previous, now=250.0, stall_seconds=120.0) + assert not restart + assert state["consecutive"] == 0 diff --git a/tests/sdk/python/test_session_timeout.py b/tests/sdk/python/test_session_timeout.py new file mode 100644 index 00000000..ba3b7f48 --- /dev/null +++ b/tests/sdk/python/test_session_timeout.py @@ -0,0 +1,41 @@ +import threading + +import pytest + +from kakeya.errors import InterTokenTimeoutError +from kakeya.session import Session + + +class BlockingCall: + def __init__(self): + self.released = threading.Event() + self.cancelled = False + + def __iter__(self): + return self + + def __next__(self): + self.released.wait(timeout=1) + raise StopIteration + + def cancel(self): + self.cancelled = True + self.released.set() + + +def test_generate_inter_token_timeout_cancels_stream(): + call = BlockingCall() + client = type("Client", (), { + "_stub": type("Stub", (), {"Generate": lambda _self, _request: call})(), + })() + session = Session(client=client, session_id="s") + with pytest.raises(InterTokenTimeoutError, match="no Generate frame"): + list(session.generate(max_tokens=1, inter_token_timeout_s=0.01)) + assert call.cancelled + + +def test_generate_rejects_non_positive_inter_token_timeout(): + client = type("Client", (), {"_stub": object()})() + session = Session(client=client, session_id="s") + with pytest.raises(ValueError, match="must be > 0"): + list(session.generate(max_tokens=1, inter_token_timeout_s=0))