diff --git a/docs/ops/distributed-prefill-kv-network.md b/docs/ops/distributed-prefill-kv-network.md index d2fc598e..e4363858 100644 --- a/docs/ops/distributed-prefill-kv-network.md +++ b/docs/ops/distributed-prefill-kv-network.md @@ -308,7 +308,7 @@ Run two logical agents through real multi-round model inference: ```bash bash scripts/run_agent_gan_demo.sh \ - --rounds 2 \ + --rounds 1 \ --output-tokens 64 \ --report /tmp/kakeya-agent-gan-demo.json ``` @@ -320,6 +320,10 @@ persisted reports contain only output length/hash and metrics. The report separates inference-only KV hit rate from whole-workload hit rate including warmup, plus per-agent and aggregate token throughput/latency. +One Generator/Critic cycle is the safe default for the current 16GB allens +Gemma worker. Additional rounds grow agent histories and may exceed the strict +remote-prefill timeout; increase worker memory/timeout before enabling them. + ## Rollback The cache is an optimization; inference correctness does not depend on it. diff --git a/scripts/agent_gan_inference_demo.py b/scripts/agent_gan_inference_demo.py index 1171618a..20d9172c 100644 --- a/scripts/agent_gan_inference_demo.py +++ b/scripts/agent_gan_inference_demo.py @@ -68,7 +68,13 @@ def main() -> int: parser.add_argument("--dashboard", default="http://127.0.0.1:8090") parser.add_argument("--api-key-file", default="~/.kakeya/network_api_key") parser.add_argument("--tokenizer-id", required=True) - parser.add_argument("--rounds", type=int, default=2) + parser.add_argument( + "--rounds", + type=int, + default=1, + help="Generator/Critic cycles. The 16GB Gemma worker supports one " + "reliably; larger values require more worker memory or timeout.", + ) parser.add_argument("--output-tokens", type=int, default=64) parser.add_argument("--report", default="/tmp/kakeya-agent-gan-demo.json") parser.add_argument("--skip-ensure", action="store_true")