From 9db3961ed6f6b9340c8a6691a6ee7ada6b75cdd5 Mon Sep 17 00:00:00 2001 From: Vladimir Yakunin Date: Fri, 11 Sep 2026 21:09:51 +0000 Subject: [PATCH] Say that a server-declared prompt is recorded as sent and is not the task The server block under inference.policy.server.* is the handshake metadata as the server sent it. A prompt field in it is the deployment's own declaration. It is fixed for the deployment. The instruction an episode sent is keys.TASK. The client reads only the declared stack and compress_images from the block, and nothing checks the rest. A test pins it: a declared prompt is recorded whole, and the task the episode sends reaches the wire unchanged, with no prompt beside it. Ticket: Positronic-Robotics/internal#1290 #refs --- docs/inference.md | 2 ++ positronic/offboard/tests/test_remote_policy.py | 14 ++++++++++++++ positronic/policy/keys.py | 3 +++ positronic/policy/remote.py | 1 + 4 files changed, 20 insertions(+) diff --git a/docs/inference.md b/docs/inference.md index 71cc4e9f6..e87b7320e 100644 --- a/docs/inference.md +++ b/docs/inference.md @@ -78,6 +78,8 @@ The model source (`checkpoints_dir`, `checkpoint`, device...) is fixed at server **What crosses the wire is the server's call, not the client's.** A server that wants smaller frames declares `RestrictImageSize` in its rig-side stack (640x640 by default); one behind a proxy with a message-size cap declares `remote(compress_images=True)` and the rig JPEG-encodes frames before sending. A server whose checkpoint speaks a different end-effector frame declares `ChangeEEFrame` with the transform placing that frame relative to the rig's `default`, and the rig converts poses (see [End-effector frames](codecs.md#end-effector-frames)). The client builds whatever the handshake declares, and only that — connecting to a server that declares no stack fails with an error naming the version it runs. What the declared stack must achieve is checked where it matters: the harness refuses to emit an action scheduled further than `MAX_ACTION_SKEW_SEC` from now, which is what a stack that never anchored its chunk to the rig's clock produces. +**The handshake is recorded as the server sent it.** Every episode stores the server's handshake metadata under `inference.policy.server.*`. The client reads the declared stack and `compress_images` from it and records the rest without a check. A `prompt` field there is the deployment's own declaration. It is fixed for the deployment and does not follow the episode. The instruction an episode sent is `task`. + > **Recording inference I/O:** Pass `--policy.recording_dir=s3://bucket/path` to write a rerun `.rrd` file per episode capturing the raw and server-side observation/action boundaries. Useful for debugging codec behavior and visualizing what the policy actually received. ## Local Inference diff --git a/positronic/offboard/tests/test_remote_policy.py b/positronic/offboard/tests/test_remote_policy.py index 9f2f6201e..c1d68918a 100644 --- a/positronic/offboard/tests/test_remote_policy.py +++ b/positronic/offboard/tests/test_remote_policy.py @@ -313,6 +313,20 @@ def test_remote_session_normalizes_single_dict(open_session): assert round_trip(session, rt, {}) == [{keys.ROBOT_COMMAND: 'X', 'timestamp': 0.0}] +def test_a_prompt_the_server_declares_is_recorded_as_sent_and_the_task_still_goes_out(open_session): + """A ``prompt`` in the handshake is the deployment's declaration. It is recorded whole under the server + block, it is not the episode's task, and the task the episode sends reaches the wire unchanged.""" + declared = 'Pick up the green cube and place it on the red cube.' + endpoint, mock_ws = _mock_endpoint(metadata={'model_name': 'm', 'prompt': declared}) + session, rt = open_session(endpoint) + + assert session.meta['server.prompt'] == declared + round_trip(session, rt, {keys.TASK: 'put the banana on the plate'}) + sent = mock_ws.infer.call_args.args[0] + assert sent[keys.TASK] == 'put the banana on the plate' + assert 'prompt' not in sent + + def test_remote_session_passes_through_none(open_session): endpoint, mock_ws = _mock_endpoint() mock_ws.infer.return_value = None diff --git a/positronic/policy/keys.py b/positronic/policy/keys.py index f679cfad8..07b0c6b66 100644 --- a/positronic/policy/keys.py +++ b/positronic/policy/keys.py @@ -3,6 +3,9 @@ # What a policy reports about itself through its ``meta``; a remote policy nests the server's meta under # ``SERVER``, and the harness records the result under ``POLICY_META``. ``TYPE`` names the policy at the top # level and the vendor under ``SERVER``, so a reader composes a prefix with a field: f'{SERVER_META}.{TYPE}'. +# The block under ``SERVER`` is the handshake metadata as the server sent it. The client reads its declared +# stack and ``compress_images`` from it and records the rest without a check. A ``prompt`` in it is the +# deployment's own declaration, fixed for that deployment. The instruction an episode sent is ``keys.TASK``. TYPE = 'type' CHECKPOINT_PATH = 'checkpoint_path' EXPERIMENT_NAME = 'experiment_name' diff --git a/positronic/policy/remote.py b/positronic/policy/remote.py index b6a4c7b8e..a5ffda9d5 100644 --- a/positronic/policy/remote.py +++ b/positronic/policy/remote.py @@ -103,6 +103,7 @@ def cancel(self): @property def meta(self) -> dict[str, Any]: + # The server block is the handshake as the server sent it, recorded whole and read nowhere here. return flatten_dict({policy_keys.TYPE: 'remote', policy_keys.SERVER: self._session.metadata}) def close(self):