diff --git a/docs/inference.md b/docs/inference.md index 71cc4e9f6..e87b7320e 100644 --- a/docs/inference.md +++ b/docs/inference.md @@ -78,6 +78,8 @@ The model source (`checkpoints_dir`, `checkpoint`, device...) is fixed at server **What crosses the wire is the server's call, not the client's.** A server that wants smaller frames declares `RestrictImageSize` in its rig-side stack (640x640 by default); one behind a proxy with a message-size cap declares `remote(compress_images=True)` and the rig JPEG-encodes frames before sending. A server whose checkpoint speaks a different end-effector frame declares `ChangeEEFrame` with the transform placing that frame relative to the rig's `default`, and the rig converts poses (see [End-effector frames](codecs.md#end-effector-frames)). The client builds whatever the handshake declares, and only that — connecting to a server that declares no stack fails with an error naming the version it runs. What the declared stack must achieve is checked where it matters: the harness refuses to emit an action scheduled further than `MAX_ACTION_SKEW_SEC` from now, which is what a stack that never anchored its chunk to the rig's clock produces. +**The handshake is recorded as the server sent it.** Every episode stores the server's handshake metadata under `inference.policy.server.*`. The client reads the declared stack and `compress_images` from it and records the rest without a check. A `prompt` field there is the deployment's own declaration. It is fixed for the deployment and does not follow the episode. The instruction an episode sent is `task`. + > **Recording inference I/O:** Pass `--policy.recording_dir=s3://bucket/path` to write a rerun `.rrd` file per episode capturing the raw and server-side observation/action boundaries. Useful for debugging codec behavior and visualizing what the policy actually received. ## Local Inference diff --git a/positronic/offboard/tests/test_remote_policy.py b/positronic/offboard/tests/test_remote_policy.py index 9f2f6201e..c1d68918a 100644 --- a/positronic/offboard/tests/test_remote_policy.py +++ b/positronic/offboard/tests/test_remote_policy.py @@ -313,6 +313,20 @@ def test_remote_session_normalizes_single_dict(open_session): assert round_trip(session, rt, {}) == [{keys.ROBOT_COMMAND: 'X', 'timestamp': 0.0}] +def test_a_prompt_the_server_declares_is_recorded_as_sent_and_the_task_still_goes_out(open_session): + """A ``prompt`` in the handshake is the deployment's declaration. It is recorded whole under the server + block, it is not the episode's task, and the task the episode sends reaches the wire unchanged.""" + declared = 'Pick up the green cube and place it on the red cube.' + endpoint, mock_ws = _mock_endpoint(metadata={'model_name': 'm', 'prompt': declared}) + session, rt = open_session(endpoint) + + assert session.meta['server.prompt'] == declared + round_trip(session, rt, {keys.TASK: 'put the banana on the plate'}) + sent = mock_ws.infer.call_args.args[0] + assert sent[keys.TASK] == 'put the banana on the plate' + assert 'prompt' not in sent + + def test_remote_session_passes_through_none(open_session): endpoint, mock_ws = _mock_endpoint() mock_ws.infer.return_value = None diff --git a/positronic/policy/keys.py b/positronic/policy/keys.py index f679cfad8..07b0c6b66 100644 --- a/positronic/policy/keys.py +++ b/positronic/policy/keys.py @@ -3,6 +3,9 @@ # What a policy reports about itself through its ``meta``; a remote policy nests the server's meta under # ``SERVER``, and the harness records the result under ``POLICY_META``. ``TYPE`` names the policy at the top # level and the vendor under ``SERVER``, so a reader composes a prefix with a field: f'{SERVER_META}.{TYPE}'. +# The block under ``SERVER`` is the handshake metadata as the server sent it. The client reads its declared +# stack and ``compress_images`` from it and records the rest without a check. A ``prompt`` in it is the +# deployment's own declaration, fixed for that deployment. The instruction an episode sent is ``keys.TASK``. TYPE = 'type' CHECKPOINT_PATH = 'checkpoint_path' EXPERIMENT_NAME = 'experiment_name' diff --git a/positronic/policy/remote.py b/positronic/policy/remote.py index b6a4c7b8e..a5ffda9d5 100644 --- a/positronic/policy/remote.py +++ b/positronic/policy/remote.py @@ -103,6 +103,7 @@ def cancel(self): @property def meta(self) -> dict[str, Any]: + # The server block is the handshake as the server sent it, recorded whole and read nowhere here. return flatten_dict({policy_keys.TYPE: 'remote', policy_keys.SERVER: self._session.metadata}) def close(self):