perf(daemon-client): cache remote health by daemon identity

docs/adr/0006-daemon-rpc-protocol-version.md

@@ -24,6 +24,17 @@ Package `version` is diagnostic only and must not be used as a compatibility gat
 `rpcProtocolVersion` is treated as a legacy remote daemon and is allowed unless a later security or
 protocol decision explicitly retires legacy compatibility.
 
+A persistent client may cache a successful probe when `/health` advertises an instance ID for
+both the daemon endpoint and, if proxied, its upstream daemon. It sends those IDs as conditional
+headers on every cached RPC. A daemon refuses a stale instance with a typed HTTP 409 response
+before dispatch, and a proxy refuses its own stale instance before forwarding. The proxy passes
+the expected upstream instance to the daemon, which makes the same pre-dispatch check. On a
+mismatch the client clears the cache, probes `/health` again, and retries the refused RPC once
+only when the new peer's protocol is compatible. Peers without complete instance IDs retain
+per-command probes. A peer that advertises an instance ID must continue honoring the conditional
+RPC headers in future versions: older clients may hold a cached identity across a restart, and
+must receive a pre-dispatch refusal before retrying a mutating command.
+
 `rpcProtocolVersion` changes only when an older client and newer daemon, or newer client and older
 daemon, cannot safely communicate over the HTTP RPC boundary for existing commands. Bump it for
 breaking changes to:

packages/contracts/src/daemon-http.ts

@@ -38,24 +38,38 @@ export function buildDaemonHttpTenantHeaders(tenantId: string | undefined): Reco
 // an acknowledged-compatible entry (#1432).
 export const DAEMON_RPC_PROTOCOL_VERSION = 2;
 
+export const DAEMON_HTTP_INSTANCE_HEADER = 'x-agent-device-instance';
+export const DAEMON_HTTP_UPSTREAM_INSTANCE_HEADER = 'x-agent-device-upstream-instance';
+export const DAEMON_HTTP_INSTANCE_MISMATCH_HEADER = 'x-agent-device-instance-mismatch';
+
+export function buildDaemonInstanceMismatchRpcResponse<Id>(
+  id: Id,
+  message: string,
+  data: Record<string, unknown>,
+) {
+  return { jsonrpc: '2.0' as const, id, error: { code: -32001, message, data } };
+}
+
 export type DaemonHealthPayload = {
   ok: true;
   service: 'agent-device-daemon' | 'agent-device-proxy';
   version: string;
   rpcProtocolVersion: number;
+  instanceId?: string;
   upstream?: unknown;
 };
 
 export function buildDaemonHealthPayload(
   service: DaemonHealthPayload['service'],
   version: string,
-  options: { upstream?: unknown } = {},
+  options: { upstream?: unknown; instanceId?: string } = {},
 ): DaemonHealthPayload {
   return {
     ok: true,
     service,
     version,
     rpcProtocolVersion: DAEMON_RPC_PROTOCOL_VERSION,
+    ...(options.instanceId !== undefined ? { instanceId: options.instanceId } : {}),
     ...(options.upstream !== undefined ? { upstream: options.upstream } : {}),
   };
 }

src/__tests__/daemon-proxy.test.ts

@@ -8,6 +8,9 @@ import { getRequestSignal } from '@agent-device/host-kit/request';
 import { executeRunScriptHttpRequest } from '@agent-device/maestro/run-script-http';
 import {
   DAEMON_HTTP_NETWORK_ACCESS_HEADER,
+  DAEMON_HTTP_INSTANCE_HEADER,
+  DAEMON_HTTP_INSTANCE_MISMATCH_HEADER,
+  DAEMON_HTTP_UPSTREAM_INSTANCE_HEADER,
   DAEMON_HTTP_PUBLIC_NETWORK_ACCESS,
   DAEMON_RPC_PROTOCOL_VERSION,
 } from '@agent-device/contracts/daemon-http';
@@ -32,24 +35,34 @@ test('daemon proxy forwards rpc requests with upstream daemon token', async (t)
   let upstreamAuth = '';
   let upstreamTokenHeader = '';
   let upstreamNetworkAccess = '';
+  let upstreamExpectedInstance = '';
+  let upstreamExecutions = 0;
   let upstreamBody: Record<string, any> | undefined;
   const upstream = http.createServer((req, res) => {
     if (req.url === '/health') {
       res.setHeader('content-type', 'application/json');
-      res.end(JSON.stringify({ ok: true }));
+      res.end(JSON.stringify({ ok: true, instanceId: 'upstream-instance' }));
       return;
     }
     assert.equal(req.url, '/rpc');
     upstreamAuth = String(req.headers.authorization ?? '');
     upstreamTokenHeader = String(req.headers['x-agent-device-token'] ?? '');
     upstreamNetworkAccess = String(req.headers[DAEMON_HTTP_NETWORK_ACCESS_HEADER] ?? '');
+    upstreamExpectedInstance = String(req.headers[DAEMON_HTTP_INSTANCE_HEADER] ?? '');
+    if (upstreamExpectedInstance && upstreamExpectedInstance !== 'upstream-instance') {
+      res.statusCode = 409;
+      res.setHeader(DAEMON_HTTP_INSTANCE_MISMATCH_HEADER, 'true');
+      res.end(JSON.stringify({ jsonrpc: '2.0', id: 'req-1', error: { code: -32001 } }));
+      return;
+    }
     let body = '';
     req.setEncoding('utf8');
     req.on('data', (chunk) => {
       body += chunk;
     });
     req.on('end', () => {
       upstreamBody = JSON.parse(body) as Record<string, any>;
+      upstreamExecutions += 1;
       res.setHeader('content-type', 'application/json');
       res.end(
         JSON.stringify({
@@ -69,11 +82,18 @@ test('daemon proxy forwards rpc requests with upstream daemon token', async (t)
 
   try {
     const proxyPort = await listenOnLoopback(proxy);
+    const healthResponse = await fetch(`http://127.0.0.1:${proxyPort}/agent-device/health`);
+    const health = (await healthResponse.json()) as Record<string, any>;
+    assert.equal(healthResponse.status, 200);
+    assert.equal(typeof health.instanceId, 'string');
+    assert.equal(typeof health.upstream?.instanceId, 'string');
     const response = await fetch(`http://127.0.0.1:${proxyPort}/agent-device/rpc`, {
       method: 'POST',
       headers: {
         'content-type': 'application/json',
         authorization: 'Bearer proxy-secret',
+        [DAEMON_HTTP_INSTANCE_HEADER]: health.instanceId as string,
+        [DAEMON_HTTP_UPSTREAM_INSTANCE_HEADER]: health.upstream.instanceId as string,
       },
       body: JSON.stringify({
         jsonrpc: '2.0',
@@ -98,8 +118,46 @@ test('daemon proxy forwards rpc requests with upstream daemon token', async (t)
     assert.equal(upstreamAuth, 'Bearer daemon-secret');
     assert.equal(upstreamTokenHeader, 'daemon-secret');
     assert.equal(upstreamNetworkAccess, DAEMON_HTTP_PUBLIC_NETWORK_ACCESS);
+    assert.equal(upstreamExpectedInstance, 'upstream-instance');
+    assert.equal(upstreamExecutions, 1);
     assert.equal(upstreamBody?.params?.token, 'daemon-secret');
     assert.equal(upstreamBody?.params?.command, 'devices');
+    const staleProxyResponse = await fetch(`http://127.0.0.1:${proxyPort}/agent-device/rpc`, {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        authorization: 'Bearer proxy-secret',
+        [DAEMON_HTTP_INSTANCE_HEADER]: 'old-proxy',
+      },
+      body: JSON.stringify({
+        jsonrpc: '2.0',
+        id: 'req-1',
+        method: 'agent_device.command',
+        params: {},
+      }),
+    });
+    assert.equal(staleProxyResponse.status, 409);
+    assert.equal(staleProxyResponse.headers.get(DAEMON_HTTP_INSTANCE_MISMATCH_HEADER), 'true');
+    assert.equal(upstreamExecutions, 1);
+
+    const staleUpstreamResponse = await fetch(`http://127.0.0.1:${proxyPort}/agent-device/rpc`, {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        authorization: 'Bearer proxy-secret',
+        [DAEMON_HTTP_INSTANCE_HEADER]: health.instanceId as string,
+        [DAEMON_HTTP_UPSTREAM_INSTANCE_HEADER]: 'old-upstream',
+      },
+      body: JSON.stringify({
+        jsonrpc: '2.0',
+        id: 'req-1',
+        method: 'agent_device.command',
+        params: {},
+      }),
+    });
+    assert.equal(staleUpstreamResponse.status, 409);
+    assert.equal(staleUpstreamResponse.headers.get(DAEMON_HTTP_INSTANCE_MISMATCH_HEADER), 'true');
+    assert.equal(upstreamExecutions, 1);
   } finally {
     await closeLoopbackServer(proxy);
     await closeLoopbackServer(upstream);
@@ -399,6 +457,7 @@ test('daemon proxy leaves health endpoint unauthenticated', async (t) => {
     assert.equal(payload.service, 'agent-device-proxy');
     assert.equal(typeof payload.version, 'string');
     assert.equal(payload.rpcProtocolVersion, DAEMON_RPC_PROTOCOL_VERSION);
+    assert.equal(typeof payload.instanceId, 'string');
     assert.deepEqual(payload.upstream, { ok: true });
     assert.equal(upstreamAuth, 'Bearer daemon-secret');
     assert.equal(upstreamTokenHeader, 'daemon-secret');