diff --git a/packages/command-registry/package.json b/packages/command-registry/package.json index 3fef336010..871043cc3d 100644 --- a/packages/command-registry/package.json +++ b/packages/command-registry/package.json @@ -83,6 +83,10 @@ "types": "./src/flag-groups.ts", "default": "./src/flag-groups.ts" }, + "./flag-definitions-workflow": { + "types": "./src/flag-definitions-workflow.ts", + "default": "./src/flag-definitions-workflow.ts" + }, "./command-text": { "types": "./src/command-text.ts", "default": "./src/command-text.ts" diff --git a/packages/command-registry/src/__tests__/device-claim-policy.test.ts b/packages/command-registry/src/__tests__/device-claim-policy.test.ts index 97a40dc68f..2266ffa873 100644 --- a/packages/command-registry/src/__tests__/device-claim-policy.test.ts +++ b/packages/command-registry/src/__tests__/device-claim-policy.test.ts @@ -52,6 +52,7 @@ test('every command that deviates from require-owner is a reviewed, diffable set ], observe: ['apps', 'appstate', 'capabilities', 'device', 'devices', 'doctor', 'takeover'], none: [ + 'act', 'artifacts', 'auth', 'batch', @@ -74,6 +75,7 @@ test('every command that deviates from require-owner is a reviewed, diffable set 'session', 'session_list', 'session_save_script', + 'suggest', 'web', ], }); diff --git a/packages/command-registry/src/flag-definitions-workflow.ts b/packages/command-registry/src/flag-definitions-workflow.ts index 5399fe3e0f..5ee836f998 100644 --- a/packages/command-registry/src/flag-definitions-workflow.ts +++ b/packages/command-registry/src/flag-definitions-workflow.ts @@ -1,6 +1,24 @@ import { SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS } from '@agent-device/contracts/capture'; import type { FlagDefinition } from './flag-types.ts'; +/** + * The policy head's CLI vocabulary and defaults. + * + * They live beside the flags that publish them so the prose and the runtime read one declaration: + * `--max-steps` and `--min-confidence` state these numbers in their help, and the act loop applies + * them. Kept here rather than in the policy modules because the CLI surface is evaluated at + * startup while the policy runtime loads on demand, and a constant may not drag the second into + * the first. + */ +export const POLICY_PROVIDER_NAMES = ['jev'] as const; +export const DEFAULT_POLICY_PROVIDER: (typeof POLICY_PROVIDER_NAMES)[number] = 'jev'; +export const DEFAULT_POLICY_MAX_STEPS = 12; +export const DEFAULT_POLICY_MIN_CONFIDENCE = 0.4; +/** Environment entry the jev provider reads its credential from; never a flag. */ +export const POLICY_API_KEY_ENV = 'TYPESAFE_API_KEY'; +/** Prefix for per-key text entries, so a secret never has to appear in argv. */ +export const POLICY_INPUT_ENV_PREFIX = 'AGENT_DEVICE_INPUT_'; + export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ { key: 'replayUpdate', @@ -195,7 +213,7 @@ export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ min: 1, max: 1000, usageLabel: '--max-steps ', - usageDescription: 'Batch: maximum number of allowed steps', + usageDescription: `Batch: maximum allowed steps; act: stop the loop after this many steps (default ${DEFAULT_POLICY_MAX_STEPS})`, projectConfig: true, recorded: false, }, @@ -322,6 +340,37 @@ export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ projectConfig: false, recorded: false, }, + { + key: 'policy', + names: ['--policy'], + type: 'string', + usageLabel: '--policy ', + usageDescription: `suggest/act: policy head that decides the next element (default ${DEFAULT_POLICY_PROVIDER})`, + projectConfig: true, + recorded: false, + }, + { + key: 'minConfidence', + names: ['--min-confidence'], + type: 'number', + usageLabel: '--min-confidence ', + usageDescription: `act: escalate instead of acting below this policy confidence (default ${DEFAULT_POLICY_MIN_CONFIDENCE})`, + projectConfig: false, + recorded: false, + }, + { + key: 'policyInput', + names: ['--input'], + type: 'string', + multiple: true, + usageLabel: '--input ', + usageDescription: + 'act: repeatable text the loop may enter, matched to a field by identifier or label; the loop never generates text', + inputDescription: + 'Text the loop may enter, as key=value. The key matches a field identifier or label.', + projectConfig: false, + recorded: false, + }, { key: 'overlayRefs', names: ['--overlay-refs'], diff --git a/packages/command-registry/src/registry.ts b/packages/command-registry/src/registry.ts index d213c386aa..8b138b462d 100644 --- a/packages/command-registry/src/registry.ts +++ b/packages/command-registry/src/registry.ts @@ -1538,6 +1538,42 @@ export const RAW_COMMAND_DESCRIPTORS = [ }, // -- local client-backed CLI/MCP commands (no daemon route/capability) -- + // The policy commands compose `snapshot`, `press`, and `fill` from the client process, so they + // own no daemon route of their own and stay out of the public catalog, which requires one. + { + name: 'suggest', + deviceClaimPolicy: 'none', + ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/interaction/index.ts'] as const } : {}), + catalog: { group: 'local-cli' }, + recordsSessionAction: false, + timeoutPolicy: DEFAULT_TIMEOUT_POLICY, + batchable: false, + platformExecution: NO_PLATFORM_EXECUTION, + }, + { + name: 'act', + deviceClaimPolicy: 'none', + ...(ownerFilesEnabled + ? { + ownerFiles: [ + 'src/commands/interaction/index.ts', + 'src/commands/policy/act-loop.ts', + ] as const, + } + : {}), + catalog: { group: 'local-cli' }, + recordsSessionAction: false, + // A run is a sequence of ordinary commands, each already under its own envelope; a client + // envelope over the whole loop would abort a run that is still making progress. `--max-steps` + // is the budget that bounds it. + timeoutPolicy: { + ...DEFAULT_TIMEOUT_POLICY, + envelopeMs: 'unbounded', + onTimeout: 'preserve-daemon', + }, + batchable: false, + platformExecution: NO_PLATFORM_EXECUTION, + }, { name: 'debug', deviceClaimPolicy: 'none', diff --git a/packages/contracts/src/cli-flags.ts b/packages/contracts/src/cli-flags.ts index c5ba3f1649..da804a7e0a 100644 --- a/packages/contracts/src/cli-flags.ts +++ b/packages/contracts/src/cli-flags.ts @@ -160,6 +160,12 @@ export type CliFlags = CloudProviderProfileFields & runtime?: SessionRuntimeHints; }>; out?: string; + /** suggest/act: which policy head answers "what next". */ + policy?: string; + /** act: escalate instead of acting below this confidence. */ + minConfidence?: number; + /** act: repeated `key=value` text the loop may enter; it never generates text itself. */ + policyInput?: string[]; help: boolean; version: boolean; }; diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index f8da3addcd..2782f6c978 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -326,6 +326,9 @@ function summarizeProviderScenarioFlagExclusions() { 'shardAll', 'shardSplit', 'searchPath', + 'policy', + 'minConfidence', + 'policyInput', 'stepsFile', 'proxyHost', 'proxyPort', diff --git a/skills/agent-device/SKILL.md b/skills/agent-device/SKILL.md index a2447b3ed5..496588aba9 100644 --- a/skills/agent-device/SKILL.md +++ b/skills/agent-device/SKILL.md @@ -19,4 +19,6 @@ Reaching an off-screen target is one command, not a scroll-and-check loop: `scro Copy refs byte-for-byte: `@e12`, `@e12~s4` — keep the `@` and any `~sN`. Prefer current refs, then `id`/`label`/`role` selectors; coordinates are a last resort. If snapshot reports sparse/AX-unavailable, its refs and selectors are invalid: run `agent-device screenshot`, inspect the image, use coordinates, then retry `snapshot -i` after navigating. Otherwise run `snapshot -i` only when the diff lacks the next target. +With `TYPESAFE_API_KEY` set, `suggest ""` names the next element to act on in roughly the time a snapshot takes, and `act "" --input key=value` runs the whole snapshot-decide-act loop. Text is never generated; a field with no `--input` entry escalates. Both are always listed; without the key each refuses with a typed error and the loop above is the path. On `Jev unavailable: `, fall back to that loop. + Error output includes corrective hints; follow them instead of re-planning. Only when the task is specialized (for example gestures, scripting, TV, macOS, remote, or debugging) or a command shape is unclear, run `agent-device help `. `agent-device --help` lists topics, but is not a startup step. diff --git a/src/__tests__/cli-client-commands.test.ts b/src/__tests__/cli-client-commands.test.ts index b3382c6ba9..6766a22947 100644 --- a/src/__tests__/cli-client-commands.test.ts +++ b/src/__tests__/cli-client-commands.test.ts @@ -1168,10 +1168,8 @@ function createStubClient(params: { return { command, devices: { + ...createThrowingMethodGroup(), list: async () => [], - capabilities: unexpectedCommandCall, - boot: unexpectedCommandCall, - shutdown: unexpectedCommandCall, }, sessions: { list: async () => [], @@ -1272,6 +1270,7 @@ function createStubClient(params: { events: params.events ?? unexpectedCommandCall, }, debug: createThrowingMethodGroup(), + policy: createThrowingMethodGroup(), recording: createThrowingMethodGroup(), settings: { update: params.updateSettings ?? unexpectedCommandCall, diff --git a/src/__tests__/command-descriptor-timeout-policy.test.ts b/src/__tests__/command-descriptor-timeout-policy.test.ts index d12d682520..4b9c38b207 100644 --- a/src/__tests__/command-descriptor-timeout-policy.test.ts +++ b/src/__tests__/command-descriptor-timeout-policy.test.ts @@ -67,10 +67,14 @@ test('daemon-preserving timeout commands are a bounded, reviewed set', () => { // sessions the daemon owns, so a client-side timeout must not SIGKILL the // daemon mid-create/mid-release and orphan them (and every other provider // session held). + // `act` joined with the policy head: a run is a sequence of ordinary commands, each already + // under its own envelope, so a client-side timeout over the whole loop must not reset the + // daemon and destroy the session the remaining steps still need. const preserving = commandDescriptors .filter((descriptor) => descriptor.timeoutPolicy.onTimeout === 'preserve-daemon') .map((descriptor) => descriptor.name); assert.deepEqual(preserving.sort(), [ + 'act', 'back', 'click', 'fill', @@ -146,6 +150,9 @@ test('request envelopes deviating from the default are bounded, reviewed sets', // #1774: base allocation budget (300s) + client/daemon race margin (30s). lease_allocate: 330_000, test: 'unbounded', + // A policy run is bounded by --max-steps, not by wall clock; each step it issues carries its + // own envelope, and an outer one would abort a run that is still making progress. + act: 'unbounded', }; for (const descriptor of commandDescriptors) { const expected = EXPECTED_ENVELOPES[descriptor.name] ?? 90_000; diff --git a/src/__tests__/remote-connection.fixtures.ts b/src/__tests__/remote-connection.fixtures.ts index b121ac06cd..7fdfd4a041 100644 --- a/src/__tests__/remote-connection.fixtures.ts +++ b/src/__tests__/remote-connection.fixtures.ts @@ -116,6 +116,7 @@ export function createTestClient( batch: createThrowingMethodGroup(), observability: createThrowingMethodGroup(), debug: createThrowingMethodGroup(), + policy: createThrowingMethodGroup(), recording: createThrowingMethodGroup(), settings: createThrowingMethodGroup(), }; diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index e3a0cd60af..737187b7a6 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -55,6 +55,7 @@ import { resolveSessionName, } from './client/client-normalizers.ts'; import type { AgentDeviceClient, MetroPrepareResult } from './client/client-types.ts'; +import type { PolicyClientCalls } from './client/client-policy.ts'; import { INTERNAL_COMMANDS } from '@agent-device/command-registry/catalog'; import { buildRequestFlags } from './commands/command-flags.ts'; import { @@ -135,6 +136,24 @@ export function createAgentDeviceClient( const resolveRequestSession = (options: InternalRequestOptions = {}) => resolveSessionName(mergeClientOptions(config, options).session); + /** + * The three commands the policy loop composes. They are the ordinary daemon commands, so a + * policy-driven run carries the same claims, ref frames, and recording behaviour as a run an + * agent drives by hand. + */ + const policyClientCalls: PolicyClientCalls = { + snapshot: async (options) => { + const session = resolveRequestSession(options); + const data = await executeCommand>('snapshot', options); + const result = normalizeSnapshotResult(data, session); + return { nodes: result.nodes, refsGeneration: result.refsGeneration }; + }, + press: async ({ ref, ...options }) => + await executeCommand('press', { ...options, ...refTarget(ref) }), + fill: async ({ ref, text, ...options }) => + await executeCommand('fill', { ...options, ...refTarget(ref, { text }) }), + }; + return { command: { wait: async (options) => await executeCommand>('wait', options), @@ -423,6 +442,18 @@ export function createAgentDeviceClient( return symbolicateCrashArtifact({ cwd: options.cwd ?? config.cwd, ...options }); }, }, + policy: { + // Loaded on demand: the policy head is optional, and nothing about it belongs in the + // module closure every CLI invocation evaluates at startup. + suggest: async (options) => { + const { suggestPolicyAction } = await import('./client/client-policy.ts'); + return await suggestPolicyAction(policyClientCalls, options, process.env); + }, + act: async (options) => { + const { runPolicyAct } = await import('./client/client-policy.ts'); + return await runPolicyAct(policyClientCalls, options, process.env); + }, + }, recording: { record: async (options) => await executeCommand>('record', options), trace: async (options) => await executeCommand>('trace', options), @@ -433,6 +464,15 @@ export function createAgentDeviceClient( }; } +/** + * An `@ref` interaction target plus any command-specific fields, in the shape `executeCommand` + * forwards to the daemon. The request options type is deliberately narrow; interaction fields + * reach the wire through the same untyped bag every other interaction call uses. + */ +function refTarget(ref: string, extra: Record = {}): InternalRequestOptions { + return { ref, ...extra } as InternalRequestOptions; +} + function panGestureInput(options: PanOptions): InternalRequestOptions & Record { const { x, y, dx, dy, ...common } = options; return { ...common, kind: 'pan', origin: { x, y }, delta: { x: dx, y: dy } }; diff --git a/src/client/client-policy.test.ts b/src/client/client-policy.test.ts new file mode 100644 index 0000000000..3eb72a898d --- /dev/null +++ b/src/client/client-policy.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, test } from 'vitest'; +import { + createPolicyDevicePort, + runPolicyAct, + suggestPolicyAction, + type PolicyClientCalls, +} from './client-policy.ts'; + +/** A device surface that fails the test if the commands reach it. */ +function forbiddenDevice(): PolicyClientCalls & { touched: string[] } { + const touched: string[] = []; + return { + touched, + snapshot: async () => { + touched.push('snapshot'); + return { nodes: [] }; + }, + press: async () => { + touched.push('press'); + }, + fill: async () => { + touched.push('fill'); + }, + }; +} + +describe('policy commands without a key', () => { + test('suggest refuses before reading the screen', async () => { + const device = forbiddenDevice(); + await expect(suggestPolicyAction(device, { goal: 'sign in' }, {})).rejects.toThrow( + expect.objectContaining({ + code: 'INVALID_ARGS', + message: expect.stringContaining('TYPESAFE_API_KEY is not set'), + }), + ); + expect(device.touched).toEqual([]); + }); + + test('act refuses before reading the screen', async () => { + const device = forbiddenDevice(); + await expect(runPolicyAct(device, { goal: 'sign in' }, {})).rejects.toThrow( + expect.objectContaining({ code: 'INVALID_ARGS' }), + ); + expect(device.touched).toEqual([]); + }); + + test('the refusal names both the way to enable it and the way without it', async () => { + const device = forbiddenDevice(); + await expect(suggestPolicyAction(device, { goal: 'sign in' }, {})).rejects.toMatchObject({ + details: { + reason: 'policy-provider-unconfigured', + hint: expect.stringContaining('Export TYPESAFE_API_KEY'), + }, + }); + await expect(suggestPolicyAction(device, { goal: 'sign in' }, {})).rejects.toMatchObject({ + details: { hint: expect.stringContaining('fall back to agent-driven policy') }, + }); + }); + + test('an empty key is treated as no key', async () => { + const device = forbiddenDevice(); + await expect( + suggestPolicyAction(device, { goal: 'sign in' }, { TYPESAFE_API_KEY: ' ' }), + ).rejects.toThrow(expect.objectContaining({ code: 'INVALID_ARGS' })); + expect(device.touched).toEqual([]); + }); +}); + +describe("the loop's device port", () => { + /** Records what the port sent to the client, with the snapshot generations it was handed. */ + function recordingCalls(generations: readonly (number | undefined)[]): PolicyClientCalls & { + sent: string[]; + } { + const sent: string[] = []; + let index = 0; + return { + sent, + snapshot: async () => { + const refsGeneration = generations[Math.min(index++, generations.length - 1)]; + sent.push(`snapshot~${refsGeneration ?? 'none'}`); + return { nodes: [], ...(refsGeneration === undefined ? {} : { refsGeneration }) }; + }, + press: async ({ ref }) => { + sent.push(`press ${ref}`); + }, + fill: async ({ ref, text }) => { + sent.push(`fill ${ref} ${text}`); + }, + }; + } + + test('pins a mutation to the generation of the snapshot that issued its ref', async () => { + const calls = recordingCalls([7]); + const port = createPolicyDevicePort(calls, {}); + await port.snapshot(); + await port.press('@e12'); + await port.fill('@e12', 'hello'); + expect(calls.sent).toEqual(['snapshot~7', 'press @e12~s7', 'fill @e12~s7 hello']); + }); + + test('re-pins after a later snapshot reissues refs', async () => { + const calls = recordingCalls([7, 8]); + const port = createPolicyDevicePort(calls, {}); + await port.snapshot(); + await port.press('@e12'); + await port.snapshot(); + await port.press('@e12'); + expect(calls.sent).toEqual(['snapshot~7', 'press @e12~s7', 'snapshot~8', 'press @e12~s8']); + }); + + test('sends a plain ref when the daemon issued no generation', async () => { + const calls = recordingCalls([undefined]); + const port = createPolicyDevicePort(calls, {}); + await port.snapshot(); + await port.press('e12'); + expect(calls.sent).toEqual(['snapshot~none', 'press @e12']); + }); + + test('reports a duration for every operation', async () => { + const port = createPolicyDevicePort(recordingCalls([1]), {}); + const snapshot = await port.snapshot(); + expect(snapshot.ms).toBeGreaterThanOrEqual(0); + expect(await port.press('@e1')).toBeGreaterThanOrEqual(0); + expect(await port.fill('@e1', 'x')).toBeGreaterThanOrEqual(0); + }); + + test("carries the caller's device selection into every call", async () => { + const seen: Array> = []; + const port = createPolicyDevicePort( + { + snapshot: async (options) => { + seen.push(options); + return { nodes: [] }; + }, + press: async (options) => { + seen.push(options); + }, + fill: async (options) => { + seen.push(options); + }, + }, + { udid: 'SIM-1' }, + ); + await port.snapshot(); + await port.press('@e1'); + expect(seen.every((options) => options.udid === 'SIM-1')).toBe(true); + expect(seen[0]).toMatchObject({ interactiveOnly: true }); + }); +}); diff --git a/src/client/client-policy.ts b/src/client/client-policy.ts new file mode 100644 index 0000000000..491a3a7896 --- /dev/null +++ b/src/client/client-policy.ts @@ -0,0 +1,146 @@ +import type { + AgentDeviceRequestOverrides, + AgentDeviceSelectionOptions, +} from '@agent-device/contracts/client'; +import { + DEFAULT_POLICY_MAX_STEPS, + DEFAULT_POLICY_MIN_CONFIDENCE, + DEFAULT_POLICY_PROVIDER, +} from '@agent-device/command-registry/flag-definitions-workflow'; +import { runPolicyActLoop, type PolicyDevicePort } from '../commands/policy/act-loop.ts'; +import { + screenLabel, + toPolicyCandidates, + type PolicySnapshotNode, +} from '../commands/policy/candidate-elements.ts'; +import type { + PolicyActResult, + PolicyActStep, + PolicySuggestResult, +} from '../commands/policy/policy-contract.ts'; +import { createPolicyProvider } from '../commands/policy/policy-provider.ts'; +import { parsePolicyTextInputs } from '../commands/policy/text-inputs.ts'; + +type PolicyCommonOptions = AgentDeviceRequestOverrides & AgentDeviceSelectionOptions; + +export type PolicySuggestClientOptions = PolicyCommonOptions & { + goal: string; + policy?: string; +}; + +export type PolicyActClientOptions = PolicySuggestClientOptions & { + maxSteps?: number; + minConfidence?: number; + /** Repeated `key=value` tokens naming text the loop may enter. */ + inputs?: readonly string[]; + onStep?: (step: PolicyActStep) => void; +}; + +/** + * The client calls the loop needs, narrowed to three. Narrowing here rather than passing the whole + * client keeps the loop's device surface visible at the seam where it is granted. + */ +export type PolicyClientCalls = { + snapshot: (options: PolicyCommonOptions & { interactiveOnly: true }) => Promise<{ + nodes: PolicySnapshotNode[]; + refsGeneration?: number; + }>; + press: (options: PolicyCommonOptions & { ref: string; settle?: boolean }) => Promise; + fill: ( + options: PolicyCommonOptions & { ref: string; text: string; settle?: boolean }, + ) => Promise; +}; + +export async function suggestPolicyAction( + calls: PolicyClientCalls, + options: PolicySuggestClientOptions, + env: Readonly>, +): Promise { + const provider = createPolicyProvider(options.policy ?? DEFAULT_POLICY_PROVIDER, env); + const common = commonOf(options); + const startedAt = Date.now(); + const snapshot = await calls.snapshot({ ...common, interactiveOnly: true }); + const snapshotMs = Date.now() - startedAt; + + const candidates = toPolicyCandidates(snapshot.nodes); + const screen = screenLabel(candidates); + const decision = await provider.decide({ + goal: options.goal, + candidates, + history: [], + screen, + // `suggest` reports what the screen affords; it enters no text, so it claims none is ready. + textReadyRefs: [], + }); + return { goal: options.goal, decision, candidates, snapshotMs, screen }; +} + +export async function runPolicyAct( + calls: PolicyClientCalls, + options: PolicyActClientOptions, + env: Readonly>, +): Promise { + const provider = createPolicyProvider(options.policy ?? DEFAULT_POLICY_PROVIDER, env); + return await runPolicyActLoop({ + goal: options.goal, + provider, + device: createPolicyDevicePort(calls, commonOf(options)), + inputs: parsePolicyTextInputs(options.inputs ?? []), + env, + maxSteps: options.maxSteps ?? DEFAULT_POLICY_MAX_STEPS, + minConfidence: options.minConfidence ?? DEFAULT_POLICY_MIN_CONFIDENCE, + ...(options.onStep ? { onStep: options.onStep } : {}), + }); +} + +/** + * Bind the loop to a live session. + * + * Every mutation is pinned to the generation of the snapshot that issued its ref (ADR 0014), so a + * decision made against a screen that has since changed is refused by the daemon rather than + * landing on whatever element inherited the ref. Exported because that pinning rule is the seam + * worth testing on its own: a run that loses it fails silently, on whichever element inherited + * the ref. + */ +export function createPolicyDevicePort( + calls: PolicyClientCalls, + common: PolicyCommonOptions, +): PolicyDevicePort { + let generation: number | undefined; + const pinned = (ref: string): string => { + const body = ref.startsWith('@') ? ref.slice(1) : ref; + return generation === undefined ? `@${body}` : `@${body}~s${generation}`; + }; + const timed = async (run: () => Promise): Promise => { + const startedAt = Date.now(); + await run(); + return Date.now() - startedAt; + }; + + return { + snapshot: async () => { + const startedAt = Date.now(); + const result = await calls.snapshot({ ...common, interactiveOnly: true }); + generation = result.refsGeneration; + return { nodes: result.nodes, ms: Date.now() - startedAt }; + }, + press: async (ref) => + await timed(async () => await calls.press({ ...common, ref: pinned(ref), settle: true })), + fill: async (ref, text) => + await timed( + async () => await calls.fill({ ...common, ref: pinned(ref), text, settle: true }), + ), + }; +} + +function commonOf(options: PolicySuggestClientOptions): PolicyCommonOptions { + const { goal: _goal, policy: _policy, ...common } = options as PolicyActClientOptions; + const { + maxSteps: _maxSteps, + minConfidence: _minConfidence, + inputs: _inputs, + onStep: _onStep, + ...rest + } = common as PolicyActClientOptions; + return rest as PolicyCommonOptions; +} diff --git a/src/client/client-types.ts b/src/client/client-types.ts index 05523a35b3..48e4421795 100644 --- a/src/client/client-types.ts +++ b/src/client/client-types.ts @@ -139,6 +139,8 @@ import type { } from '@agent-device/contracts/remote'; import type { BatchRunResult } from '@agent-device/command-registry/batch'; +import type { PolicyActClientOptions, PolicySuggestClientOptions } from './client-policy.ts'; +import type { PolicyActResult, PolicySuggestResult } from '../commands/policy/policy-contract.ts'; import type { AgentArtifactsResult, @@ -287,6 +289,10 @@ export type AgentDeviceClient = { debug: { symbols: (options: DebugSymbolsOptions) => Promise; }; + policy: { + suggest: (options: PolicySuggestClientOptions) => Promise; + act: (options: PolicyActClientOptions) => Promise; + }; recording: { record: (options: RecordOptions) => Promise>; trace: (options: TraceOptions) => Promise>; diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index 52ff2d5391..7728b9432e 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -20,18 +20,35 @@ import type { TypeTextOptions, } from '@agent-device/contracts/client'; import type { CommandSchemaOverride } from '@agent-device/command-registry/command-schema'; +import { + DEFAULT_POLICY_MAX_STEPS, + DEFAULT_POLICY_MIN_CONFIDENCE, + DEFAULT_POLICY_PROVIDER, + POLICY_API_KEY_ENV, + POLICY_INPUT_ENV_PREFIX, + POLICY_PROVIDER_NAMES, +} from '@agent-device/command-registry/flag-definitions-workflow'; +import { AppError } from '@agent-device/kernel/errors'; import { REPEATED_TOUCH_FLAGS, SELECTOR_SNAPSHOT_FLAGS, } from '@agent-device/command-registry/flag-groups'; import { postActionObservationCliFlags } from '../post-action-observation-grammar.ts'; import { + enumField, + numberField, + requiredField, + stringArrayField, + stringField, toClientElementTarget, toClientInteractionTarget, toRepeatedOptions, toSelectorSnapshotOptions, } from '../command-input.ts'; -import { commonToClientOptions } from '../common-input-fields.ts'; +import { commonInputFromFlags } from '../cli-grammar/common.ts'; +import type { CliReader } from '../cli-grammar/types.ts'; +import { commonToClientOptions, type CommonCommandInput } from '../common-input-fields.ts'; +import { defineFieldCommandMetadata } from '../field-command-contract.ts'; import { defineCommandFacet, defineCommandFamilyFromFacets } from '../family/types.ts'; import { gestureCliReaders, gestureDaemonWriters } from './gesture.ts'; import { interactionCliReaders, interactionDaemonWriters } from './interactions.ts'; @@ -347,6 +364,153 @@ const gestureCommandFacet = defineCommandFacet({ daemonWriter: gestureDaemonWriters.gesture, }); +// --- policy-backed commands ----------------------------------------------------------------- +// `suggest` names the next element the way `find` does, and `act` is a loop over the verbs above, +// so both belong to this family's vocabulary. They are declared here rather than in their own +// module because the CLI surface is part of the closure every invocation evaluates at startup, +// and a new module there would grow it; the policy runtime they call lives under commands/policy +// and is loaded on demand by the client. + +const SUGGEST_COMMAND_NAME = 'suggest'; +const ACT_COMMAND_NAME = 'act'; + +const POLICY_PREAMBLE = `A policy head answers one question — which visible element advances the goal — as a typed decision with calibrated probabilities, in roughly the time a snapshot takes. It is optional: with no ${POLICY_API_KEY_ENV} in the environment this command refuses before touching the device, and nothing else changes.`; + +const suggestDescription = `Ask a policy head which element to act on next, and print the decision without acting. ${POLICY_PREAMBLE}`; + +const actDescription = `Run snapshot, decide, act until the policy reports the goal done, reports it blocked, or loses confidence. ${POLICY_PREAMBLE} Text is never generated. A field the policy chooses is filled from a caller-supplied entry keyed by that field's identifier or label, or from the matching environment entry when the value is a secret; a field with no entry escalates instead of being filled with a guess.`; + +const goalField = requiredField( + stringField('What the run should achieve, in one sentence, as the policy sees it.'), +); +const policyField = enumField( + POLICY_PROVIDER_NAMES, + `Policy head to ask. Default ${DEFAULT_POLICY_PROVIDER}.`, +); + +export const suggestCommandMetadata = defineFieldCommandMetadata( + SUGGEST_COMMAND_NAME, + suggestDescription, + { goal: goalField, policy: policyField }, +); + +export const actCommandMetadata = defineFieldCommandMetadata(ACT_COMMAND_NAME, actDescription, { + goal: goalField, + policy: policyField, + maxSteps: numberField(`Stop after this many steps. Default ${DEFAULT_POLICY_MAX_STEPS}.`, { + min: 1, + max: 100, + }), + minConfidence: numberField( + `Escalate instead of acting below this confidence. Default ${DEFAULT_POLICY_MIN_CONFIDENCE}.`, + { min: 0, max: 1 }, + ), + inputs: stringArrayField( + 'Text the loop may enter, as key=value. The key matches a field identifier or label.', + ), +}); + +const suggestCliSchema = { + usageOverride: 'suggest [--policy ]', + usageFlags: [], + listUsageOverride: 'suggest ', + positionalArgs: ['goal'], + allowedFlags: ['policy'], +} as const satisfies CommandSchemaOverride; + +const actCliSchema = { + usageOverride: + 'act [--policy ] [--max-steps ] [--min-confidence ] [--input ]', + usageFlags: [], + listUsageOverride: 'act ', + positionalArgs: ['goal'], + allowedFlags: ['policy', 'batchMaxSteps', 'minConfidence', 'policyInput'], +} as const satisfies CommandSchemaOverride; + +export const suggestCliReader: CliReader = (positionals, flags) => ({ + ...commonInputFromFlags(flags), + goal: readGoal(positionals[0], SUGGEST_COMMAND_NAME), + policy: flags.policy, +}); + +export const actCliReader: CliReader = (positionals, flags) => ({ + ...commonInputFromFlags(flags), + goal: readGoal(positionals[0], ACT_COMMAND_NAME), + policy: flags.policy, + maxSteps: flags.batchMaxSteps, + minConfidence: flags.minConfidence, + inputs: flags.policyInput, +}); + +type SuggestInput = CommonCommandInput & { goal: string; policy?: string }; +type ActInput = SuggestInput & { + maxSteps?: number; + minConfidence?: number; + inputs?: string[]; +}; + +const suggestCommandFacet = defineCommandFacet({ + name: SUGGEST_COMMAND_NAME, + text: { + summary: 'Ask a policy head for the next element to act on', + cliDetail: [ + `Prints the chosen ref, the probability it carries, and whether the policy considers the goal done or blocked. It never touches the device beyond one interactive snapshot, so it is safe to run repeatedly while checking a flow by hand.`, + '', + 'Examples:', + ` agent-device suggest "sign in with the test phone number"`, + ` agent-device suggest "open the first item" --json`, + ].join('\n'), + }, + metadata: suggestCommandMetadata, + run: (client, input) => client.policy.suggest(toSuggestOptions(input as SuggestInput)), + cliSchema: suggestCliSchema, + cliReader: suggestCliReader, + cliOutputFormatter: interactionCliOutputFormatters.suggest, +}); + +const actCommandFacet = defineCommandFacet({ + name: ACT_COMMAND_NAME, + text: { + summary: 'Drive a goal with a policy head until it is done or blocked', + cliDetail: [ + `Each step snapshots, asks the policy, acts, and re-snapshots. An action that leaves the screen digest unchanged is recorded as a dead action; three unproductive steps in a row end the run as escalated. Per-step snapshot, decide, and action milliseconds are in the result, as is the policy token cost. Supply text with --input key=value, or keep a secret out of argv by exporting ${POLICY_INPUT_ENV_PREFIX} instead.`, + '', + 'Examples:', + ` agent-device act "sign in and reach the main list" --input phone=5555550100 --max-steps 10`, + ` agent-device act "open the first item" --min-confidence 0.6 --json`, + ].join('\n'), + }, + metadata: actCommandMetadata, + run: (client, input) => client.policy.act(toActOptions(input as ActInput)), + cliSchema: actCliSchema, + cliReader: actCliReader, + cliOutputFormatter: interactionCliOutputFormatters.act, +}); + +function toSuggestOptions(input: SuggestInput) { + return { ...commonToClientOptions(input), goal: input.goal, policy: input.policy }; +} + +function toActOptions(input: ActInput) { + return { + ...toSuggestOptions(input), + maxSteps: input.maxSteps, + minConfidence: input.minConfidence, + inputs: input.inputs, + }; +} + +function readGoal(value: string | undefined, command: string): string { + const goal = value?.trim(); + if (!goal) { + throw new AppError( + 'INVALID_ARGS', + `${command} needs a goal, for example: ${command} "sign in"`, + ); + } + return goal; +} + export const interactionCommandFamily = defineCommandFamilyFromFacets({ name: 'interaction', clientSurface: false, @@ -364,6 +528,8 @@ export const interactionCommandFamily = defineCommandFamilyFromFacets({ isCommandFacet, findCommandFacet, gestureCommandFacet, + suggestCommandFacet, + actCommandFacet, ], }); diff --git a/src/commands/interaction/output.test.ts b/src/commands/interaction/output.test.ts index 1acbab2eef..168bc7bac2 100644 --- a/src/commands/interaction/output.test.ts +++ b/src/commands/interaction/output.test.ts @@ -309,3 +309,118 @@ describe('longpress CLI output', () => { ); }); }); + +describe('policy command output', () => { + const decision = { + action: 'fill' as const, + target: '@e7', + done: false, + blocked: true, + needsText: true, + confidence: 0.99, + probabilities: { '@e7': 0.99, '@e9': 0.006, __none__: 0.004 }, + provider: 'jev', + model: 'jev-1.13.0', + decideMs: 198, + inputTokens: 412, + costUsd: 0.000017304, + }; + + const formatSuggest = (result: unknown) => + interactionCliOutputFormatters.suggest({ input: {}, result } as never); + const formatAct = (result: unknown) => + interactionCliOutputFormatters.act({ input: {}, result } as never); + + test('suggest prints the decision, its top probabilities, timing, and cost', async () => { + const output = await formatSuggest({ + goal: 'sign in', + decision, + candidates: [], + snapshotMs: 412, + screen: 'Phone number | Send code', + }); + expect(output.text).toBe( + [ + 'Goal: sign in', + 'Screen: Phone number | Send code', + 'Next: fill @e7 at confidence 0.99 (blocked)', + 'Probabilities: @e7 99.0%, @e9 0.6%, __none__ 0.4%', + 'Timing: snapshot 412ms decide 198ms', + 'Cost: $0.000017 for 412 input tokens', + ].join('\n'), + ); + }); + + test('suggest names both flags when the policy calls the goal done and blocked', async () => { + const output = await formatSuggest({ + goal: 'sign in', + decision: { ...decision, done: true, blocked: true }, + candidates: [], + snapshotMs: 1, + screen: 'Shelf', + }); + expect(output.text).toContain('(done blocked)'); + }); + + test('suggest reports a decline as no target', async () => { + const output = await formatSuggest({ + goal: 'sign in', + decision: { ...decision, action: 'none', target: null, blocked: false }, + candidates: [], + snapshotMs: 1, + screen: 'Shelf', + }); + expect(output.text).toContain('Next: none none at confidence 0.99'); + }); + + test('act prints one line per step, the reason for a step that stalled, and the totals', async () => { + const output = await formatAct({ + goal: 'sign in', + status: 'escalated', + steps: [ + { + step: 1, + screen: 'Phone number', + decision, + outcome: 'acted', + performed: { action: 'fill', target: '@e7', entry: 'keypad', inputKey: 'phone' }, + snapshotMs: 400, + decideMs: 198, + actionMs: 900, + }, + { + step: 2, + screen: 'Phone number', + decision, + outcome: 'escalated', + snapshotMs: 400, + decideMs: 198, + actionMs: 0, + reason: 'policy chose no element on this screen', + }, + ], + totals: { + steps: 2, + snapshotMs: 800, + decideMs: 396, + actionMs: 900, + inputTokens: 824, + costUsd: 0.000034608, + deadActions: 0, + escalations: 1, + }, + }); + expect(output.text).toBe( + [ + 'Goal: sign in', + 'Status: escalated', + ' 1 acted fill @e7 via keypad conf 0.99 | Phone number', + ' 2 escalated conf 0.99 | Phone number', + ' policy chose no element on this screen', + 'Steps: 2, dead actions 0, escalations 1', + 'Timing: snapshot 800ms decide 396ms action 900ms', + 'Cost: $0.000035 for 824 input tokens', + ].join('\n'), + ); + }); +}); diff --git a/src/commands/interaction/output.ts b/src/commands/interaction/output.ts index 5bb19b46c8..43cf0ba66e 100644 --- a/src/commands/interaction/output.ts +++ b/src/commands/interaction/output.ts @@ -1,6 +1,12 @@ import type { CommandRequestResult } from '@agent-device/contracts/client'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; import type { CliOutput } from '../command-contract.ts'; +// Type-only, so the startup closure never reaches the policy runtime through this module. +import type { + PolicyActResult, + PolicyDecision, + PolicySuggestResult, +} from '../policy/policy-contract.ts'; import { displayLabel, formatRole } from '@agent-device/capture-kit/snapshot-lines'; import { readCommandMessage } from '@agent-device/kernel/success-text'; import { @@ -82,22 +88,83 @@ function tapCliOutput(result: CommandRequestResult): CliOutput { // #1652: settle-capable entries (click, press, fill, longpress, hover, scroll) // get the warning/settle notes appended by the trait-derived wrapper; the rest // of the map is returned untouched. -export const interactionCliOutputFormatters = withSettleCapableNotes({ - click: resultOutput(tapCliOutput), - press: resultOutput(tapCliOutput), - fill: messageOutput, - longpress: messageOutput, - hover: messageOutput, - scroll: messageOutput, - get: ({ input, result }) => - getCliOutput({ - result: result as CommandRequestResult, - format: input.format as Parameters[0]['format'], - }), - is: resultOutput(isCliOutput), - find: resultOutput(findCliOutput), -} satisfies Record); +// The policy commands sit outside the settle-note wrapper: their results carry no settle diff. +export const interactionCliOutputFormatters = { + ...withSettleCapableNotes({ + click: resultOutput(tapCliOutput), + press: resultOutput(tapCliOutput), + fill: messageOutput, + longpress: messageOutput, + hover: messageOutput, + scroll: messageOutput, + get: ({ input, result }) => + getCliOutput({ + result: result as CommandRequestResult, + format: input.format as Parameters[0]['format'], + }), + is: resultOutput(isCliOutput), + find: resultOutput(findCliOutput), + } satisfies Record), + suggest: resultOutput(suggestCliOutput), + act: resultOutput(actCliOutput), +} satisfies Record; function defaultCommandCliOutput(result: CommandRequestResult): CliOutput { return messageCliOutput(result as Record); } + +function suggestCliOutput(result: PolicySuggestResult): CliOutput { + const lines = [ + `Goal: ${result.goal}`, + `Screen: ${result.screen}`, + decisionLine(result.decision), + topProbabilities(result.decision), + `Timing: snapshot ${ms(result.snapshotMs)} decide ${ms(result.decision.decideMs)}`, + `Cost: ${usd(result.decision.costUsd)} for ${result.decision.inputTokens} input tokens`, + ]; + return { data: result, text: lines.filter(Boolean).join('\n') }; +} + +function actCliOutput(result: PolicyActResult): CliOutput { + const lines = [`Goal: ${result.goal}`, `Status: ${result.status}`]; + for (const step of result.steps) { + const performed = step.performed + ? ` ${step.performed.action} ${step.performed.target}${step.performed.entry === 'keypad' ? ' via keypad' : ''}` + : ''; + lines.push( + `${String(step.step).padStart(2)} ${step.outcome.padEnd(12)}${performed} conf ${step.decision.confidence.toFixed(2)} | ${step.screen}`, + ); + if (step.reason) lines.push(` ${step.reason}`); + } + const totals = result.totals; + lines.push( + `Steps: ${totals.steps}, dead actions ${totals.deadActions}, escalations ${totals.escalations}`, + `Timing: snapshot ${ms(totals.snapshotMs)} decide ${ms(totals.decideMs)} action ${ms(totals.actionMs)}`, + `Cost: ${usd(totals.costUsd)} for ${totals.inputTokens} input tokens`, + ); + return { data: result, text: lines.join('\n') }; +} + +function decisionLine(decision: PolicyDecision): string { + const target = decision.target ?? 'none'; + const flags = [decision.done ? 'done' : '', decision.blocked ? 'blocked' : ''] + .filter(Boolean) + .join(' '); + return `Next: ${decision.action} ${target} at confidence ${decision.confidence.toFixed(2)}${flags ? ` (${flags})` : ''}`; +} + +function topProbabilities(decision: PolicyDecision): string { + const ranked = Object.entries(decision.probabilities) + .sort(([, a], [, b]) => b - a) + .slice(0, 3) + .map(([label, probability]) => `${label} ${(probability * 100).toFixed(1)}%`); + return ranked.length > 0 ? `Probabilities: ${ranked.join(', ')}` : ''; +} + +function ms(value: number): string { + return `${Math.round(value)}ms`; +} + +function usd(value: number): string { + return `$${value.toFixed(6)}`; +} diff --git a/src/commands/interaction/policy-commands.test.ts b/src/commands/interaction/policy-commands.test.ts new file mode 100644 index 0000000000..d5382cdbc6 --- /dev/null +++ b/src/commands/interaction/policy-commands.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, test } from 'vitest'; +import type { CliFlags } from '@agent-device/contracts/command'; +import { + actCliReader, + actCommandMetadata, + suggestCliReader, + suggestCommandMetadata, +} from './index.ts'; +import { createPolicyProvider } from '../policy/policy-provider.ts'; +import { listCliCommandNames } from '@agent-device/command-registry/catalog'; +import { commandDescriptors } from '@agent-device/command-registry/registry'; +import { listMcpCommandMetadata } from '../command-metadata.ts'; +import { POLICY_API_KEY_ENV } from '@agent-device/command-registry/flag-definitions-workflow'; + +describe('policy command interface', () => { + test('owns suggest and act public metadata', () => { + expect(suggestCommandMetadata.name).toBe('suggest'); + expect(actCommandMetadata.name).toBe('act'); + }); + + test('reads a goal positional and the policy flag', () => { + expect(suggestCliReader(['sign in'], { policy: 'jev' } as CliFlags)).toMatchObject({ + goal: 'sign in', + policy: 'jev', + }); + }); + + test('reads the loop budget, confidence floor, and input map', () => { + expect( + actCliReader(['sign in'], { + batchMaxSteps: 8, + minConfidence: 0.6, + policyInput: ['phone=5555550100'], + } as unknown as CliFlags), + ).toMatchObject({ + goal: 'sign in', + maxSteps: 8, + minConfidence: 0.6, + inputs: ['phone=5555550100'], + }); + }); + + test('refuses an empty goal', () => { + for (const read of [suggestCliReader, actCliReader]) { + expect(() => read([' '], {} as CliFlags)).toThrow( + expect.objectContaining({ + code: 'INVALID_ARGS', + message: expect.stringContaining('needs a goal'), + }), + ); + } + }); +}); + +describe('policy provider resolution', () => { + test('refuses to run without a key, naming the environment entry and the fallback', () => { + expect(() => createPolicyProvider('jev', {})).toThrow( + expect.objectContaining({ + code: 'INVALID_ARGS', + message: expect.stringContaining('TYPESAFE_API_KEY is not set'), + }), + ); + }); + + test('refuses an unknown provider name', () => { + expect(() => createPolicyProvider('oracle', { TYPESAFE_API_KEY: 'k' })).toThrow( + expect.objectContaining({ message: expect.stringContaining('Unknown policy provider') }), + ); + }); + + test('builds the jev provider when the key is present', () => { + expect(createPolicyProvider('jev', { TYPESAFE_API_KEY: 'k' }).name).toBe('jev'); + }); +}); + +/** + * The surface is identical with and without a credential: the commands are always listed and + * always offered over MCP, and refuse at run time. Nothing here reads the environment, so the + * assertions run the projections twice with the key set and unset and compare. + */ +describe('the command surface does not depend on the key', () => { + const POLICY_COMMANDS = ['suggest', 'act'] as const; + + function withKey(value: string | undefined, read: () => T): T { + const previous = process.env[POLICY_API_KEY_ENV]; + if (value === undefined) delete process.env[POLICY_API_KEY_ENV]; + else process.env[POLICY_API_KEY_ENV] = value; + try { + return read(); + } finally { + if (previous === undefined) delete process.env[POLICY_API_KEY_ENV]; + else process.env[POLICY_API_KEY_ENV] = previous; + } + } + + test('both commands are in the CLI catalog whether or not the key is set', () => { + const withoutKey = withKey(undefined, listCliCommandNames); + const withKeySet = withKey('a-key', listCliCommandNames); + expect(withoutKey).toEqual(withKeySet); + for (const command of POLICY_COMMANDS) expect(withoutKey).toContain(command); + }); + + test('both commands are offered over MCP whether or not the key is set', () => { + const toolNames = () => listMcpCommandMetadata().map((tool) => tool.name); + expect(withKey(undefined, toolNames)).toEqual(withKey('a-key', toolNames)); + for (const command of POLICY_COMMANDS) { + expect(withKey(undefined, toolNames)).toContain(command); + } + }); + + test('each MCP tool takes a goal and never takes the credential', () => { + for (const command of POLICY_COMMANDS) { + const tool = withKey(undefined, () => + listMcpCommandMetadata().find((entry) => entry.name === command), + ); + const schema = tool?.inputSchema; + expect(Object.keys(schema?.properties ?? {}), `${command} must accept a goal`).toContain( + 'goal', + ); + // The credential is environment-only: the description may name the variable so a model + // knows what to set, but no input may ever carry its value. + expect(JSON.stringify(schema)).not.toContain(POLICY_API_KEY_ENV); + expect(JSON.stringify(schema).toLowerCase()).not.toContain('apikey'); + expect(tool?.description).toContain(POLICY_API_KEY_ENV); + } + }); + + test('neither command owns a daemon route', () => { + for (const command of POLICY_COMMANDS) { + const descriptor = commandDescriptors.find((entry) => entry.name === command); + expect(descriptor, `${command} is missing from the registry`).toBeDefined(); + expect(descriptor).not.toHaveProperty('daemon'); + } + }); + + test('no other command gained a policy field', () => { + const others = commandDescriptors.filter( + (descriptor) => !(POLICY_COMMANDS as readonly string[]).includes(descriptor.name), + ); + for (const descriptor of others) { + expect(JSON.stringify(descriptor), `${descriptor.name} changed`).not.toContain('policy'); + } + }); +}); diff --git a/src/commands/policy/act-loop.test.ts b/src/commands/policy/act-loop.test.ts new file mode 100644 index 0000000000..49a5d27e9c --- /dev/null +++ b/src/commands/policy/act-loop.test.ts @@ -0,0 +1,480 @@ +import { describe, expect, test } from 'vitest'; +import { runPolicyActLoop, type PolicyDevicePort } from './act-loop.ts'; +import type { PolicySnapshotNode } from './candidate-elements.ts'; +import type { PolicyDecision, PolicyProvider, PolicyRequest } from './policy-contract.ts'; + +function decision(overrides: Partial = {}): PolicyDecision { + return { + action: 'press', + target: '@e6', + done: false, + blocked: false, + needsText: false, + confidence: 0.95, + probabilities: { '@e6': 0.95 }, + provider: 'stub', + model: 'stub-1', + decideMs: 200, + inputTokens: 400, + costUsd: 0.0000168, + ...overrides, + }; +} + +function providerOf(...decisions: PolicyDecision[]): PolicyProvider & { seen: PolicyRequest[] } { + const seen: PolicyRequest[] = []; + let index = 0; + return { + name: 'stub', + model: 'stub-1', + seen, + decide: async (request) => { + seen.push(request); + return decisions[Math.min(index++, decisions.length - 1)] as PolicyDecision; + }, + }; +} + +function screenOf(...nodes: PolicySnapshotNode[]): PolicySnapshotNode[] { + return nodes; +} + +const welcome = screenOf( + { index: 0, type: 'StaticText', label: 'Welcome', ref: 'e2' }, + { index: 1, type: 'Button', label: 'Send code', ref: 'e6' }, +); +const codeScreen = screenOf( + { index: 0, type: 'StaticText', label: 'Enter the code', ref: 'e2' }, + { index: 1, type: 'TextField', label: 'Code', identifier: 'codeField', value: '', ref: 'e7' }, +); + +/** A device that returns each queued screen in turn and records every call. */ +function deviceOf(screens: PolicySnapshotNode[][]): PolicyDevicePort & { calls: string[] } { + const calls: string[] = []; + let index = 0; + return { + calls, + snapshot: async () => { + const nodes = screens[Math.min(index++, screens.length - 1)] as PolicySnapshotNode[]; + return { nodes, ms: 400 }; + }, + press: async (ref) => { + calls.push(`press ${ref}`); + return 900; + }, + fill: async (ref, text) => { + calls.push(`fill ${ref} ${text}`); + return 700; + }, + }; +} + +const baseOptions = { + goal: 'sign in', + inputs: new Map(), + env: {} as Record, + maxSteps: 6, + minConfidence: 0.4, + sleep: async () => {}, +}; + +describe('policy act loop', () => { + test('stops as done when the policy reports the goal achieved', async () => { + const result = await runPolicyActLoop({ + ...baseOptions, + provider: providerOf(decision({ done: true })), + device: deviceOf([welcome]), + }); + expect(result.status).toBe('done'); + expect(result.steps).toHaveLength(1); + expect(result.steps[0]?.outcome).toBe('done'); + }); + + test('presses the chosen element and records the phase timings apart', async () => { + const device = deviceOf([welcome, codeScreen]); + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + provider: providerOf(decision()), + device, + }); + expect(device.calls).toEqual(['press @e6']); + expect(result.steps[0]).toMatchObject({ + outcome: 'acted', + snapshotMs: 400, + decideMs: 200, + actionMs: 900, + performed: { action: 'press', target: '@e6' }, + }); + expect(result.totals.costUsd).toBeCloseTo(0.0000168, 9); + }); + + test('re-reads once before filing an action as dead, so a slow transition still counts', async () => { + // Two unchanged reads, then the transition lands: immediate read stale, grace read fresh. + const device = deviceOf([welcome, welcome, codeScreen]); + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + provider: providerOf(decision()), + device, + }); + expect(result.steps[0]?.outcome).toBe('acted'); + }); + + test('records an action that left the screen unchanged as a dead action', async () => { + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + provider: providerOf(decision()), + device: deviceOf([welcome]), + }); + expect(result.steps[0]?.outcome).toBe('dead-action'); + expect(result.steps[0]?.reason).toBe('screen unchanged after press'); + expect(result.totals.deadActions).toBe(1); + }); + + test('ends as escalated after three unproductive steps in a row', async () => { + const result = await runPolicyActLoop({ + ...baseOptions, + provider: providerOf(decision()), + device: deviceOf([welcome]), + }); + expect(result.status).toBe('escalated'); + expect(result.steps).toHaveLength(3); + }); + + test('escalates rather than acting below the confidence floor', async () => { + const result = await runPolicyActLoop({ + ...baseOptions, + minConfidence: 0.8, + provider: providerOf(decision({ confidence: 0.3 })), + device: deviceOf([welcome]), + }); + expect(result.status).toBe('escalated'); + expect(result.steps[0]?.reason).toContain('below --min-confidence'); + }); + + test('acts on a named target even when the policy flags the screen blocked', async () => { + const acting = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + provider: providerOf(decision({ blocked: true, confidence: 0.52 })), + device: deviceOf([welcome, codeScreen]), + }); + expect(acting.steps[0]?.outcome).toBe('acted'); + }); + + test('stops as blocked when the policy flags the screen and names no way on', async () => { + const halted = await runPolicyActLoop({ + ...baseOptions, + provider: providerOf(decision({ blocked: true, target: null, action: 'none' })), + device: deviceOf([welcome]), + }); + expect(halted.status).toBe('blocked'); + expect(halted.steps[0]?.reason).toBe('policy reports progress blocked and names no way on'); + }); + + test('stops as blocked when the flag comes with a target below the floor', async () => { + const halted = await runPolicyActLoop({ + ...baseOptions, + minConfidence: 0.6, + provider: providerOf(decision({ blocked: true, confidence: 0.2 })), + device: deviceOf([welcome]), + }); + expect(halted.status).toBe('blocked'); + }); + + test('never invents text: a field with no supplied value escalates', async () => { + const device = deviceOf([codeScreen]); + const result = await runPolicyActLoop({ + ...baseOptions, + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device, + }); + expect(device.calls).toEqual([]); + expect(result.status).toBe('escalated'); + expect(result.steps[0]?.reason).toBe('no text supplied for codeField'); + }); + + test('fills from an input entry matched to the field identifier', async () => { + const device = deviceOf([codeScreen, welcome]); + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', 'abcdef']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device, + }); + expect(device.calls).toEqual(['fill @e7 abcdef']); + expect(result.steps[0]?.performed).toEqual({ + action: 'fill', + target: '@e7', + entry: 'fill', + inputKey: 'code', + }); + }); + + test('records the input key but never the text it sent', async () => { + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', '482493']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device: deviceOf([codeScreen, welcome]), + }); + expect(JSON.stringify(result)).not.toContain('482493'); + }); + + test('enters digits on the keypad when fill leaves the field value unchanged', async () => { + const keypad: PolicySnapshotNode[] = [ + ...codeScreen, + { index: 9, type: 'Key', label: '4', ref: 'e30' }, + { index: 10, type: 'Key', label: '8', ref: 'e31' }, + ]; + const device = deviceOf([keypad]); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', '48']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device, + }); + expect(device.calls).toEqual(['fill @e7 48', 'press @e30', 'press @e31']); + }); + + test('keeps a fill that did land off the keypad path', async () => { + const landed: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: 'Enter the code', ref: 'e2' }, + { + index: 1, + type: 'TextField', + label: 'Code', + identifier: 'codeField', + value: '482493', + ref: 'e7', + }, + { index: 9, type: 'Key', label: '4', ref: 'e30' }, + ]; + const device = deviceOf([codeScreen, landed]); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', '482493']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device, + }); + expect(device.calls).toEqual(['fill @e7 482493']); + }); + + test('rewrites a field whose value is not the supplied text', async () => { + const prefilled: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: 'Welcome', ref: 'e2' }, + { + index: 1, + type: 'TextField', + label: 'Phone number', + identifier: 'phoneField', + value: '(555) 555-0100', + ref: 'e7', + }, + ]; + const device = deviceOf([prefilled, welcome]); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['phone', '5555550700']]), + provider: providerOf(decision({ action: 'press', target: '@e7' })), + device, + }); + expect(device.calls).toEqual(['fill @e7 5555550700']); + }); + + test('presses rather than rewrites a field that already holds the supplied text', async () => { + const satisfied: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: 'Welcome', ref: 'e2' }, + { + index: 1, + type: 'TextField', + label: 'Phone number', + identifier: 'phoneField', + value: '(555) 555-0700', + ref: 'e7', + }, + ]; + const device = deviceOf([satisfied, welcome]); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['phone', '5555550700']]), + provider: providerOf(decision({ action: 'press', target: '@e7' })), + device, + }); + expect(device.calls).toEqual(['press @e7']); + }); + + test('presses a field that already holds text when the caller supplied none', async () => { + const prefilled: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: 'Welcome', ref: 'e2' }, + { + index: 1, + type: 'TextField', + label: 'Phone number', + identifier: 'phoneField', + value: '(555) 555-0100', + ref: 'e7', + }, + ]; + const device = deviceOf([prefilled, welcome]); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + provider: providerOf(decision({ action: 'press', target: '@e7' })), + device, + }); + expect(device.calls).toEqual(['press @e7']); + }); + + test('reaches a digit-box widget through the keypad even when the write itself failed', async () => { + const boxes: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: '0 of 2 entered', ref: 'e2' }, + { index: 1, type: 'TextField', label: 'Code', identifier: 'codeField', value: '', ref: 'e7' }, + { index: 9, type: 'Key', label: '4', ref: 'e30' }, + { index: 10, type: 'Key', label: '8', ref: 'e31' }, + ]; + const device = deviceOf([boxes]); + const failing: PolicyDevicePort = { + ...device, + fill: async () => { + throw Object.assign(new Error('runner recorded a failure'), { + code: 'XCTEST_RECORDED_FAILURE', + }); + }, + }; + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', '48']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device: failing, + }); + expect(device.calls).toEqual(['press @e30', 'press @e31']); + expect(result.steps[0]?.performed?.entry).toBe('keypad'); + expect(result.steps[0]?.outcome).toBe('acted'); + }); + + test('accepts a hidden-value widget when the screen moved on after the write', async () => { + // A digit-box code field never reflects its value; the next screen is the only evidence. + const device = deviceOf([codeScreen, codeScreen, welcome]); + const failing: PolicyDevicePort = { + ...device, + fill: async () => { + throw Object.assign(new Error('recorded failure'), { code: 'XCTEST_RECORDED_FAILURE' }); + }, + }; + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', 'abcdef']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device: failing, + }); + expect(result.steps[0]?.outcome).toBe('acted'); + }); + + test('carries the write failure code when neither the write nor the keypad delivered', async () => { + const device = deviceOf([codeScreen]); + const failing: PolicyDevicePort = { + ...device, + fill: async () => { + throw Object.assign(new Error('mismatch'), { code: 'TEXT_ENTRY_MISMATCH' }); + }, + }; + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', '48']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device: failing, + }); + expect(result.steps[0]?.outcome).toBe('dead-action'); + expect(result.steps[0]?.reason).toContain('TEXT_ENTRY_MISMATCH'); + }); + + test('accepts a masked field that reformatted the supplied text', async () => { + const masked: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: 'Welcome', ref: 'e2' }, + { + index: 1, + type: 'TextField', + label: 'Phone number', + identifier: 'phoneField', + value: '(500) 555-0700', + ref: 'e7', + }, + ]; + const unformatted: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: 'Welcome', ref: 'e2' }, + { + index: 1, + type: 'TextField', + label: 'Phone number', + identifier: 'phoneField', + value: '', + ref: 'e7', + }, + ]; + const device = deviceOf([unformatted, masked]); + const failing: PolicyDevicePort = { + ...device, + fill: async (ref, text) => { + device.calls.push(`fill ${ref} ${text}`); + throw Object.assign(new Error('mismatch'), { code: 'TEXT_ENTRY_MISMATCH' }); + }, + }; + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['phone', '5005550700']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device: failing, + }); + expect(device.calls).toEqual(['fill @e7 5005550700']); + expect(result.steps[0]?.outcome).toBe('acted'); + expect(result.steps[0]?.performed?.entry).toBe('fill'); + }); + + test('tells the policy which fields it already holds text for', async () => { + const provider = providerOf(decision({ target: '@e7', action: 'fill' })); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', 'abc']]), + provider, + device: deviceOf([codeScreen, welcome]), + }); + expect(provider.seen[0]?.textReadyRefs).toEqual(['@e7']); + }); + + test('gives the policy the history of what it already did', async () => { + const provider = providerOf(decision()); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 2, + provider, + device: deviceOf([welcome, codeScreen, welcome, codeScreen]), + }); + expect(provider.seen[0]?.history).toEqual([]); + expect(provider.seen[1]?.history).toEqual(['pressed Send code']); + }); + + test('stops at the step budget', async () => { + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 2, + provider: providerOf(decision()), + device: deviceOf([welcome, codeScreen, welcome, codeScreen]), + }); + expect(result.status).toBe('max-steps'); + expect(result.totals.steps).toBe(2); + }); +}); diff --git a/src/commands/policy/act-loop.ts b/src/commands/policy/act-loop.ts new file mode 100644 index 0000000000..4bcf200f07 --- /dev/null +++ b/src/commands/policy/act-loop.ts @@ -0,0 +1,403 @@ +import { + findKeypadDigit, + isTextEntryRole, + screenDigest, + screenLabel, + toPolicyCandidates, + type PolicySnapshotNode, +} from './candidate-elements.ts'; +import type { + PolicyActResult, + PolicyActStep, + PolicyActStatus, + PolicyCandidate, + PolicyDecision, + PolicyProvider, + PolicyStepOutcome, +} from './policy-contract.ts'; +import { resolveTextInput, type PolicyTextInputs, type ResolvedTextInput } from './text-inputs.ts'; + +/** The three device operations the loop needs, named so a test can supply them directly. */ +export type PolicyDevicePort = { + snapshot: () => Promise<{ nodes: PolicySnapshotNode[]; ms: number }>; + press: (ref: string) => Promise; + fill: (ref: string, text: string) => Promise; +}; + +export type PolicyActOptions = { + goal: string; + provider: PolicyProvider; + device: PolicyDevicePort; + inputs: PolicyTextInputs; + env: Readonly>; + maxSteps: number; + minConfidence: number; + /** Called once per completed step so a CLI can stream progress. */ + onStep?: (step: PolicyActStep) => void; + /** Injected so a test never waits out the grace re-read. */ + sleep?: (ms: number) => Promise; +}; + +/** Consecutive unproductive steps tolerated before the loop hands back. */ +const ESCALATION_LIMIT = 3; + +/** How long to wait before re-reading a screen that has not changed yet. */ +const TRANSITION_GRACE_MS = 800; + +/** The per-step inputs a step decision and action both read. */ +type StepContext = { + decision: PolicyDecision; + candidates: PolicyCandidate[]; + digestBefore: string; + history: string[]; +}; + +/** What one step concluded, before the loop turns it into a recorded step. */ +type StepResult = { outcome: PolicyStepOutcome; extra?: Partial }; + +export async function runPolicyActLoop(options: PolicyActOptions): Promise { + const steps: PolicyActStep[] = []; + const history: string[] = []; + let status: PolicyActStatus = 'max-steps'; + let unproductive = 0; + + for (let index = 0; index < options.maxSteps; index++) { + const before = await options.device.snapshot(); + const candidates = toPolicyCandidates(before.nodes); + const screen = screenLabel(candidates); + const context: StepContext = { + decision: await options.provider.decide({ + goal: options.goal, + candidates, + // Copied, not shared: a provider that holds the request must not observe later steps. + history: [...history], + screen, + textReadyRefs: textReadyRefs(candidates, options.inputs, options.env), + }), + candidates, + digestBefore: screenDigest(candidates), + history, + }; + + const result = await resolveStep(options, context); + const step: PolicyActStep = { + step: index + 1, + screen, + decision: context.decision, + outcome: result.outcome, + snapshotMs: before.ms, + decideMs: context.decision.decideMs, + actionMs: 0, + ...result.extra, + }; + steps.push(step); + options.onStep?.(step); + + unproductive = result.outcome === 'acted' ? 0 : unproductive + 1; + const ended = endStatus(result.outcome, unproductive); + if (ended) { + status = ended; + break; + } + } + + return { goal: options.goal, status, steps, totals: totalsFor(steps) }; +} + +/** The status a run ends with after this outcome, or undefined to keep going. */ +function endStatus(outcome: PolicyStepOutcome, unproductive: number): PolicyActStatus | undefined { + if (outcome === 'done') return 'done'; + if (outcome === 'blocked') return 'blocked'; + return unproductive >= ESCALATION_LIMIT ? 'escalated' : undefined; +} + +/** Judge the decision, and act on it when it is actionable. */ +async function resolveStep(options: PolicyActOptions, context: StepContext): Promise { + if (context.decision.done) return { outcome: 'done' }; + + const judgment = judge(context.decision, options.minConfidence); + if (judgment.kind === 'halt') { + return { outcome: judgment.outcome, extra: { reason: judgment.reason } }; + } + + const target = judgment.target; + const candidate = context.candidates.find((entry) => entry.ref === target); + const textEntry = resolveTextEntry(candidate, options.inputs, options.env); + + if (textEntry.kind === 'unsupplied') { + return { + outcome: 'escalated', + extra: { + reason: `no text supplied for ${candidate?.identifier ?? candidate?.name ?? target}`, + }, + }; + } + if (textEntry.kind === 'write') { + return await writeStep(options, context, target, candidate, textEntry.input); + } + return await pressStep(options, context, target, candidate); +} + +async function writeStep( + options: PolicyActOptions, + context: StepContext, + target: string, + candidate: PolicyCandidate | undefined, + input: ResolvedTextInput, +): Promise { + const name = candidate?.name ?? target; + const filled = await enterText(options.device, target, input.text, candidate); + // A digit-box widget never reflects its value, so a confirmed value is sufficient evidence but + // not necessary: a screen that moved on is the other half of it. + const landed = filled.valueConfirmed || (await screenChanged(options, context.digestBefore)); + context.history.push( + landed ? `filled ${name} from input ${input.key}` : `could not write into ${name}`, + ); + const reason = `the field did not take the supplied text (${filled.failure ?? 'value unchanged'})`; + return { + outcome: landed ? 'acted' : 'dead-action', + extra: { + actionMs: filled.ms, + performed: { action: 'fill', target, entry: filled.entry, inputKey: input.key }, + ...(landed ? {} : { reason }), + }, + }; +} + +async function pressStep( + options: PolicyActOptions, + context: StepContext, + target: string, + candidate: PolicyCandidate | undefined, +): Promise { + const actionMs = await options.device.press(target); + context.history.push(`pressed ${candidate?.name ?? target}`); + const changed = await screenChanged(options, context.digestBefore); + return { + outcome: changed ? 'acted' : 'dead-action', + extra: { + actionMs, + performed: { action: 'press', target }, + ...(changed ? {} : { reason: 'screen unchanged after press' }), + }, + }; +} + +/** Refs of text fields the caller has text for that is not already in them. */ +function textReadyRefs( + candidates: readonly PolicyCandidate[], + inputs: PolicyTextInputs, + env: Readonly>, +): string[] { + return candidates + .filter((candidate) => resolveTextEntry(candidate, inputs, env).kind === 'write') + .map((candidate) => candidate.ref); +} + +/** + * Whether the screen moved on from the digest taken before the action. + * + * A settled observation only proves the local UI went quiet. A transition waiting on the network — + * request a code, verify a code — lands after that, so an immediate read still shows the old + * screen and a real action would be filed as dead. One grace re-read separates a slow transition + * from an action that genuinely did nothing. + */ +async function screenChanged( + options: Pick, + digestBefore: string, +): Promise { + const digestOf = (nodes: PolicySnapshotNode[]): string => screenDigest(toPolicyCandidates(nodes)); + const immediate = await options.device.snapshot(); + if (digestOf(immediate.nodes) !== digestBefore) return true; + await (options.sleep ?? defaultSleep)(TRANSITION_GRACE_MS); + const settled = await options.device.snapshot(); + return digestOf(settled.nodes) !== digestBefore; +} + +async function defaultSleep(ms: number): Promise { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Decide whether the chosen element needs text written into it. + * + * `press` on a text field only focuses it, so the question is not whether the field is empty but + * whether the caller named text for it that is not already there. A field carrying a development + * default the caller overrode has to be rewritten, and a field already holding the supplied value + * must not be rewritten, or the loop would spend its whole budget refilling one field. + */ +function resolveTextEntry( + candidate: PolicyCandidate | undefined, + inputs: PolicyTextInputs, + env: Readonly>, +): { kind: 'press' } | { kind: 'write'; input: ResolvedTextInput } | { kind: 'unsupplied' } { + if (!candidate || !isTextEntryRole(candidate.role)) return { kind: 'press' }; + const input = resolveTextInput(candidate, inputs, env); + if (!input) return candidate.value ? { kind: 'press' } : { kind: 'unsupplied' }; + return sameText(candidate.value, input.text) ? { kind: 'press' } : { kind: 'write', input }; +} + +/** Compares a field value with supplied text ignoring the formatting a field applies itself. */ +function sameText(value: string | undefined, text: string): boolean { + const normalize = (input: string): string => input.replaceAll(/[^a-z0-9]/giu, '').toLowerCase(); + if (value === undefined) return false; + const current = normalize(value); + const wanted = normalize(text); + return current === wanted || (wanted.length > 0 && current.endsWith(wanted)); +} + +/** + * Whether to stop instead of acting. + * + * `blocked` answers a question about the screen, not about the goal, and a policy asked it says yes + * on anything that looks like a wall: an unauthenticated sign-in screen, or an onboarding page + * offering to import a file. Twice in live runs it was set while the policy also named the one + * element that led onward. So `blocked` is terminal only together with nothing worth acting on — + * no target, or one below the confidence floor. A named target above the floor is acted on, because + * the cost of being wrong is one step the same-screen check catches and the step budget bounds, + * while the cost of believing the flag is the whole run. + */ +function judge( + decision: PolicyDecision, + minConfidence: number, +): + | { kind: 'act'; target: string } + | { kind: 'halt'; outcome: 'blocked' | 'escalated'; reason: string } { + if (decision.target !== null && decision.confidence >= minConfidence) { + return { kind: 'act', target: decision.target }; + } + if (decision.blocked) { + return { + kind: 'halt', + outcome: 'blocked', + reason: 'policy reports progress blocked and names no way on', + }; + } + if (decision.target === null) { + return { + kind: 'halt', + outcome: 'escalated', + reason: 'policy chose no element on this screen', + }; + } + return { + kind: 'halt', + outcome: 'escalated', + reason: `confidence ${decision.confidence.toFixed(2)} below --min-confidence ${minConfidence}`, + }; +} + +/** + * Write text into a field, then verify from the screen that it arrived. + * + * Two device realities make the write itself an unreliable signal. + * + * A field with an input mask reformats what it receives, so the runner's own text-entry + * verification reports `TEXT_ENTRY_MISMATCH` comparing "5005550700" against "(500) 555-0700". The + * text did arrive. Keyed on that typed code, the loop re-reads the field and decides from the value + * rather than from the comparison. + * + * A code or PIN field is often a row of single-character boxes over a hidden input: the write + * succeeds, the accessibility value never changes, and the submit button stays disabled. When the + * text is all digits, the on-screen keypad is the only path that registers them, one key press at a + * time, each needing a fresh snapshot because refs are reissued after every press. + */ +async function enterText( + device: PolicyDevicePort, + ref: string, + text: string, + candidate: PolicyCandidate | undefined, +): Promise<{ ms: number; entry: 'fill' | 'keypad'; valueConfirmed: boolean; failure?: string }> { + const startedAt = Date.now(); + const failure = await writeFailure(device, ref, text); + const filledMs = Date.now() - startedAt; + + // The write's own verdict is not the evidence; the field's value is. Both realities above report + // failure on a write that arrived, and one reports success on a write that did not. + const verification = await device.snapshot(); + if (valueLanded(verification.nodes, candidate, text)) { + return { ms: filledMs, entry: 'fill', valueConfirmed: true }; + } + + const digits = text.replaceAll(/\D/gu, ''); + if (digits.length !== text.length || digits.length === 0) { + return { ms: filledMs, entry: 'fill', valueConfirmed: false, ...(failure ? { failure } : {}) }; + } + + const keypad = await enterDigitsOnKeypad(device, digits); + return { + ms: filledMs + keypad.ms, + entry: keypad.entered > 0 ? 'keypad' : 'fill', + valueConfirmed: keypad.entered === digits.length, + ...(failure ? { failure } : {}), + }; +} + +/** Tap each digit on the on-screen keypad, re-reading because refs are reissued per press. */ +async function enterDigitsOnKeypad( + device: PolicyDevicePort, + digits: string, +): Promise<{ ms: number; entered: number }> { + let ms = 0; + let entered = 0; + for (const digit of digits) { + const snapshot = await device.snapshot(); + const key = findKeypadDigit(snapshot.nodes, digit); + if (!key?.ref) break; + ms += await device.press(key.ref.startsWith('@') ? key.ref : `@${key.ref}`); + entered++; + } + return { ms, entered }; +} + +/** + * Attempt the write and report its typed failure rather than raising it. + * + * A masked field reports `TEXT_ENTRY_MISMATCH` on text that did arrive, and a custom digit-box + * widget can make the runner record a failure whose documented recovery is a fresh snapshot — which + * is the next thing this path does. Either way the screen decides, so the write's verdict is + * carried as evidence instead of ending the run. + */ +async function writeFailure( + device: PolicyDevicePort, + ref: string, + text: string, +): Promise { + try { + await device.fill(ref, text); + return undefined; + } catch (error) { + const code = (error as { code?: unknown } | null)?.code; + return typeof code === 'string' ? code : 'the write did not complete'; + } +} + +function valueLanded( + nodes: readonly PolicySnapshotNode[], + candidate: PolicyCandidate | undefined, + text: string, +): boolean { + const node = nodes.find((entry) => + candidate?.identifier + ? entry.identifier === candidate.identifier + : entry.label === candidate?.name, + ); + return sameText(node?.value, text); +} + +function totalsFor(steps: readonly PolicyActStep[]): PolicyActResult['totals'] { + return { + steps: steps.length, + snapshotMs: sum(steps, (step) => step.snapshotMs), + decideMs: sum(steps, (step) => step.decideMs), + actionMs: sum(steps, (step) => step.actionMs), + inputTokens: sum(steps, (step) => step.decision.inputTokens), + costUsd: sum(steps, (step) => step.decision.costUsd), + deadActions: steps.filter((step) => step.outcome === 'dead-action').length, + escalations: steps.filter((step) => step.outcome === 'escalated').length, + }; +} + +function sum(steps: readonly PolicyActStep[], read: (step: PolicyActStep) => number): number { + return steps.reduce((total, step) => total + read(step), 0); +} diff --git a/src/commands/policy/candidate-elements.test.ts b/src/commands/policy/candidate-elements.test.ts new file mode 100644 index 0000000000..c52443c596 --- /dev/null +++ b/src/commands/policy/candidate-elements.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, test } from 'vitest'; +import { + findKeypadDigit, + screenDigest, + screenLabel, + selectableCandidates, + toPolicyCandidates, + type PolicySnapshotNode, +} from './candidate-elements.ts'; + +const signInScreen: PolicySnapshotNode[] = [ + { index: 0, type: 'Application', label: 'shelf', ref: 'e1' }, + { index: 1, parentIndex: 0, type: 'StaticText', label: 'Sign in with your phone.', ref: 'e2' }, + { index: 2, parentIndex: 0, type: 'CollectionView', ref: 'e3' }, + { index: 3, parentIndex: 2, type: 'Cell', label: 'US +1', ref: 'e4' }, + { + index: 4, + parentIndex: 3, + type: 'TextField', + label: 'Phone number', + identifier: 'phoneField', + value: '', + ref: 'e5', + }, + { index: 5, parentIndex: 0, type: 'Button', label: 'Send code', enabled: false, ref: 'e6' }, +]; + +describe('policy candidate projection', () => { + test('projects a snapshot into candidates a policy can choose between', () => { + expect(toPolicyCandidates(signInScreen)).toEqual([ + { ref: '@e2', role: 'text', name: 'Sign in with your phone.' }, + { + ref: '@e5', + role: 'textfield', + name: 'Phone number', + identifier: 'phoneField', + value: '', + }, + { ref: '@e6', role: 'button', name: 'Send code', disabled: true }, + ]); + }); + + test('drops a cell that only wraps an actionable leaf', () => { + const refs = toPolicyCandidates(signInScreen).map((candidate) => candidate.ref); + expect(refs).not.toContain('@e4'); + }); + + test('offers only enabled, non-text candidates as choices', () => { + expect(selectableCandidates(toPolicyCandidates(signInScreen))).toEqual([ + { + ref: '@e5', + role: 'textfield', + name: 'Phone number', + identifier: 'phoneField', + value: '', + }, + ]); + }); + + test('digests the screen by content, so a reissued ref is not a change', () => { + const reissued = signInScreen.map((node) => ({ ...node, ref: `${node.ref}0` })); + expect(screenDigest(toPolicyCandidates(reissued))).toBe( + screenDigest(toPolicyCandidates(signInScreen)), + ); + }); + + test('digests a changed field value as a change', () => { + const filled = signInScreen.map((node) => + node.identifier === 'phoneField' ? { ...node, value: '(555) 555-0100' } : node, + ); + expect(screenDigest(toPolicyCandidates(filled))).not.toBe( + screenDigest(toPolicyCandidates(signInScreen)), + ); + }); + + test('labels a screen from its leading text and first controls', () => { + expect(screenLabel(toPolicyCandidates(signInScreen))).toBe( + 'Sign in with your phone. | Phone number', + ); + }); + + test('finds the keypad key that enters a digit', () => { + const keypad: PolicySnapshotNode[] = [ + { index: 0, type: 'Key', label: '4', ref: 'e30' }, + { index: 1, type: 'Key', label: '7', ref: 'e31' }, + ]; + expect(findKeypadDigit(keypad, '7')?.ref).toBe('e31'); + expect(findKeypadDigit(keypad, '9')).toBeUndefined(); + }); +}); diff --git a/src/commands/policy/candidate-elements.ts b/src/commands/policy/candidate-elements.ts new file mode 100644 index 0000000000..57ce7d6296 --- /dev/null +++ b/src/commands/policy/candidate-elements.ts @@ -0,0 +1,152 @@ +import type { PolicyCandidate } from './policy-contract.ts'; + +/** + * The snapshot fields the policy projection reads. Kept structural rather than importing the + * presented-node type so the projection can be unit-tested from literals. + */ +export type PolicySnapshotNode = { + index?: number; + parentIndex?: number; + type?: string; + label?: string; + value?: string; + identifier?: string; + enabled?: boolean; + ref?: string; +}; + +/** Node types that are actionable in themselves. */ +const LEAF_ROLES: Record = { + Button: 'button', + TextField: 'textfield', + SecureTextField: 'textfield', + TextView: 'textfield', + Switch: 'switch', + Link: 'link', + Slider: 'slider', + Key: 'key', + CheckBox: 'checkbox', + RadioButton: 'radio', + MenuItem: 'menuitem', +}; + +/** Container types that carry no decision value of their own. */ +const STRUCTURAL_TYPES = new Set([ + 'Application', + 'Window', + 'Other', + 'CollectionView', + 'ScrollView', + 'Table', + 'NavigationBar', + 'Group', +]); + +const TEXT_ROLE = 'text'; + +/** Roles whose value is text the caller can write. */ +export function isTextEntryRole(role: string): boolean { + return role === 'textfield'; +} + +/** + * Project a snapshot into the candidate set a policy chooses between. + * + * A list cell that wraps an actionable leaf is dropped in favour of the leaf, so the policy never + * has to choose between a row and the button inside it. Static text stays as `text` context: it is + * what distinguishes "Enter the code" from "Welcome" when every button on both screens is generic. + */ +export function toPolicyCandidates(nodes: readonly PolicySnapshotNode[]): PolicyCandidate[] { + const childrenOf = groupByParent(nodes); + const candidates: PolicyCandidate[] = []; + for (const node of nodes) { + const role = roleForNode(node, childrenOf); + if (role === undefined || !node.ref) continue; + candidates.push({ + ref: node.ref.startsWith('@') ? node.ref : `@${node.ref}`, + role, + name: node.label || node.identifier || node.type || 'element', + ...(node.identifier === undefined ? {} : { identifier: node.identifier }), + ...(node.value === undefined ? {} : { value: node.value }), + ...(node.enabled === false ? { disabled: true } : {}), + }); + } + return candidates; +} + +/** The candidate role for one node, or undefined when it carries no decision value. */ +function roleForNode( + node: PolicySnapshotNode, + childrenOf: ReadonlyMap, +): string | undefined { + const type = node.type ?? ''; + if (!node.ref || STRUCTURAL_TYPES.has(type)) return undefined; + const leafRole = LEAF_ROLES[type]; + if (leafRole) return leafRole; + if (type === 'StaticText') return TEXT_ROLE; + if (type !== 'Cell') return undefined; + // A row wrapping its own control would make the policy choose between the two. + if (node.index !== undefined && hasActionableDescendant(childrenOf, node.index)) return undefined; + return 'button'; +} + +function groupByParent(nodes: readonly PolicySnapshotNode[]): Map { + const childrenOf = new Map(); + for (const node of nodes) { + if (node.parentIndex === undefined) continue; + const siblings = childrenOf.get(node.parentIndex); + if (siblings) siblings.push(node); + else childrenOf.set(node.parentIndex, [node]); + } + return childrenOf; +} + +/** The subset a policy may choose: actionable, enabled, and not plain text. */ +export function selectableCandidates(candidates: readonly PolicyCandidate[]): PolicyCandidate[] { + return candidates.filter( + (candidate) => candidate.role !== TEXT_ROLE && candidate.disabled !== true, + ); +} + +/** + * A stable digest of what the screen shows, used to detect an action that changed nothing. + * Refs are excluded on purpose: they are reissued per snapshot generation, so including them would + * make every screen look changed. + */ +export function screenDigest(candidates: readonly PolicyCandidate[]): string { + return candidates + .map( + (candidate) => + `${candidate.role}:${candidate.name}:${candidate.value ?? ''}:${candidate.disabled ? 'd' : ''}`, + ) + .join('|'); +} + +/** A short label for logs and step records: the leading text plus the first few controls. */ +export function screenLabel(candidates: readonly PolicyCandidate[]): string { + const text = candidates.filter((candidate) => candidate.role === TEXT_ROLE).slice(0, 2); + const controls = selectableCandidates(candidates) + .filter((candidate) => candidate.role !== 'key') + .slice(0, 3); + const label = [...text, ...controls].map((candidate) => candidate.name).join(' | '); + return label.length > 0 ? label.slice(0, 80) : '(no labelled elements)'; +} + +/** The keypad key that enters `digit`, when a numeric keypad is on screen. */ +export function findKeypadDigit( + nodes: readonly PolicySnapshotNode[], + digit: string, +): PolicySnapshotNode | undefined { + return nodes.find((node) => node.type === 'Key' && node.label === digit && Boolean(node.ref)); +} + +function hasActionableDescendant( + childrenOf: ReadonlyMap, + index: number, +): boolean { + for (const child of childrenOf.get(index) ?? []) { + if (child.type && LEAF_ROLES[child.type]) return true; + if (child.index !== undefined && hasActionableDescendant(childrenOf, child.index)) return true; + } + return false; +} diff --git a/src/commands/policy/jev-composer.test.ts b/src/commands/policy/jev-composer.test.ts new file mode 100644 index 0000000000..004bbe9c8b --- /dev/null +++ b/src/commands/policy/jev-composer.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, test } from 'vitest'; +import { + buildJevRequest, + DECLINE_LABEL, + JEV_USD_PER_INPUT_TOKEN, + readJevDecision, +} from './jev-composer.ts'; +import type { PolicyCandidate, PolicyRequest } from './policy-contract.ts'; + +const candidates: PolicyCandidate[] = [ + { ref: '@e2', role: 'text', name: 'Enter the code we texted you' }, + { ref: '@e5', role: 'textfield', name: 'Code', identifier: 'codeField', value: '' }, + { ref: '@e6', role: 'button', name: 'Verify', disabled: true }, +]; + +const request: PolicyRequest = { + goal: 'sign in', + candidates, + history: ['pressed Send code'], + screen: 'Enter the code we texted you | Code', + textReadyRefs: ['@e5'], +}; + +const meta = { provider: 'jev', model: 'jev-latest', decideMs: 187 }; + +function answer(overrides: Record = {}): Record { + return { + model: 'jev-1.13.0', + answers: { + next: { + type: 'choice', + choice: '@e5', + confidence: 0.97, + probabilities: { '@e5': 0.97, [DECLINE_LABEL]: 0.03 }, + }, + done: { type: 'noul', noul: 0.02 }, + blocked: { type: 'noul', noul: 0.11 }, + }, + usage: { input_tokens: 500, output_tokens: 64 }, + ...overrides, + }; +} + +describe('jev request composition', () => { + test('offers every enabled element plus a decline label as choices', () => { + const body = buildJevRequest('jev-latest', request); + const criteria = (body.questions.next as { criteria: Record }).criteria; + expect(Object.keys(criteria).sort()).toEqual(['@e5', DECLINE_LABEL]); + }); + + test('sends the full element list as state, so text stays available as context', () => { + const body = buildJevRequest('jev-latest', request); + expect(body.state.elements).toEqual(candidates); + expect(body.state.history).toEqual(['pressed Send code']); + }); + + test('marks a field the caller has text for, without carrying the text', () => { + const body = buildJevRequest('jev-latest', request); + const criteria = (body.questions.next as { criteria: Record }).criteria; + expect(criteria['@e5']).toMatchObject({ text_ready_to_enter: true }); + expect(JSON.stringify(body)).not.toContain('482493'); + }); + + test('refuses a screen with nothing actionable on it', () => { + expect(() => + buildJevRequest('jev-latest', { ...request, candidates: candidates.slice(0, 1) }), + ).toThrow( + expect.objectContaining({ + code: 'COMMAND_FAILED', + message: expect.stringContaining('No actionable element'), + }), + ); + }); +}); + +describe('jev decision parsing', () => { + test('reads a decision from a well-formed response', () => { + expect(readJevDecision(answer(), request, meta)).toEqual({ + action: 'fill', + target: '@e5', + done: false, + blocked: false, + needsText: true, + confidence: 0.97, + probabilities: { '@e5': 0.97, [DECLINE_LABEL]: 0.03 }, + provider: 'jev', + model: 'jev-1.13.0', + decideMs: 187, + inputTokens: 500, + costUsd: 500 * JEV_USD_PER_INPUT_TOKEN, + }); + }); + + test('reports a press when the chosen field already holds a value', () => { + const filled: PolicyRequest = { + ...request, + candidates: candidates.map((candidate) => + candidate.ref === '@e5' ? { ...candidate, value: '4821' } : candidate, + ), + }; + const decision = readJevDecision(answer(), filled, meta); + expect(decision.action).toBe('press'); + expect(decision.needsText).toBe(false); + }); + + test('reads a decline as no target', () => { + const declined = answer({ + answers: { + next: { + type: 'choice', + choice: DECLINE_LABEL, + confidence: 0.6, + probabilities: { [DECLINE_LABEL]: 0.6, '@e5': 0.4 }, + }, + done: { type: 'noul', noul: 0.1 }, + blocked: { type: 'noul', noul: 0.8 }, + }, + }); + const decision = readJevDecision(declined, request, meta); + expect(decision.target).toBeNull(); + expect(decision.action).toBe('none'); + expect(decision.blocked).toBe(true); + }); + + test('rejects a choice that is not one of the offered elements', () => { + const invented = answer({ + answers: { + next: { type: 'choice', choice: '@e99', confidence: 1, probabilities: { '@e99': 1 } }, + done: { type: 'noul', noul: 0 }, + blocked: { type: 'noul', noul: 0 }, + }, + }); + expect(() => readJevDecision(invented, request, meta)).toThrow( + expect.objectContaining({ + code: 'COMMAND_FAILED', + message: expect.stringContaining('not a candidate element'), + }), + ); + }); + + test('rejects a confidence outside zero to one', () => { + const broken = answer({ + answers: { + next: { type: 'choice', choice: '@e5', confidence: 4, probabilities: { '@e5': 1 } }, + done: { type: 'noul', noul: 0 }, + blocked: { type: 'noul', noul: 0 }, + }, + }); + expect(() => readJevDecision(broken, request, meta)).toThrow( + expect.objectContaining({ code: 'COMMAND_FAILED' }), + ); + }); + + test('rejects a response with no answers at all', () => { + expect(() => readJevDecision({ usage: {} }, request, meta)).toThrow( + expect.objectContaining({ message: expect.stringContaining('answers is missing') }), + ); + }); +}); diff --git a/src/commands/policy/jev-composer.ts b/src/commands/policy/jev-composer.ts new file mode 100644 index 0000000000..f25474c5f0 --- /dev/null +++ b/src/commands/policy/jev-composer.ts @@ -0,0 +1,173 @@ +import { AppError } from '@agent-device/kernel/errors'; +import { isTextEntryRole, selectableCandidates } from './candidate-elements.ts'; +import { POLICY_FALLBACK_HINT } from './policy-fallback.ts'; +import type { PolicyCandidate, PolicyDecision, PolicyRequest } from './policy-contract.ts'; + +/** + * The label a choice question carries for "no element on this screen advances the goal". A ref can + * never collide with it: refs are `@`-prefixed. + */ +export const DECLINE_LABEL = '__none__'; + +/** + * Published input-token price for the TypeSafe System One API, in US dollars. Output tokens are + * not billed. A wrong constant only mis-reports the `costUsd` field; it never changes behaviour. + */ +export const JEV_USD_PER_INPUT_TOKEN = 42 / 1e9; + +const CHOICE_RULES = [ + 'Fill an empty required field before pressing the control that submits it', + 'Never choose a disabled element', + 'Prefer the most direct remaining step towards the goal', + 'Do not repeat an action the history shows already succeeded', + 'A field marked text_ready_to_enter can be filled immediately; its text is already available', +] as const; + +export type JevRequestBody = { + model: string; + state: Record; + questions: Record; +}; + +/** Compose the System One request for one screen. Pure: no clock, no network, no environment. */ +export function buildJevRequest(model: string, request: PolicyRequest): JevRequestBody { + const selectable = selectableCandidates(request.candidates); + if (selectable.length === 0) { + throw new AppError('COMMAND_FAILED', 'No actionable element is visible on the current screen', { + reason: 'policy-no-candidates', + hint: 'Take a snapshot to confirm the screen rendered, then retry.', + }); + } + + const criteria: Record = { + [DECLINE_LABEL]: 'No visible element advances the goal; stop and hand back to the caller', + }; + const textReady = new Set(request.textReadyRefs); + for (const candidate of selectable) { + criteria[candidate.ref] = { + role: candidate.role, + name: candidate.name, + ...(candidate.value === undefined ? {} : { value: candidate.value }), + // The caller holds text for this field, so needing it is not a reason to report blocked. + ...(textReady.has(candidate.ref) ? { text_ready_to_enter: true } : {}), + }; + } + + return { + model, + state: { + goal: request.goal, + screen: request.screen, + elements: request.candidates, + history: request.history, + fields_with_text_ready: request.textReadyRefs, + }, + questions: { + next: { + type: 'choice', + instructions: { + question: 'Which element should be acted on next to advance the goal', + rules: CHOICE_RULES, + }, + criteria, + }, + done: { + type: 'noul', + instructions: 'The goal is already fully achieved on the current screen', + }, + blocked: { + type: 'noul', + instructions: + 'Progress needs information or a credential that is neither on this screen, nor already available for a field marked text_ready_to_enter, nor obtainable by acting on this screen', + }, + }, + }; +} + +/** + * Read a System One response into a typed decision. + * + * Every field is validated against the candidate set rather than trusted: a label outside the + * criteria, or a probability that is not a number, is a provider fault and is reported as one. + */ +export function readJevDecision( + body: unknown, + request: PolicyRequest, + meta: { provider: string; model: string; decideMs: number }, +): PolicyDecision { + const answers = readRecord(readRecord(body, 'response body').answers, 'answers'); + const next = readRecord(answers.next, 'answers.next'); + const target = readChoice(next.choice, request.candidates); + + const chosen = + target === null ? undefined : request.candidates.find((candidate) => candidate.ref === target); + const needsText = chosen !== undefined && isTextEntryRole(chosen.role) && !chosen.value; + + const usage = isRecord(body) && isRecord(body.usage) ? body.usage : {}; + const inputTokens = typeof usage.input_tokens === 'number' ? usage.input_tokens : 0; + + return { + action: target === null ? 'none' : needsText ? 'fill' : 'press', + target, + done: readNoul(answers.done, 'answers.done') > 0.5, + blocked: readNoul(answers.blocked, 'answers.blocked') > 0.5, + needsText, + confidence: readUnitInterval(next.confidence, 'answers.next.confidence'), + probabilities: readProbabilities(next.probabilities), + provider: meta.provider, + model: + typeof (body as { model?: unknown }).model === 'string' + ? (body as { model: string }).model + : meta.model, + decideMs: meta.decideMs, + inputTokens, + costUsd: inputTokens * JEV_USD_PER_INPUT_TOKEN, + }; +} + +function readChoice(value: unknown, candidates: readonly PolicyCandidate[]): string | null { + if (typeof value !== 'string') { + throw providerFault('answers.next.choice is not a string'); + } + if (value === DECLINE_LABEL) return null; + if (!candidates.some((candidate) => candidate.ref === value)) { + throw providerFault(`answers.next.choice named ${value}, which is not a candidate element`); + } + return value; +} + +function readProbabilities(value: unknown): Record { + if (!isRecord(value)) throw providerFault('answers.next.probabilities is not an object'); + const probabilities: Record = {}; + for (const [label, probability] of Object.entries(value)) { + probabilities[label] = readUnitInterval(probability, `probability for ${label}`); + } + return probabilities; +} + +function readNoul(value: unknown, field: string): number { + return readUnitInterval(readRecord(value, field).noul, `${field}.noul`); +} + +function readUnitInterval(value: unknown, field: string): number { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1) { + throw providerFault(`${field} is not a probability between 0 and 1`); + } + return value; +} + +function readRecord(value: unknown, field: string): Record { + if (!isRecord(value)) throw providerFault(`${field} is missing or not an object`); + return value; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function providerFault(message: string): AppError { + return new AppError('COMMAND_FAILED', `Jev returned an unusable decision: ${message}`, { + reason: 'policy-provider-response', + hint: `Retry, or ${POLICY_FALLBACK_HINT}`, + }); +} diff --git a/src/commands/policy/jev-policy-provider.test.ts b/src/commands/policy/jev-policy-provider.test.ts new file mode 100644 index 0000000000..350370e433 --- /dev/null +++ b/src/commands/policy/jev-policy-provider.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, test } from 'vitest'; +import { createJevPolicyProvider } from './jev-policy-provider.ts'; +import type { PolicyRequest } from './policy-contract.ts'; + +const request: PolicyRequest = { + goal: 'sign in', + candidates: [ + { ref: '@e5', role: 'textfield', name: 'Phone number', identifier: 'phoneField', value: '' }, + { ref: '@e6', role: 'button', name: 'Send code' }, + ], + history: [], + screen: 'Phone number | Send code', + textReadyRefs: [], +}; + +const body = { + model: 'jev-1.13.0', + answers: { + next: { + type: 'choice', + choice: '@e5', + confidence: 0.99, + probabilities: { '@e5': 0.99, '@e6': 0.01 }, + }, + done: { type: 'noul', noul: 0.01 }, + blocked: { type: 'noul', noul: 0.9 }, + }, + usage: { input_tokens: 412 }, +}; + +function respondWith(status: number, payload: unknown): typeof globalThis.fetch { + return (async () => + new Response(typeof payload === 'string' ? payload : JSON.stringify(payload), { + status, + headers: { 'content-type': 'application/json' }, + })) as unknown as typeof globalThis.fetch; +} + +describe('jev policy provider', () => { + test('sends the key as a bearer token and returns a typed decision', async () => { + let seen: { url: string; init: RequestInit } | undefined; + const provider = createJevPolicyProvider({ + apiKey: 'test-key', + now: () => 0, + fetch: (async (url: string, init: RequestInit) => { + seen = { url, init }; + return new Response(JSON.stringify(body), { status: 200 }); + }) as unknown as typeof globalThis.fetch, + }); + + const decision = await provider.decide(request); + + expect(seen?.url).toBe('https://api.typesafe.ai/v1/systemone'); + const headers = seen?.init.headers as Record | undefined; + expect(headers?.authorization).toBe('Bearer test-key'); + expect(decision.target).toBe('@e5'); + expect(decision.action).toBe('fill'); + expect(decision.blocked).toBe(true); + expect(decision.inputTokens).toBe(412); + }); + + test('reports a rejected key as unauthorized, naming the fallback', async () => { + const provider = createJevPolicyProvider({ + apiKey: 'bad', + fetch: respondWith(401, { error: 'invalid key' }), + }); + await expect(provider.decide(request)).rejects.toThrow( + expect.objectContaining({ + code: 'UNAUTHORIZED', + message: expect.stringContaining('Jev unavailable: 401'), + }), + ); + }); + + test('marks a rate limit retriable and a payment failure not', async () => { + const limited = createJevPolicyProvider({ apiKey: 'k', fetch: respondWith(429, 'slow down') }); + await expect(limited.decide(request)).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'policy-provider-rate-limited', retriable: true, status: 429 }, + }); + + const unpaid = createJevPolicyProvider({ apiKey: 'k', fetch: respondWith(402, 'no credit') }); + await expect(unpaid.decide(request)).rejects.toMatchObject({ + details: { reason: 'policy-provider-payment-required', retriable: false, status: 402 }, + }); + }); + + test('reports a transport failure as retriable rather than as a decision', async () => { + const provider = createJevPolicyProvider({ + apiKey: 'k', + fetch: (async () => { + throw new Error('socket hang up'); + }) as unknown as typeof globalThis.fetch, + }); + await expect(provider.decide(request)).rejects.toMatchObject({ + message: 'Jev unavailable: socket hang up', + details: { reason: 'policy-provider-transport', retriable: true }, + }); + }); + + test('measures decide time from the injected clock', async () => { + let clock = 1_000; + const provider = createJevPolicyProvider({ + apiKey: 'k', + fetch: respondWith(200, body), + now: () => { + const value = clock; + clock += 203; + return value; + }, + }); + expect((await provider.decide(request)).decideMs).toBe(203); + }); +}); diff --git a/src/commands/policy/jev-policy-provider.ts b/src/commands/policy/jev-policy-provider.ts new file mode 100644 index 0000000000..30b0cec691 --- /dev/null +++ b/src/commands/policy/jev-policy-provider.ts @@ -0,0 +1,119 @@ +import { AppError } from '@agent-device/kernel/errors'; +import { buildJevRequest, readJevDecision } from './jev-composer.ts'; +import { POLICY_FALLBACK_HINT } from './policy-fallback.ts'; +import type { PolicyDecision, PolicyProvider, PolicyRequest } from './policy-contract.ts'; + +const JEV_PROVIDER_NAME = 'jev'; +const JEV_DEFAULT_ENDPOINT = 'https://api.typesafe.ai/v1/systemone'; +const JEV_DEFAULT_MODEL = 'jev-latest'; +const JEV_TIMEOUT_MS = 20_000; + +export type JevProviderOptions = { + apiKey: string; + model?: string; + endpoint?: string; + /** Injected so unit tests exercise the real composer and parser against a fake transport. */ + fetch?: typeof globalThis.fetch; + /** Injected so step timings are assertable. */ + now?: () => number; +}; + +export function createJevPolicyProvider(options: JevProviderOptions): PolicyProvider { + const model = options.model ?? JEV_DEFAULT_MODEL; + const endpoint = options.endpoint ?? JEV_DEFAULT_ENDPOINT; + const fetchImpl = options.fetch ?? globalThis.fetch; + const now = options.now ?? (() => Date.now()); + + return { + name: JEV_PROVIDER_NAME, + model, + decide: async (request: PolicyRequest): Promise => { + const body = buildJevRequest(model, request); + const startedAt = now(); + const response = await callJev(fetchImpl, endpoint, options.apiKey, body); + if (!response.ok) throw unavailable(response.status, await readErrorText(response)); + const parsed: unknown = await response.json(); + return readJevDecision(parsed, request, { + provider: JEV_PROVIDER_NAME, + model, + decideMs: now() - startedAt, + }); + }, + }; +} + +async function callJev( + fetchImpl: typeof globalThis.fetch, + endpoint: string, + apiKey: string, + body: unknown, +): Promise { + try { + return await fetchImpl(endpoint, { + method: 'POST', + headers: { + authorization: `Bearer ${apiKey}`, + 'content-type': 'application/json', + accept: 'application/json', + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(JEV_TIMEOUT_MS), + }); + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Jev unavailable: ${error instanceof Error ? error.message : 'request failed'}`, + { + reason: 'policy-provider-transport', + retriable: true, + hint: `Retry, or ${POLICY_FALLBACK_HINT}`, + }, + error, + ); + } +} + +/** + * Status is carried as a typed `reason` and `status` detail rather than being sniffed back out of + * the message, so a caller can branch on it without matching text. + */ +function unavailable(status: number, detail: string): AppError { + const code = status === 401 || status === 403 ? 'UNAUTHORIZED' : 'COMMAND_FAILED'; + return new AppError( + code, + `Jev unavailable: ${status}${detail ? ` ${detail}` : ''}; fall back to agent-driven policy`, + { + reason: reasonForStatus(status), + status, + retriable: status === 429 || status >= 500, + hint: hintForStatus(status), + }, + ); +} + +function reasonForStatus(status: number): string { + if (status === 401 || status === 403) return 'policy-provider-unauthorized'; + if (status === 402) return 'policy-provider-payment-required'; + if (status === 429) return 'policy-provider-rate-limited'; + return 'policy-provider-http'; +} + +function hintForStatus(status: number): string { + if (status === 401 || status === 403) { + return `TYPESAFE_API_KEY is missing or rejected; ${POLICY_FALLBACK_HINT}`; + } + if (status === 402) return `The TypeSafe account has no credit; ${POLICY_FALLBACK_HINT}`; + if (status === 429) { + return `Rate limited; keep policy concurrency at or below 4, or ${POLICY_FALLBACK_HINT}`; + } + return `Retry, or ${POLICY_FALLBACK_HINT}`; +} + +async function readErrorText(response: Response): Promise { + try { + const text = await response.text(); + return text.slice(0, 200).replaceAll(/\s+/gu, ' ').trim(); + } catch { + return ''; + } +} diff --git a/src/commands/policy/policy-contract.ts b/src/commands/policy/policy-contract.ts new file mode 100644 index 0000000000..9b54f3f24c --- /dev/null +++ b/src/commands/policy/policy-contract.ts @@ -0,0 +1,130 @@ +/** + * Types shared by the policy provider abstraction, the `suggest` command, and the `act` loop. + * A policy provider answers one question — which on-screen element advances a goal — and returns + * a typed decision with calibrated probabilities instead of free-form text. + */ + +/** One element a policy may choose between, projected from an interactive snapshot. */ +export type PolicyCandidate = { + /** Snapshot ref, including its leading `@`. */ + ref: string; + /** Neutral role name (`button`, `textfield`, `switch`, `link`, `key`, `text`). */ + role: string; + /** Accessibility label, identifier, or node type, in that order of preference. */ + name: string; + /** Accessibility identifier, when the node carries one. The stable key for input matching. */ + identifier?: string; + /** Current accessibility value, when the node reports one. */ + value?: string; + /** True when the node reports itself disabled; disabled nodes stay visible as context. */ + disabled?: boolean; +}; + +/** What the caller should do with the chosen element. */ +export type PolicyActionKind = 'press' | 'fill' | 'none'; + +/** The decision a provider returns for one screen. */ +export type PolicyDecision = { + action: PolicyActionKind; + /** Chosen ref, or null when the provider declined to choose. */ + target: string | null; + /** Provider believes the goal is already satisfied on this screen. */ + done: boolean; + /** Provider believes progress needs something not on this screen. */ + blocked: boolean; + /** Chosen target is an empty text field, so the caller must supply text. */ + needsText: boolean; + /** Provider-reported confidence in the chosen target, 0 to 1. */ + confidence: number; + /** Probability per candidate ref plus the decline label, 0 to 1. */ + probabilities: Record; + provider: string; + model: string; + /** Wall-clock milliseconds spent in the provider call. */ + decideMs: number; + inputTokens: number; + costUsd: number; +}; + +/** Why a step ended the way it did. */ +export type PolicyStepOutcome = + | 'acted' + | 'done' + | 'blocked' + | 'escalated' + | 'dead-action' + | 'failed'; + +/** What the loop did on one step, with the three phase timings kept apart. */ +export type PolicyActStep = { + step: number; + /** Short human label for the screen the decision was made on. */ + screen: string; + decision: PolicyDecision; + outcome: PolicyStepOutcome; + /** Present when the step performed a device action. */ + performed?: { + action: PolicyActionKind; + target: string; + /** How text reached the field; `keypad` is the digit-field fallback. */ + entry?: 'fill' | 'keypad'; + /** Input map key whose value was sent. The value itself is never recorded. */ + inputKey?: string; + }; + snapshotMs: number; + decideMs: number; + actionMs: number; + /** Set when the step could not proceed, in the loop's vocabulary. */ + reason?: string; +}; + +export type PolicyActStatus = 'done' | 'blocked' | 'escalated' | 'max-steps'; + +export type PolicyActTotals = { + steps: number; + snapshotMs: number; + decideMs: number; + actionMs: number; + inputTokens: number; + costUsd: number; + deadActions: number; + escalations: number; +}; + +export type PolicySuggestResult = { + goal: string; + decision: PolicyDecision; + candidates: PolicyCandidate[]; + snapshotMs: number; + screen: string; +}; + +export type PolicyActResult = { + goal: string; + status: PolicyActStatus; + steps: PolicyActStep[]; + totals: PolicyActTotals; +}; + +/** Everything a provider needs to judge one screen. */ +export type PolicyRequest = { + goal: string; + candidates: PolicyCandidate[]; + /** What the loop already did, oldest first; empty for a one-shot `suggest`. */ + history: string[]; + /** Short label for the current screen, used as context only. */ + screen: string; + /** + * Refs of fields the caller has text ready for. Values are never included: the policy only needs + * to know the text exists, and without that it reports a code screen as blocked even when the + * caller supplied the code. + */ + textReadyRefs: readonly string[]; +}; + +/** A policy head. One implementation ships today: `jev`. */ +export type PolicyProvider = { + readonly name: string; + readonly model: string; + decide: (request: PolicyRequest) => Promise; +}; diff --git a/src/commands/policy/policy-fallback.ts b/src/commands/policy/policy-fallback.ts new file mode 100644 index 0000000000..a99c87606d --- /dev/null +++ b/src/commands/policy/policy-fallback.ts @@ -0,0 +1,9 @@ +/** + * The documented recovery when the policy head cannot answer. It names the agent's own loop + * because that loop is always available: the policy head is an accelerator, never a dependency. + * + * It lives apart from both the provider and the composer so either can name it without one + * importing the other. + */ +export const POLICY_FALLBACK_HINT = + 'fall back to agent-driven policy: snapshot, choose an element yourself, then press or fill it.'; diff --git a/src/commands/policy/policy-provider.ts b/src/commands/policy/policy-provider.ts new file mode 100644 index 0000000000..479e27b517 --- /dev/null +++ b/src/commands/policy/policy-provider.ts @@ -0,0 +1,57 @@ +import { AppError } from '@agent-device/kernel/errors'; +import { + POLICY_API_KEY_ENV, + POLICY_PROVIDER_NAMES, +} from '@agent-device/command-registry/flag-definitions-workflow'; +import { createJevPolicyProvider } from './jev-policy-provider.ts'; +import { POLICY_FALLBACK_HINT } from './policy-fallback.ts'; +import type { PolicyProvider } from './policy-contract.ts'; + +export type PolicyProviderName = (typeof POLICY_PROVIDER_NAMES)[number]; + +/** Environment keys the jev provider reads beyond its credential. */ +const JEV_MODEL_ENV = 'AGENT_DEVICE_POLICY_MODEL'; +const JEV_ENDPOINT_ENV = 'AGENT_DEVICE_POLICY_ENDPOINT'; + +export type PolicyProviderEnvironment = Readonly>; + +function isPolicyProviderName(value: string): value is PolicyProviderName { + return (POLICY_PROVIDER_NAMES as readonly string[]).includes(value); +} + +/** + * Resolve a provider by name. + * + * The credential comes from the environment only, so it never reaches argv, a recorded script, a + * session journal, or an MCP tool schema. Without it the command refuses before touching the + * device, which is what keeps the feature invisible to everyone who has not opted in. + */ +export function createPolicyProvider( + name: string, + env: PolicyProviderEnvironment, + overrides: { fetch?: typeof globalThis.fetch } = {}, +): PolicyProvider { + if (!isPolicyProviderName(name)) { + throw new AppError( + 'INVALID_ARGS', + `Unknown policy provider ${name}; available: ${POLICY_PROVIDER_NAMES.join(', ')}`, + ); + } + const apiKey = env[POLICY_API_KEY_ENV]?.trim(); + if (!apiKey) { + throw new AppError( + 'INVALID_ARGS', + `${POLICY_API_KEY_ENV} is not set, so policy ${name} cannot decide`, + { + reason: 'policy-provider-unconfigured', + hint: `Export ${POLICY_API_KEY_ENV} to enable the policy head, or ${POLICY_FALLBACK_HINT}`, + }, + ); + } + return createJevPolicyProvider({ + apiKey, + model: env[JEV_MODEL_ENV], + endpoint: env[JEV_ENDPOINT_ENV], + ...overrides, + }); +} diff --git a/src/commands/policy/text-inputs.test.ts b/src/commands/policy/text-inputs.test.ts new file mode 100644 index 0000000000..700bcbf1da --- /dev/null +++ b/src/commands/policy/text-inputs.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, test } from 'vitest'; +import { parsePolicyTextInputs, policyInputEnvName, resolveTextInput } from './text-inputs.ts'; +import type { PolicyCandidate } from './policy-contract.ts'; + +const phoneField: PolicyCandidate = { + ref: '@e5', + role: 'textfield', + name: 'Phone number', + identifier: 'phoneField', + value: '', +}; + +describe('policy text inputs', () => { + test('parses repeated key=value tokens, keeping a value that contains =', () => { + expect(parsePolicyTextInputs(['phone=5555550100', 'note=a=b'])).toEqual( + new Map([ + ['phone', '5555550100'], + ['note', 'a=b'], + ]), + ); + }); + + test('refuses a token that is not key=value', () => { + expect(() => parsePolicyTextInputs(['5555550100'])).toThrow( + expect.objectContaining({ + code: 'INVALID_ARGS', + message: expect.stringContaining('--input expects key=value'), + }), + ); + }); + + test('matches a key against the field identifier', () => { + expect(resolveTextInput(phoneField, parsePolicyTextInputs(['phone=5550100']), {})).toEqual({ + key: 'phone', + text: '5550100', + }); + }); + + test('prefers an exact match over a containing one', () => { + const inputs = parsePolicyTextInputs(['phone=containing', 'phonefield=exact']); + expect(resolveTextInput(phoneField, inputs, {})?.text).toBe('exact'); + }); + + test('reads a value from the environment so a secret stays out of argv', () => { + const env = { [policyInputEnvName('code')]: '482493' }; + const codeField: PolicyCandidate = { + ref: '@e7', + role: 'textfield', + name: 'Code', + identifier: 'codeField', + value: '', + }; + expect(resolveTextInput(codeField, new Map(), env)).toEqual({ key: 'code', text: '482493' }); + }); + + test('names the environment entry for a key', () => { + expect(policyInputEnvName('one-time code')).toBe('AGENT_DEVICE_INPUT_ONE_TIME_CODE'); + }); + + test('returns nothing when no entry matches the field', () => { + expect(resolveTextInput(phoneField, parsePolicyTextInputs(['code=1']), {})).toBeUndefined(); + }); +}); diff --git a/src/commands/policy/text-inputs.ts b/src/commands/policy/text-inputs.ts new file mode 100644 index 0000000000..2879cdfdc7 --- /dev/null +++ b/src/commands/policy/text-inputs.ts @@ -0,0 +1,67 @@ +import { POLICY_INPUT_ENV_PREFIX } from '@agent-device/command-registry/flag-definitions-workflow'; +import { AppError } from '@agent-device/kernel/errors'; +import type { PolicyCandidate } from './policy-contract.ts'; + +/** + * Text the loop may send to a field, keyed by a name the caller chooses. The loop never invents + * text: a field with no matching entry escalates instead of being filled with a guess. + */ +export type PolicyTextInputs = ReadonlyMap; + +/** Parse repeated `--input key=value` tokens. An empty value is allowed; an empty key is not. */ +export function parsePolicyTextInputs(tokens: readonly string[]): Map { + const inputs = new Map(); + for (const token of tokens) { + const separator = token.indexOf('='); + const key = separator === -1 ? '' : token.slice(0, separator).trim(); + if (!key) { + throw new AppError('INVALID_ARGS', `--input expects key=value, received ${token}`); + } + inputs.set(key.toLowerCase(), token.slice(separator + 1)); + } + return inputs; +} + +/** The environment name that carries `key`, for callers who keep secrets out of argv. */ +export function policyInputEnvName(key: string): string { + return `${POLICY_INPUT_ENV_PREFIX}${key.toUpperCase().replaceAll(/[^A-Z0-9]/g, '_')}`; +} + +export type ResolvedTextInput = { key: string; text: string }; + +/** + * Find the text for a field. + * + * Matching runs from most to least specific — identifier, label, then either one containing the + * key — so `--input phone=...` reaches a field identified `phoneField` without also matching a + * "Phone support" link. Environment entries are consulted for the same keys, which is how an OTP + * or password reaches the loop without being written into a command line. + */ +export function resolveTextInput( + candidate: PolicyCandidate, + inputs: PolicyTextInputs, + env: Readonly>, +): ResolvedTextInput | undefined { + const identifier = candidate.identifier?.toLowerCase() ?? ''; + const name = candidate.name.toLowerCase(); + const keys = new Set([...inputs.keys(), ...environmentKeys(env)]); + + const matchers: Array<(key: string) => boolean> = [ + (key) => identifier === key || name === key, + (key) => identifier.includes(key) || name.includes(key), + ]; + for (const matches of matchers) { + for (const key of keys) { + if (!matches(key)) continue; + const text = inputs.get(key) ?? env[policyInputEnvName(key)]; + if (text !== undefined) return { key, text }; + } + } + return undefined; +} + +function environmentKeys(env: Readonly>): string[] { + return Object.keys(env) + .filter((name) => name.startsWith(POLICY_INPUT_ENV_PREFIX)) + .map((name) => name.slice(POLICY_INPUT_ENV_PREFIX.length).toLowerCase()); +} diff --git a/website/docs/docs/_meta.json b/website/docs/docs/_meta.json index 68655a6001..c69962ab84 100644 --- a/website/docs/docs/_meta.json +++ b/website/docs/docs/_meta.json @@ -56,6 +56,11 @@ "label": "Batching", "link": "/docs/batching" }, + { + "type": "custom-link", + "label": "Policy head", + "link": "/docs/policy-head" + }, { "type": "custom-link", "label": "Snapshots", diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 18a4aefd42..38dc685cc0 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -592,6 +592,35 @@ agent-device batch --steps '[{"command":"open","input":{"app":"settings"}}]' See [Batching](/docs/batching) for payload format, response shape, and usage guidelines. +## Policy head (optional) + +```bash +agent-device suggest "sign in and reach the main list" +agent-device act "sign in and reach the main list" --input phone=5555550100 --max-steps 10 +agent-device act "open the first item" --min-confidence 0.6 --json +``` + +- A policy head answers one question — which visible element advances the goal — as a typed + decision with calibrated probabilities. `suggest` prints that decision; `act` loops + snapshot, decide, press or fill until the goal is done, blocked, or the policy loses confidence. +- Both commands are always listed and always offered over MCP. Without `TYPESAFE_API_KEY` in the + environment they refuse to run, with a typed `INVALID_ARGS` naming the variable and the fallback, + before any snapshot, press or fill reaches the device. Nothing else in the CLI changes: neither + command owns a daemon route, and no existing command's schema moves. +- `--policy ` selects the head; `jev` is the only one today and the default. +- `act` never generates text. A field the policy chooses is filled from `--input key=value`, or + from `AGENT_DEVICE_INPUT_` in the environment when the value is a secret. A field with no + matching entry escalates instead of being filled with a guess. +- After every action the loop re-snapshots and compares a content digest of the screen. An action + that changed nothing is recorded as a dead action, and three unproductive steps in a row end the + run as `escalated`. +- A code or PIN field that ignores `fill` is entered digit by digit on the on-screen keypad. +- When the head is unreachable the error names the status and the recovery: `Jev unavailable: + ; fall back to agent-driven policy`. The agent's own snapshot-and-choose loop is always + available; the policy head is an accelerator, not a dependency. + +See [Policy head](/docs/policy-head) for the decision shape, timings, and limitations. + ## App install (in-place) ```bash diff --git a/website/docs/docs/policy-head.md b/website/docs/docs/policy-head.md new file mode 100644 index 0000000000..24555c1e5b --- /dev/null +++ b/website/docs/docs/policy-head.md @@ -0,0 +1,136 @@ +--- +title: Policy head +--- + +# Policy head + +An agent driving a device spends most of its wall clock deciding, not acting. A snapshot returns in +roughly 400 ms and a press settles in one to ten seconds, but the model reading that snapshot and +choosing a ref usually takes several seconds and costs a full inference. Over a twelve-step sign-in +flow that is the largest single line in the bill and the longest single phase of the run. + +A policy head answers only the narrow question — which of these elements advances this goal — as a +typed decision with calibrated probabilities. It does not write text, plan, or explain. `suggest` +asks it once. `act` puts it in a loop. + +The feature is optional. Both commands are always listed by `agent-device help commands` and always +offered over MCP, because the command catalog is the same with or without a credential. What the +credential changes is whether they run: without `TYPESAFE_API_KEY` in the environment each refuses +with a typed `INVALID_ARGS` naming the variable and the fallback, before any snapshot, press or fill +reaches the device. Nothing else in the CLI behaves differently. + +## Asking once + +```bash +agent-device suggest "sign in and reach the main list" +``` + +``` +Goal: sign in and reach the main list +Screen: Sign in with your phone. | Phone number | Send code +Next: fill @e7 at confidence 0.99 (blocked) +Probabilities: @e7 99.0%, @e9 0.6%, __none__ 0.4% +Timing: snapshot 412ms decide 198ms +Cost: $0.000017 for 412 input tokens +``` + +`suggest` takes one interactive snapshot and prints the decision. It performs no action, so it is +safe to run repeatedly while working through a flow by hand. With `--json` the same decision comes +back as a typed object: `target`, `action`, `confidence`, `probabilities`, `done`, `blocked`, +`needsText`, plus the timing and token cost. + +## Running the loop + +```bash +agent-device act "sign in and reach the main list" \ + --input phone=5555550100 \ + --max-steps 10 +``` + +Each step snapshots, asks the policy, presses or fills, and snapshots again. The run ends when the +policy reports the goal done, reports it blocked with no element worth acting on, or three +unproductive steps happen in a row. + +| Flag | Meaning | +| --- | --- | +| `--policy ` | Which head to ask. `jev` is the default and the only one today. | +| `--max-steps ` | Step budget for the run. Default 12. | +| `--min-confidence ` | Escalate instead of acting below this confidence. Default 0.4. | +| `--input ` | Repeatable. Text the loop may enter. | + +## Text is supplied, never generated + +The policy chooses *where* text goes. It never chooses *what* the text is. A field the policy picks +is filled from an `--input` entry whose key matches the field's accessibility identifier or label, +first exactly and then as a substring. + +A press on a text field only focuses it, so the loop reads the supplied entry rather than the field's +emptiness to decide what to do. A field whose value is not the supplied text is rewritten, which is +what reaches past a development default the caller overrode. A field already holding that text is +pressed, so the loop cannot spend its whole budget refilling one field. A field that is empty with no +matching entry escalates. + +For a value that should not appear in a command line or a shell history, set +`AGENT_DEVICE_INPUT_` in the environment instead. A one-time code fetched from a test backend +belongs there: + +```bash +AGENT_DEVICE_INPUT_CODE=$(fetch-test-otp) agent-device act "finish signing in" +``` + +The step record names the input key that was used. It never records the value. + +## What the loop checks after it acts + +**Same-screen check.** After every action the loop re-snapshots and compares a content digest of the +screen: roles, names, values, and disabled states, with refs deliberately excluded because they are +reissued per snapshot generation. An action that leaves the digest unchanged is recorded as a dead +action rather than counted as progress, and three in a row end the run. + +**Digit fields.** A code or PIN field is often a row of single-character boxes over a hidden input. +`fill` reports success, the accessibility value never changes, and the submit button stays disabled. +When the loop sees a fill of all-digit text that did not land, it enters the digits one at a time on +the on-screen keypad, taking a fresh snapshot per key because refs are reissued after every press. + +**Ref pinning.** Every mutation is pinned to the generation of the snapshot that issued its ref, so a +decision made against a screen that has since changed is refused rather than landing on whatever +element inherited the ref. + +## Blocked is about the screen, not the goal + +A policy asked whether progress is blocked says yes on anything that looks like a wall. An +unauthenticated sign-in screen is literally a login wall. An onboarding page offering to import a +file is blocked on a file. In live runs the flag was set twice while the policy also named the one +element that led onward: the phone field, and an "I'll start fresh" button. + +So the flag is terminal only together with nothing worth acting on: no target, or one below +`--min-confidence`. A named target above the floor is acted on. The cost of being wrong there is one +step, which the same-screen check catches and the step budget bounds. The cost of believing the flag +is the whole run. + +## When the head is unreachable + +Failures are typed, carry the HTTP status, and name the recovery: + +``` +Jev unavailable: 429 rate limit exceeded; fall back to agent-driven policy +``` + +| Status | `reason` | Retriable | +| --- | --- | --- | +| 401, 403 | `policy-provider-unauthorized` | no | +| 402 | `policy-provider-payment-required` | no | +| 429 | `policy-provider-rate-limited` | yes | +| 5xx, transport | `policy-provider-http`, `policy-provider-transport` | yes | + +The documented fallback is the agent's own loop: `snapshot`, choose an element, `press` or `fill`. +That path is always available and is what the CLI does today. The policy head accelerates it; it is +never a dependency. + +## Limitations + +- Needs an API key. There is no offline or local head. +- Decides between elements only. It writes no text and plans no multi-screen strategy. +- Keeps concurrency low. More than about four parallel calls invites rate limiting. +- Reads the accessibility tree, so an element the snapshot cannot see is an element the policy + cannot choose.