From 073d46e6639d856e5feddcf537d3fa22cd6d8f56 Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 18:06:19 -0400 Subject: [PATCH 01/17] feat(cli): declare the policy-head flags `--policy`, `--min-confidence`, and `--input` for the policy commands, and one consumer added to the existing `--max-steps` declaration rather than a second flag under the same name. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/flag-definitions-workflow.ts | 33 ++++++++++++++++++- packages/contracts/src/cli-flags.ts | 6 ++++ scripts/integration-progress-model.ts | 3 ++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/packages/command-registry/src/flag-definitions-workflow.ts b/packages/command-registry/src/flag-definitions-workflow.ts index 5399fe3e0f..e20c8c14a1 100644 --- a/packages/command-registry/src/flag-definitions-workflow.ts +++ b/packages/command-registry/src/flag-definitions-workflow.ts @@ -195,7 +195,7 @@ export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ min: 1, max: 1000, usageLabel: '--max-steps ', - usageDescription: 'Batch: maximum number of allowed steps', + usageDescription: 'Batch: maximum allowed steps; act: stop the loop after this many steps', projectConfig: true, recorded: false, }, @@ -322,6 +322,37 @@ export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ projectConfig: false, recorded: false, }, + { + key: 'policy', + names: ['--policy'], + type: 'string', + usageLabel: '--policy ', + usageDescription: 'suggest/act: policy head that decides the next element (default jev)', + projectConfig: true, + recorded: false, + }, + { + key: 'minConfidence', + names: ['--min-confidence'], + type: 'number', + usageLabel: '--min-confidence ', + usageDescription: 'act: escalate instead of acting below this policy confidence (default 0.4)', + projectConfig: false, + recorded: false, + }, + { + key: 'policyInput', + names: ['--input'], + type: 'string', + multiple: true, + usageLabel: '--input ', + usageDescription: + 'act: repeatable text the loop may enter, matched to a field by identifier or label; the loop never generates text', + inputDescription: + 'Text the loop may enter, as key=value. The key matches a field identifier or label.', + projectConfig: false, + recorded: false, + }, { key: 'overlayRefs', names: ['--overlay-refs'], diff --git a/packages/contracts/src/cli-flags.ts b/packages/contracts/src/cli-flags.ts index c5ba3f1649..da804a7e0a 100644 --- a/packages/contracts/src/cli-flags.ts +++ b/packages/contracts/src/cli-flags.ts @@ -160,6 +160,12 @@ export type CliFlags = CloudProviderProfileFields & runtime?: SessionRuntimeHints; }>; out?: string; + /** suggest/act: which policy head answers "what next". */ + policy?: string; + /** act: escalate instead of acting below this confidence. */ + minConfidence?: number; + /** act: repeated `key=value` text the loop may enter; it never generates text itself. */ + policyInput?: string[]; help: boolean; version: boolean; }; diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index f8da3addcd..2782f6c978 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -326,6 +326,9 @@ function summarizeProviderScenarioFlagExclusions() { 'shardAll', 'shardSplit', 'searchPath', + 'policy', + 'minConfidence', + 'policyInput', 'stepsFile', 'proxyHost', 'proxyPort', From f7d9f60fbbf68110b15da063af5858edbc873bf9 Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 18:06:29 -0400 Subject: [PATCH 02/17] feat(policy): project a snapshot into policy candidates The neutral half of the policy feature: the candidate projection a policy head chooses between, a content digest that detects an action which changed nothing, and the caller-supplied text map a field is filled from. The digest excludes refs on purpose. Refs are reissued per snapshot generation, so including them would make every screen look changed. Co-Authored-By: Claude Opus 5 (1M context) --- .../policy/candidate-elements.test.ts | 90 +++++++++++ src/commands/policy/candidate-elements.ts | 145 ++++++++++++++++++ src/commands/policy/policy-contract.ts | 130 ++++++++++++++++ src/commands/policy/text-inputs.test.ts | 63 ++++++++ src/commands/policy/text-inputs.ts | 69 +++++++++ 5 files changed, 497 insertions(+) create mode 100644 src/commands/policy/candidate-elements.test.ts create mode 100644 src/commands/policy/candidate-elements.ts create mode 100644 src/commands/policy/policy-contract.ts create mode 100644 src/commands/policy/text-inputs.test.ts create mode 100644 src/commands/policy/text-inputs.ts diff --git a/src/commands/policy/candidate-elements.test.ts b/src/commands/policy/candidate-elements.test.ts new file mode 100644 index 0000000000..c52443c596 --- /dev/null +++ b/src/commands/policy/candidate-elements.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, test } from 'vitest'; +import { + findKeypadDigit, + screenDigest, + screenLabel, + selectableCandidates, + toPolicyCandidates, + type PolicySnapshotNode, +} from './candidate-elements.ts'; + +const signInScreen: PolicySnapshotNode[] = [ + { index: 0, type: 'Application', label: 'shelf', ref: 'e1' }, + { index: 1, parentIndex: 0, type: 'StaticText', label: 'Sign in with your phone.', ref: 'e2' }, + { index: 2, parentIndex: 0, type: 'CollectionView', ref: 'e3' }, + { index: 3, parentIndex: 2, type: 'Cell', label: 'US +1', ref: 'e4' }, + { + index: 4, + parentIndex: 3, + type: 'TextField', + label: 'Phone number', + identifier: 'phoneField', + value: '', + ref: 'e5', + }, + { index: 5, parentIndex: 0, type: 'Button', label: 'Send code', enabled: false, ref: 'e6' }, +]; + +describe('policy candidate projection', () => { + test('projects a snapshot into candidates a policy can choose between', () => { + expect(toPolicyCandidates(signInScreen)).toEqual([ + { ref: '@e2', role: 'text', name: 'Sign in with your phone.' }, + { + ref: '@e5', + role: 'textfield', + name: 'Phone number', + identifier: 'phoneField', + value: '', + }, + { ref: '@e6', role: 'button', name: 'Send code', disabled: true }, + ]); + }); + + test('drops a cell that only wraps an actionable leaf', () => { + const refs = toPolicyCandidates(signInScreen).map((candidate) => candidate.ref); + expect(refs).not.toContain('@e4'); + }); + + test('offers only enabled, non-text candidates as choices', () => { + expect(selectableCandidates(toPolicyCandidates(signInScreen))).toEqual([ + { + ref: '@e5', + role: 'textfield', + name: 'Phone number', + identifier: 'phoneField', + value: '', + }, + ]); + }); + + test('digests the screen by content, so a reissued ref is not a change', () => { + const reissued = signInScreen.map((node) => ({ ...node, ref: `${node.ref}0` })); + expect(screenDigest(toPolicyCandidates(reissued))).toBe( + screenDigest(toPolicyCandidates(signInScreen)), + ); + }); + + test('digests a changed field value as a change', () => { + const filled = signInScreen.map((node) => + node.identifier === 'phoneField' ? { ...node, value: '(555) 555-0100' } : node, + ); + expect(screenDigest(toPolicyCandidates(filled))).not.toBe( + screenDigest(toPolicyCandidates(signInScreen)), + ); + }); + + test('labels a screen from its leading text and first controls', () => { + expect(screenLabel(toPolicyCandidates(signInScreen))).toBe( + 'Sign in with your phone. | Phone number', + ); + }); + + test('finds the keypad key that enters a digit', () => { + const keypad: PolicySnapshotNode[] = [ + { index: 0, type: 'Key', label: '4', ref: 'e30' }, + { index: 1, type: 'Key', label: '7', ref: 'e31' }, + ]; + expect(findKeypadDigit(keypad, '7')?.ref).toBe('e31'); + expect(findKeypadDigit(keypad, '9')).toBeUndefined(); + }); +}); diff --git a/src/commands/policy/candidate-elements.ts b/src/commands/policy/candidate-elements.ts new file mode 100644 index 0000000000..eaf6bdf5e4 --- /dev/null +++ b/src/commands/policy/candidate-elements.ts @@ -0,0 +1,145 @@ +import type { PolicyCandidate } from './policy-contract.ts'; + +/** + * The snapshot fields the policy projection reads. Kept structural rather than importing the + * presented-node type so the projection can be unit-tested from literals. + */ +export type PolicySnapshotNode = { + index?: number; + parentIndex?: number; + type?: string; + label?: string; + value?: string; + identifier?: string; + enabled?: boolean; + ref?: string; +}; + +/** Node types that are actionable in themselves. */ +const LEAF_ROLES: Record = { + Button: 'button', + TextField: 'textfield', + SecureTextField: 'textfield', + TextView: 'textfield', + Switch: 'switch', + Link: 'link', + Slider: 'slider', + Key: 'key', + CheckBox: 'checkbox', + RadioButton: 'radio', + MenuItem: 'menuitem', +}; + +/** Container types that carry no decision value of their own. */ +const STRUCTURAL_TYPES = new Set([ + 'Application', + 'Window', + 'Other', + 'CollectionView', + 'ScrollView', + 'Table', + 'NavigationBar', + 'Group', +]); + +const TEXT_ROLE = 'text'; + +/** Roles whose value is text the caller can write. */ +export function isTextEntryRole(role: string): boolean { + return role === 'textfield'; +} + +/** + * Project a snapshot into the candidate set a policy chooses between. + * + * A list cell that wraps an actionable leaf is dropped in favour of the leaf, so the policy never + * has to choose between a row and the button inside it. Static text stays as `text` context: it is + * what distinguishes "Enter the code" from "Welcome" when every button on both screens is generic. + */ +export function toPolicyCandidates(nodes: readonly PolicySnapshotNode[]): PolicyCandidate[] { + const childrenOf = new Map(); + for (const node of nodes) { + if (node.parentIndex === undefined) continue; + const siblings = childrenOf.get(node.parentIndex); + if (siblings) siblings.push(node); + else childrenOf.set(node.parentIndex, [node]); + } + + const candidates: PolicyCandidate[] = []; + for (const node of nodes) { + const type = node.type ?? ''; + if (!node.ref || STRUCTURAL_TYPES.has(type)) continue; + + let role: string; + if (LEAF_ROLES[type]) { + role = LEAF_ROLES[type]; + } else if (type === 'Cell') { + if (node.index !== undefined && hasActionableDescendant(childrenOf, node.index)) continue; + role = 'button'; + } else if (type === 'StaticText') { + role = TEXT_ROLE; + } else { + continue; + } + + candidates.push({ + ref: node.ref.startsWith('@') ? node.ref : `@${node.ref}`, + role, + name: node.label || node.identifier || type, + ...(node.identifier === undefined ? {} : { identifier: node.identifier }), + ...(node.value === undefined ? {} : { value: node.value }), + ...(node.enabled === false ? { disabled: true } : {}), + }); + } + return candidates; +} + +/** The subset a policy may choose: actionable, enabled, and not plain text. */ +export function selectableCandidates(candidates: readonly PolicyCandidate[]): PolicyCandidate[] { + return candidates.filter( + (candidate) => candidate.role !== TEXT_ROLE && candidate.disabled !== true, + ); +} + +/** + * A stable digest of what the screen shows, used to detect an action that changed nothing. + * Refs are excluded on purpose: they are reissued per snapshot generation, so including them would + * make every screen look changed. + */ +export function screenDigest(candidates: readonly PolicyCandidate[]): string { + return candidates + .map( + (candidate) => + `${candidate.role}:${candidate.name}:${candidate.value ?? ''}:${candidate.disabled ? 'd' : ''}`, + ) + .join('|'); +} + +/** A short label for logs and step records: the leading text plus the first few controls. */ +export function screenLabel(candidates: readonly PolicyCandidate[]): string { + const text = candidates.filter((candidate) => candidate.role === TEXT_ROLE).slice(0, 2); + const controls = selectableCandidates(candidates) + .filter((candidate) => candidate.role !== 'key') + .slice(0, 3); + const label = [...text, ...controls].map((candidate) => candidate.name).join(' | '); + return label.length > 0 ? label.slice(0, 80) : '(no labelled elements)'; +} + +/** The keypad key that enters `digit`, when a numeric keypad is on screen. */ +export function findKeypadDigit( + nodes: readonly PolicySnapshotNode[], + digit: string, +): PolicySnapshotNode | undefined { + return nodes.find((node) => node.type === 'Key' && node.label === digit && Boolean(node.ref)); +} + +function hasActionableDescendant( + childrenOf: ReadonlyMap, + index: number, +): boolean { + for (const child of childrenOf.get(index) ?? []) { + if (child.type && LEAF_ROLES[child.type]) return true; + if (child.index !== undefined && hasActionableDescendant(childrenOf, child.index)) return true; + } + return false; +} diff --git a/src/commands/policy/policy-contract.ts b/src/commands/policy/policy-contract.ts new file mode 100644 index 0000000000..9b54f3f24c --- /dev/null +++ b/src/commands/policy/policy-contract.ts @@ -0,0 +1,130 @@ +/** + * Types shared by the policy provider abstraction, the `suggest` command, and the `act` loop. + * A policy provider answers one question — which on-screen element advances a goal — and returns + * a typed decision with calibrated probabilities instead of free-form text. + */ + +/** One element a policy may choose between, projected from an interactive snapshot. */ +export type PolicyCandidate = { + /** Snapshot ref, including its leading `@`. */ + ref: string; + /** Neutral role name (`button`, `textfield`, `switch`, `link`, `key`, `text`). */ + role: string; + /** Accessibility label, identifier, or node type, in that order of preference. */ + name: string; + /** Accessibility identifier, when the node carries one. The stable key for input matching. */ + identifier?: string; + /** Current accessibility value, when the node reports one. */ + value?: string; + /** True when the node reports itself disabled; disabled nodes stay visible as context. */ + disabled?: boolean; +}; + +/** What the caller should do with the chosen element. */ +export type PolicyActionKind = 'press' | 'fill' | 'none'; + +/** The decision a provider returns for one screen. */ +export type PolicyDecision = { + action: PolicyActionKind; + /** Chosen ref, or null when the provider declined to choose. */ + target: string | null; + /** Provider believes the goal is already satisfied on this screen. */ + done: boolean; + /** Provider believes progress needs something not on this screen. */ + blocked: boolean; + /** Chosen target is an empty text field, so the caller must supply text. */ + needsText: boolean; + /** Provider-reported confidence in the chosen target, 0 to 1. */ + confidence: number; + /** Probability per candidate ref plus the decline label, 0 to 1. */ + probabilities: Record; + provider: string; + model: string; + /** Wall-clock milliseconds spent in the provider call. */ + decideMs: number; + inputTokens: number; + costUsd: number; +}; + +/** Why a step ended the way it did. */ +export type PolicyStepOutcome = + | 'acted' + | 'done' + | 'blocked' + | 'escalated' + | 'dead-action' + | 'failed'; + +/** What the loop did on one step, with the three phase timings kept apart. */ +export type PolicyActStep = { + step: number; + /** Short human label for the screen the decision was made on. */ + screen: string; + decision: PolicyDecision; + outcome: PolicyStepOutcome; + /** Present when the step performed a device action. */ + performed?: { + action: PolicyActionKind; + target: string; + /** How text reached the field; `keypad` is the digit-field fallback. */ + entry?: 'fill' | 'keypad'; + /** Input map key whose value was sent. The value itself is never recorded. */ + inputKey?: string; + }; + snapshotMs: number; + decideMs: number; + actionMs: number; + /** Set when the step could not proceed, in the loop's vocabulary. */ + reason?: string; +}; + +export type PolicyActStatus = 'done' | 'blocked' | 'escalated' | 'max-steps'; + +export type PolicyActTotals = { + steps: number; + snapshotMs: number; + decideMs: number; + actionMs: number; + inputTokens: number; + costUsd: number; + deadActions: number; + escalations: number; +}; + +export type PolicySuggestResult = { + goal: string; + decision: PolicyDecision; + candidates: PolicyCandidate[]; + snapshotMs: number; + screen: string; +}; + +export type PolicyActResult = { + goal: string; + status: PolicyActStatus; + steps: PolicyActStep[]; + totals: PolicyActTotals; +}; + +/** Everything a provider needs to judge one screen. */ +export type PolicyRequest = { + goal: string; + candidates: PolicyCandidate[]; + /** What the loop already did, oldest first; empty for a one-shot `suggest`. */ + history: string[]; + /** Short label for the current screen, used as context only. */ + screen: string; + /** + * Refs of fields the caller has text ready for. Values are never included: the policy only needs + * to know the text exists, and without that it reports a code screen as blocked even when the + * caller supplied the code. + */ + textReadyRefs: readonly string[]; +}; + +/** A policy head. One implementation ships today: `jev`. */ +export type PolicyProvider = { + readonly name: string; + readonly model: string; + decide: (request: PolicyRequest) => Promise; +}; diff --git a/src/commands/policy/text-inputs.test.ts b/src/commands/policy/text-inputs.test.ts new file mode 100644 index 0000000000..700bcbf1da --- /dev/null +++ b/src/commands/policy/text-inputs.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, test } from 'vitest'; +import { parsePolicyTextInputs, policyInputEnvName, resolveTextInput } from './text-inputs.ts'; +import type { PolicyCandidate } from './policy-contract.ts'; + +const phoneField: PolicyCandidate = { + ref: '@e5', + role: 'textfield', + name: 'Phone number', + identifier: 'phoneField', + value: '', +}; + +describe('policy text inputs', () => { + test('parses repeated key=value tokens, keeping a value that contains =', () => { + expect(parsePolicyTextInputs(['phone=5555550100', 'note=a=b'])).toEqual( + new Map([ + ['phone', '5555550100'], + ['note', 'a=b'], + ]), + ); + }); + + test('refuses a token that is not key=value', () => { + expect(() => parsePolicyTextInputs(['5555550100'])).toThrow( + expect.objectContaining({ + code: 'INVALID_ARGS', + message: expect.stringContaining('--input expects key=value'), + }), + ); + }); + + test('matches a key against the field identifier', () => { + expect(resolveTextInput(phoneField, parsePolicyTextInputs(['phone=5550100']), {})).toEqual({ + key: 'phone', + text: '5550100', + }); + }); + + test('prefers an exact match over a containing one', () => { + const inputs = parsePolicyTextInputs(['phone=containing', 'phonefield=exact']); + expect(resolveTextInput(phoneField, inputs, {})?.text).toBe('exact'); + }); + + test('reads a value from the environment so a secret stays out of argv', () => { + const env = { [policyInputEnvName('code')]: '482493' }; + const codeField: PolicyCandidate = { + ref: '@e7', + role: 'textfield', + name: 'Code', + identifier: 'codeField', + value: '', + }; + expect(resolveTextInput(codeField, new Map(), env)).toEqual({ key: 'code', text: '482493' }); + }); + + test('names the environment entry for a key', () => { + expect(policyInputEnvName('one-time code')).toBe('AGENT_DEVICE_INPUT_ONE_TIME_CODE'); + }); + + test('returns nothing when no entry matches the field', () => { + expect(resolveTextInput(phoneField, parsePolicyTextInputs(['code=1']), {})).toBeUndefined(); + }); +}); diff --git a/src/commands/policy/text-inputs.ts b/src/commands/policy/text-inputs.ts new file mode 100644 index 0000000000..a9a3f01d8c --- /dev/null +++ b/src/commands/policy/text-inputs.ts @@ -0,0 +1,69 @@ +import { AppError } from '@agent-device/kernel/errors'; +import type { PolicyCandidate } from './policy-contract.ts'; + +/** + * Text the loop may send to a field, keyed by a name the caller chooses. The loop never invents + * text: a field with no matching entry escalates instead of being filled with a guess. + */ +export type PolicyTextInputs = ReadonlyMap; + +/** Prefix for per-key environment entries, so a secret never has to appear in argv. */ +export const POLICY_INPUT_ENV_PREFIX = 'AGENT_DEVICE_INPUT_'; + +/** Parse repeated `--input key=value` tokens. An empty value is allowed; an empty key is not. */ +export function parsePolicyTextInputs(tokens: readonly string[]): Map { + const inputs = new Map(); + for (const token of tokens) { + const separator = token.indexOf('='); + const key = separator === -1 ? '' : token.slice(0, separator).trim(); + if (!key) { + throw new AppError('INVALID_ARGS', `--input expects key=value, received ${token}`); + } + inputs.set(key.toLowerCase(), token.slice(separator + 1)); + } + return inputs; +} + +/** The environment name that carries `key`, for callers who keep secrets out of argv. */ +export function policyInputEnvName(key: string): string { + return `${POLICY_INPUT_ENV_PREFIX}${key.toUpperCase().replaceAll(/[^A-Z0-9]/g, '_')}`; +} + +export type ResolvedTextInput = { key: string; text: string }; + +/** + * Find the text for a field. + * + * Matching runs from most to least specific — identifier, label, then either one containing the + * key — so `--input phone=...` reaches a field identified `phoneField` without also matching a + * "Phone support" link. Environment entries are consulted for the same keys, which is how an OTP + * or password reaches the loop without being written into a command line. + */ +export function resolveTextInput( + candidate: PolicyCandidate, + inputs: PolicyTextInputs, + env: Readonly>, +): ResolvedTextInput | undefined { + const identifier = candidate.identifier?.toLowerCase() ?? ''; + const name = candidate.name.toLowerCase(); + const keys = new Set([...inputs.keys(), ...environmentKeys(env)]); + + const matchers: Array<(key: string) => boolean> = [ + (key) => identifier === key || name === key, + (key) => identifier.includes(key) || name.includes(key), + ]; + for (const matches of matchers) { + for (const key of keys) { + if (!matches(key)) continue; + const text = inputs.get(key) ?? env[policyInputEnvName(key)]; + if (text !== undefined) return { key, text }; + } + } + return undefined; +} + +function environmentKeys(env: Readonly>): string[] { + return Object.keys(env) + .filter((name) => name.startsWith(POLICY_INPUT_ENV_PREFIX)) + .map((name) => name.slice(POLICY_INPUT_ENV_PREFIX.length).toLowerCase()); +} From 06b71f495a97cc16b1f8829db67b1bbfc4f9ddca Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 18:06:29 -0400 Subject: [PATCH 03/17] feat(policy): add the jev policy provider One provider behind a named abstraction, over global fetch with no new dependency. The credential comes from TYPESAFE_API_KEY in the environment only, so it never reaches argv, a recorded script, or an MCP tool schema. Every field of a response is validated against the candidate set rather than trusted, and an unreachable head fails with the status as a typed reason plus the documented fallback: the agent's own snapshot-and-choose loop. Co-Authored-By: Claude Opus 5 (1M context) --- src/commands/policy/jev-composer.test.ts | 154 ++++++++++++++++ src/commands/policy/jev-composer.ts | 172 ++++++++++++++++++ .../policy/jev-policy-provider.test.ts | 114 ++++++++++++ src/commands/policy/jev-policy-provider.ts | 121 ++++++++++++ src/commands/policy/policy-provider.ts | 60 ++++++ 5 files changed, 621 insertions(+) create mode 100644 src/commands/policy/jev-composer.test.ts create mode 100644 src/commands/policy/jev-composer.ts create mode 100644 src/commands/policy/jev-policy-provider.test.ts create mode 100644 src/commands/policy/jev-policy-provider.ts create mode 100644 src/commands/policy/policy-provider.ts diff --git a/src/commands/policy/jev-composer.test.ts b/src/commands/policy/jev-composer.test.ts new file mode 100644 index 0000000000..520a712f4c --- /dev/null +++ b/src/commands/policy/jev-composer.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, test } from 'vitest'; +import { buildJevRequest, DECLINE_LABEL, readJevDecision } from './jev-composer.ts'; +import type { PolicyCandidate, PolicyRequest } from './policy-contract.ts'; + +const candidates: PolicyCandidate[] = [ + { ref: '@e2', role: 'text', name: 'Enter the code we texted you' }, + { ref: '@e5', role: 'textfield', name: 'Code', identifier: 'codeField', value: '' }, + { ref: '@e6', role: 'button', name: 'Verify', disabled: true }, +]; + +const request: PolicyRequest = { + goal: 'sign in', + candidates, + history: ['pressed Send code'], + screen: 'Enter the code we texted you | Code', + textReadyRefs: ['@e5'], +}; + +const meta = { provider: 'jev', model: 'jev-latest', decideMs: 187 }; + +function answer(overrides: Record = {}): Record { + return { + model: 'jev-1.13.0', + answers: { + next: { + type: 'choice', + choice: '@e5', + confidence: 0.97, + probabilities: { '@e5': 0.97, [DECLINE_LABEL]: 0.03 }, + }, + done: { type: 'noul', noul: 0.02 }, + blocked: { type: 'noul', noul: 0.11 }, + }, + usage: { input_tokens: 500, output_tokens: 64 }, + ...overrides, + }; +} + +describe('jev request composition', () => { + test('offers every enabled element plus a decline label as choices', () => { + const body = buildJevRequest('jev-latest', request); + const criteria = (body.questions.next as { criteria: Record }).criteria; + expect(Object.keys(criteria).sort()).toEqual(['@e5', DECLINE_LABEL]); + }); + + test('sends the full element list as state, so text stays available as context', () => { + const body = buildJevRequest('jev-latest', request); + expect(body.state.elements).toEqual(candidates); + expect(body.state.history).toEqual(['pressed Send code']); + }); + + test('marks a field the caller has text for, without carrying the text', () => { + const body = buildJevRequest('jev-latest', request); + const criteria = (body.questions.next as { criteria: Record }).criteria; + expect(criteria['@e5']).toMatchObject({ text_ready_to_enter: true }); + expect(JSON.stringify(body)).not.toContain('482493'); + }); + + test('refuses a screen with nothing actionable on it', () => { + expect(() => + buildJevRequest('jev-latest', { ...request, candidates: candidates.slice(0, 1) }), + ).toThrow( + expect.objectContaining({ + code: 'COMMAND_FAILED', + message: expect.stringContaining('No actionable element'), + }), + ); + }); +}); + +describe('jev decision parsing', () => { + test('reads a decision from a well-formed response', () => { + expect(readJevDecision(answer(), request, meta)).toEqual({ + action: 'fill', + target: '@e5', + done: false, + blocked: false, + needsText: true, + confidence: 0.97, + probabilities: { '@e5': 0.97, [DECLINE_LABEL]: 0.03 }, + provider: 'jev', + model: 'jev-1.13.0', + decideMs: 187, + inputTokens: 500, + costUsd: 500 * (42 / 1e9), + }); + }); + + test('reports a press when the chosen field already holds a value', () => { + const filled: PolicyRequest = { + ...request, + candidates: candidates.map((candidate) => + candidate.ref === '@e5' ? { ...candidate, value: '4821' } : candidate, + ), + }; + const decision = readJevDecision(answer(), filled, meta); + expect(decision.action).toBe('press'); + expect(decision.needsText).toBe(false); + }); + + test('reads a decline as no target', () => { + const declined = answer({ + answers: { + next: { + type: 'choice', + choice: DECLINE_LABEL, + confidence: 0.6, + probabilities: { [DECLINE_LABEL]: 0.6, '@e5': 0.4 }, + }, + done: { type: 'noul', noul: 0.1 }, + blocked: { type: 'noul', noul: 0.8 }, + }, + }); + const decision = readJevDecision(declined, request, meta); + expect(decision.target).toBeNull(); + expect(decision.action).toBe('none'); + expect(decision.blocked).toBe(true); + }); + + test('rejects a choice that is not one of the offered elements', () => { + const invented = answer({ + answers: { + next: { type: 'choice', choice: '@e99', confidence: 1, probabilities: { '@e99': 1 } }, + done: { type: 'noul', noul: 0 }, + blocked: { type: 'noul', noul: 0 }, + }, + }); + expect(() => readJevDecision(invented, request, meta)).toThrow( + expect.objectContaining({ + code: 'COMMAND_FAILED', + message: expect.stringContaining('not a candidate element'), + }), + ); + }); + + test('rejects a confidence outside zero to one', () => { + const broken = answer({ + answers: { + next: { type: 'choice', choice: '@e5', confidence: 4, probabilities: { '@e5': 1 } }, + done: { type: 'noul', noul: 0 }, + blocked: { type: 'noul', noul: 0 }, + }, + }); + expect(() => readJevDecision(broken, request, meta)).toThrow( + expect.objectContaining({ code: 'COMMAND_FAILED' }), + ); + }); + + test('rejects a response with no answers at all', () => { + expect(() => readJevDecision({ usage: {} }, request, meta)).toThrow( + expect.objectContaining({ message: expect.stringContaining('answers is missing') }), + ); + }); +}); diff --git a/src/commands/policy/jev-composer.ts b/src/commands/policy/jev-composer.ts new file mode 100644 index 0000000000..8300848d7f --- /dev/null +++ b/src/commands/policy/jev-composer.ts @@ -0,0 +1,172 @@ +import { AppError } from '@agent-device/kernel/errors'; +import { isTextEntryRole, selectableCandidates } from './candidate-elements.ts'; +import type { PolicyCandidate, PolicyDecision, PolicyRequest } from './policy-contract.ts'; + +/** + * The label a choice question carries for "no element on this screen advances the goal". A ref can + * never collide with it: refs are `@`-prefixed. + */ +export const DECLINE_LABEL = '__none__'; + +/** + * Published input-token price for the TypeSafe System One API, in US dollars. Output tokens are + * not billed. A wrong constant only mis-reports the `costUsd` field; it never changes behaviour. + */ +export const JEV_USD_PER_INPUT_TOKEN = 42 / 1e9; + +const CHOICE_RULES = [ + 'Fill an empty required field before pressing the control that submits it', + 'Never choose a disabled element', + 'Prefer the most direct remaining step towards the goal', + 'Do not repeat an action the history shows already succeeded', + 'A field marked text_ready_to_enter can be filled immediately; its text is already available', +] as const; + +export type JevRequestBody = { + model: string; + state: Record; + questions: Record; +}; + +/** Compose the System One request for one screen. Pure: no clock, no network, no environment. */ +export function buildJevRequest(model: string, request: PolicyRequest): JevRequestBody { + const selectable = selectableCandidates(request.candidates); + if (selectable.length === 0) { + throw new AppError('COMMAND_FAILED', 'No actionable element is visible on the current screen', { + reason: 'policy-no-candidates', + hint: 'Take a snapshot to confirm the screen rendered, then retry.', + }); + } + + const criteria: Record = { + [DECLINE_LABEL]: 'No visible element advances the goal; stop and hand back to the caller', + }; + const textReady = new Set(request.textReadyRefs); + for (const candidate of selectable) { + criteria[candidate.ref] = { + role: candidate.role, + name: candidate.name, + ...(candidate.value === undefined ? {} : { value: candidate.value }), + // The caller holds text for this field, so needing it is not a reason to report blocked. + ...(textReady.has(candidate.ref) ? { text_ready_to_enter: true } : {}), + }; + } + + return { + model, + state: { + goal: request.goal, + screen: request.screen, + elements: request.candidates, + history: request.history, + fields_with_text_ready: request.textReadyRefs, + }, + questions: { + next: { + type: 'choice', + instructions: { + question: 'Which element should be acted on next to advance the goal', + rules: CHOICE_RULES, + }, + criteria, + }, + done: { + type: 'noul', + instructions: 'The goal is already fully achieved on the current screen', + }, + blocked: { + type: 'noul', + instructions: + 'Progress needs information or a credential that is neither on this screen, nor already available for a field marked text_ready_to_enter, nor obtainable by acting on this screen', + }, + }, + }; +} + +/** + * Read a System One response into a typed decision. + * + * Every field is validated against the candidate set rather than trusted: a label outside the + * criteria, or a probability that is not a number, is a provider fault and is reported as one. + */ +export function readJevDecision( + body: unknown, + request: PolicyRequest, + meta: { provider: string; model: string; decideMs: number }, +): PolicyDecision { + const answers = readRecord(readRecord(body, 'response body').answers, 'answers'); + const next = readRecord(answers.next, 'answers.next'); + const target = readChoice(next.choice, request.candidates); + + const chosen = + target === null ? undefined : request.candidates.find((candidate) => candidate.ref === target); + const needsText = chosen !== undefined && isTextEntryRole(chosen.role) && !chosen.value; + + const usage = isRecord(body) && isRecord(body.usage) ? body.usage : {}; + const inputTokens = typeof usage.input_tokens === 'number' ? usage.input_tokens : 0; + + return { + action: target === null ? 'none' : needsText ? 'fill' : 'press', + target, + done: readNoul(answers.done, 'answers.done') > 0.5, + blocked: readNoul(answers.blocked, 'answers.blocked') > 0.5, + needsText, + confidence: readUnitInterval(next.confidence, 'answers.next.confidence'), + probabilities: readProbabilities(next.probabilities), + provider: meta.provider, + model: + typeof (body as { model?: unknown }).model === 'string' + ? (body as { model: string }).model + : meta.model, + decideMs: meta.decideMs, + inputTokens, + costUsd: inputTokens * JEV_USD_PER_INPUT_TOKEN, + }; +} + +function readChoice(value: unknown, candidates: readonly PolicyCandidate[]): string | null { + if (typeof value !== 'string') { + throw providerFault('answers.next.choice is not a string'); + } + if (value === DECLINE_LABEL) return null; + if (!candidates.some((candidate) => candidate.ref === value)) { + throw providerFault(`answers.next.choice named ${value}, which is not a candidate element`); + } + return value; +} + +function readProbabilities(value: unknown): Record { + if (!isRecord(value)) throw providerFault('answers.next.probabilities is not an object'); + const probabilities: Record = {}; + for (const [label, probability] of Object.entries(value)) { + probabilities[label] = readUnitInterval(probability, `probability for ${label}`); + } + return probabilities; +} + +function readNoul(value: unknown, field: string): number { + return readUnitInterval(readRecord(value, field).noul, `${field}.noul`); +} + +function readUnitInterval(value: unknown, field: string): number { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1) { + throw providerFault(`${field} is not a probability between 0 and 1`); + } + return value; +} + +function readRecord(value: unknown, field: string): Record { + if (!isRecord(value)) throw providerFault(`${field} is missing or not an object`); + return value; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function providerFault(message: string): AppError { + return new AppError('COMMAND_FAILED', `Jev returned an unusable decision: ${message}`, { + reason: 'policy-provider-response', + hint: 'Fall back to agent-driven policy: snapshot, choose an element yourself, then press or fill it.', + }); +} diff --git a/src/commands/policy/jev-policy-provider.test.ts b/src/commands/policy/jev-policy-provider.test.ts new file mode 100644 index 0000000000..350370e433 --- /dev/null +++ b/src/commands/policy/jev-policy-provider.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, test } from 'vitest'; +import { createJevPolicyProvider } from './jev-policy-provider.ts'; +import type { PolicyRequest } from './policy-contract.ts'; + +const request: PolicyRequest = { + goal: 'sign in', + candidates: [ + { ref: '@e5', role: 'textfield', name: 'Phone number', identifier: 'phoneField', value: '' }, + { ref: '@e6', role: 'button', name: 'Send code' }, + ], + history: [], + screen: 'Phone number | Send code', + textReadyRefs: [], +}; + +const body = { + model: 'jev-1.13.0', + answers: { + next: { + type: 'choice', + choice: '@e5', + confidence: 0.99, + probabilities: { '@e5': 0.99, '@e6': 0.01 }, + }, + done: { type: 'noul', noul: 0.01 }, + blocked: { type: 'noul', noul: 0.9 }, + }, + usage: { input_tokens: 412 }, +}; + +function respondWith(status: number, payload: unknown): typeof globalThis.fetch { + return (async () => + new Response(typeof payload === 'string' ? payload : JSON.stringify(payload), { + status, + headers: { 'content-type': 'application/json' }, + })) as unknown as typeof globalThis.fetch; +} + +describe('jev policy provider', () => { + test('sends the key as a bearer token and returns a typed decision', async () => { + let seen: { url: string; init: RequestInit } | undefined; + const provider = createJevPolicyProvider({ + apiKey: 'test-key', + now: () => 0, + fetch: (async (url: string, init: RequestInit) => { + seen = { url, init }; + return new Response(JSON.stringify(body), { status: 200 }); + }) as unknown as typeof globalThis.fetch, + }); + + const decision = await provider.decide(request); + + expect(seen?.url).toBe('https://api.typesafe.ai/v1/systemone'); + const headers = seen?.init.headers as Record | undefined; + expect(headers?.authorization).toBe('Bearer test-key'); + expect(decision.target).toBe('@e5'); + expect(decision.action).toBe('fill'); + expect(decision.blocked).toBe(true); + expect(decision.inputTokens).toBe(412); + }); + + test('reports a rejected key as unauthorized, naming the fallback', async () => { + const provider = createJevPolicyProvider({ + apiKey: 'bad', + fetch: respondWith(401, { error: 'invalid key' }), + }); + await expect(provider.decide(request)).rejects.toThrow( + expect.objectContaining({ + code: 'UNAUTHORIZED', + message: expect.stringContaining('Jev unavailable: 401'), + }), + ); + }); + + test('marks a rate limit retriable and a payment failure not', async () => { + const limited = createJevPolicyProvider({ apiKey: 'k', fetch: respondWith(429, 'slow down') }); + await expect(limited.decide(request)).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'policy-provider-rate-limited', retriable: true, status: 429 }, + }); + + const unpaid = createJevPolicyProvider({ apiKey: 'k', fetch: respondWith(402, 'no credit') }); + await expect(unpaid.decide(request)).rejects.toMatchObject({ + details: { reason: 'policy-provider-payment-required', retriable: false, status: 402 }, + }); + }); + + test('reports a transport failure as retriable rather than as a decision', async () => { + const provider = createJevPolicyProvider({ + apiKey: 'k', + fetch: (async () => { + throw new Error('socket hang up'); + }) as unknown as typeof globalThis.fetch, + }); + await expect(provider.decide(request)).rejects.toMatchObject({ + message: 'Jev unavailable: socket hang up', + details: { reason: 'policy-provider-transport', retriable: true }, + }); + }); + + test('measures decide time from the injected clock', async () => { + let clock = 1_000; + const provider = createJevPolicyProvider({ + apiKey: 'k', + fetch: respondWith(200, body), + now: () => { + const value = clock; + clock += 203; + return value; + }, + }); + expect((await provider.decide(request)).decideMs).toBe(203); + }); +}); diff --git a/src/commands/policy/jev-policy-provider.ts b/src/commands/policy/jev-policy-provider.ts new file mode 100644 index 0000000000..f85d4b9caa --- /dev/null +++ b/src/commands/policy/jev-policy-provider.ts @@ -0,0 +1,121 @@ +import { AppError } from '@agent-device/kernel/errors'; +import { buildJevRequest, readJevDecision } from './jev-composer.ts'; +import type { PolicyDecision, PolicyProvider, PolicyRequest } from './policy-contract.ts'; + +export const JEV_PROVIDER_NAME = 'jev'; +export const JEV_DEFAULT_ENDPOINT = 'https://api.typesafe.ai/v1/systemone'; +export const JEV_DEFAULT_MODEL = 'jev-latest'; +const JEV_TIMEOUT_MS = 20_000; + +/** + * The documented recovery when the policy head is unreachable. It names the agent's own loop + * because that loop is always available: the policy head is an accelerator, never a dependency. + */ +export const POLICY_FALLBACK_HINT = + 'Fall back to agent-driven policy: snapshot, choose an element yourself, then press or fill it.'; + +export type JevProviderOptions = { + apiKey: string; + model?: string; + endpoint?: string; + /** Injected so unit tests exercise the real composer and parser against a fake transport. */ + fetch?: typeof globalThis.fetch; + /** Injected so step timings are assertable. */ + now?: () => number; +}; + +export function createJevPolicyProvider(options: JevProviderOptions): PolicyProvider { + const model = options.model ?? JEV_DEFAULT_MODEL; + const endpoint = options.endpoint ?? JEV_DEFAULT_ENDPOINT; + const fetchImpl = options.fetch ?? globalThis.fetch; + const now = options.now ?? (() => Date.now()); + + return { + name: JEV_PROVIDER_NAME, + model, + decide: async (request: PolicyRequest): Promise => { + const body = buildJevRequest(model, request); + const startedAt = now(); + const response = await callJev(fetchImpl, endpoint, options.apiKey, body); + if (!response.ok) throw unavailable(response.status, await readErrorText(response)); + const parsed: unknown = await response.json(); + return readJevDecision(parsed, request, { + provider: JEV_PROVIDER_NAME, + model, + decideMs: now() - startedAt, + }); + }, + }; +} + +async function callJev( + fetchImpl: typeof globalThis.fetch, + endpoint: string, + apiKey: string, + body: unknown, +): Promise { + try { + return await fetchImpl(endpoint, { + method: 'POST', + headers: { + authorization: `Bearer ${apiKey}`, + 'content-type': 'application/json', + accept: 'application/json', + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(JEV_TIMEOUT_MS), + }); + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Jev unavailable: ${error instanceof Error ? error.message : 'request failed'}`, + { reason: 'policy-provider-transport', retriable: true, hint: POLICY_FALLBACK_HINT }, + error, + ); + } +} + +/** + * Status is carried as a typed `reason` and `status` detail rather than being sniffed back out of + * the message, so a caller can branch on it without matching text. + */ +function unavailable(status: number, detail: string): AppError { + const code = status === 401 || status === 403 ? 'UNAUTHORIZED' : 'COMMAND_FAILED'; + return new AppError( + code, + `Jev unavailable: ${status}${detail ? ` ${detail}` : ''}; fall back to agent-driven policy`, + { + reason: reasonForStatus(status), + status, + retriable: status === 429 || status >= 500, + hint: hintForStatus(status), + }, + ); +} + +function reasonForStatus(status: number): string { + if (status === 401 || status === 403) return 'policy-provider-unauthorized'; + if (status === 402) return 'policy-provider-payment-required'; + if (status === 429) return 'policy-provider-rate-limited'; + return 'policy-provider-http'; +} + +function hintForStatus(status: number): string { + if (status === 401 || status === 403) { + return `TYPESAFE_API_KEY is missing or rejected. ${POLICY_FALLBACK_HINT}`; + } + if (status === 402) return `The TypeSafe account has no credit. ${POLICY_FALLBACK_HINT}`; + if (status === 429) { + return `Rate limited; keep policy concurrency at or below 4. ${POLICY_FALLBACK_HINT}`; + } + return POLICY_FALLBACK_HINT; +} + +async function readErrorText(response: Response): Promise { + try { + const text = await response.text(); + return text.slice(0, 200).replaceAll(/\s+/gu, ' ').trim(); + } catch { + return ''; + } +} diff --git a/src/commands/policy/policy-provider.ts b/src/commands/policy/policy-provider.ts new file mode 100644 index 0000000000..59bce6fe5b --- /dev/null +++ b/src/commands/policy/policy-provider.ts @@ -0,0 +1,60 @@ +import { AppError } from '@agent-device/kernel/errors'; +import { + createJevPolicyProvider, + JEV_PROVIDER_NAME, + POLICY_FALLBACK_HINT, +} from './jev-policy-provider.ts'; +import type { PolicyProvider } from './policy-contract.ts'; + +/** Every policy head the CLI can resolve. One ships today. */ +export const POLICY_PROVIDER_NAMES = [JEV_PROVIDER_NAME] as const; +export type PolicyProviderName = (typeof POLICY_PROVIDER_NAMES)[number]; +export const DEFAULT_POLICY_PROVIDER: PolicyProviderName = JEV_PROVIDER_NAME; + +/** Environment keys the jev provider reads. The key is never accepted as a CLI flag. */ +export const JEV_API_KEY_ENV = 'TYPESAFE_API_KEY'; +export const JEV_MODEL_ENV = 'AGENT_DEVICE_POLICY_MODEL'; +export const JEV_ENDPOINT_ENV = 'AGENT_DEVICE_POLICY_ENDPOINT'; + +export type PolicyProviderEnvironment = Readonly>; + +export function isPolicyProviderName(value: string): value is PolicyProviderName { + return (POLICY_PROVIDER_NAMES as readonly string[]).includes(value); +} + +/** + * Resolve a provider by name. + * + * The credential comes from the environment only, so it never reaches argv, a recorded script, a + * session journal, or an MCP tool schema. Without it the command refuses before touching the + * device, which is what keeps the feature invisible to everyone who has not opted in. + */ +export function createPolicyProvider( + name: string, + env: PolicyProviderEnvironment, + overrides: { fetch?: typeof globalThis.fetch } = {}, +): PolicyProvider { + if (!isPolicyProviderName(name)) { + throw new AppError( + 'INVALID_ARGS', + `Unknown policy provider ${name}; available: ${POLICY_PROVIDER_NAMES.join(', ')}`, + ); + } + const apiKey = env[JEV_API_KEY_ENV]?.trim(); + if (!apiKey) { + throw new AppError( + 'INVALID_ARGS', + `${JEV_API_KEY_ENV} is not set, so policy ${name} cannot decide`, + { + reason: 'policy-provider-unconfigured', + hint: `Export ${JEV_API_KEY_ENV} to enable the policy head. Without it, ${POLICY_FALLBACK_HINT}`, + }, + ); + } + return createJevPolicyProvider({ + apiKey, + model: env[JEV_MODEL_ENV], + endpoint: env[JEV_ENDPOINT_ENV], + ...overrides, + }); +} From 98114b43fdd5ee8271f085422d4897bcc1552a6b Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 18:06:42 -0400 Subject: [PATCH 04/17] feat(policy): add the act loop Snapshot, decide, act, verify, repeat. Three device operations behind a narrow port so the loop is testable without a device. What the loop had to learn from a real iOS sign-in flow: - A settled observation only proves the local UI went quiet, so a transition waiting on the network still reads as the old screen. One grace re-read separates a slow transition from an action that did nothing. - A masked field reformats what it receives, so the runner reports TEXT_ENTRY_MISMATCH on text that arrived. The screen decides, not the write. - A digit-box code field never reflects its value at all. A confirmed value is sufficient evidence of a write but not necessary; a screen that moved on is the other half, and the on-screen keypad is the fallback when neither holds. - A press on a text field only focuses it, so the supplied entry rather than the field's emptiness decides whether to write. - A head asked whether progress is blocked says yes on any sign-in screen. A concrete target at 0.9 confidence outranks that flag, or the loop stalls on step one of every sign-in. Co-Authored-By: Claude Opus 5 (1M context) --- src/commands/policy/act-loop.test.ts | 467 +++++++++++++++++++++++++++ src/commands/policy/act-loop.ts | 369 +++++++++++++++++++++ 2 files changed, 836 insertions(+) create mode 100644 src/commands/policy/act-loop.test.ts create mode 100644 src/commands/policy/act-loop.ts diff --git a/src/commands/policy/act-loop.test.ts b/src/commands/policy/act-loop.test.ts new file mode 100644 index 0000000000..1ef54ffdfb --- /dev/null +++ b/src/commands/policy/act-loop.test.ts @@ -0,0 +1,467 @@ +import { describe, expect, test } from 'vitest'; +import { runPolicyActLoop, type PolicyDevicePort } from './act-loop.ts'; +import type { PolicySnapshotNode } from './candidate-elements.ts'; +import type { PolicyDecision, PolicyProvider, PolicyRequest } from './policy-contract.ts'; + +function decision(overrides: Partial = {}): PolicyDecision { + return { + action: 'press', + target: '@e6', + done: false, + blocked: false, + needsText: false, + confidence: 0.95, + probabilities: { '@e6': 0.95 }, + provider: 'stub', + model: 'stub-1', + decideMs: 200, + inputTokens: 400, + costUsd: 0.0000168, + ...overrides, + }; +} + +function providerOf(...decisions: PolicyDecision[]): PolicyProvider & { seen: PolicyRequest[] } { + const seen: PolicyRequest[] = []; + let index = 0; + return { + name: 'stub', + model: 'stub-1', + seen, + decide: async (request) => { + seen.push(request); + return decisions[Math.min(index++, decisions.length - 1)] as PolicyDecision; + }, + }; +} + +function screenOf(...nodes: PolicySnapshotNode[]): PolicySnapshotNode[] { + return nodes; +} + +const welcome = screenOf( + { index: 0, type: 'StaticText', label: 'Welcome', ref: 'e2' }, + { index: 1, type: 'Button', label: 'Send code', ref: 'e6' }, +); +const codeScreen = screenOf( + { index: 0, type: 'StaticText', label: 'Enter the code', ref: 'e2' }, + { index: 1, type: 'TextField', label: 'Code', identifier: 'codeField', value: '', ref: 'e7' }, +); + +/** A device that returns each queued screen in turn and records every call. */ +function deviceOf(screens: PolicySnapshotNode[][]): PolicyDevicePort & { calls: string[] } { + const calls: string[] = []; + let index = 0; + return { + calls, + snapshot: async () => { + const nodes = screens[Math.min(index++, screens.length - 1)] as PolicySnapshotNode[]; + return { nodes, ms: 400 }; + }, + press: async (ref) => { + calls.push(`press ${ref}`); + return 900; + }, + fill: async (ref, text) => { + calls.push(`fill ${ref} ${text}`); + return 700; + }, + }; +} + +const baseOptions = { + goal: 'sign in', + inputs: new Map(), + env: {} as Record, + maxSteps: 6, + minConfidence: 0.4, + sleep: async () => {}, +}; + +describe('policy act loop', () => { + test('stops as done when the policy reports the goal achieved', async () => { + const result = await runPolicyActLoop({ + ...baseOptions, + provider: providerOf(decision({ done: true })), + device: deviceOf([welcome]), + }); + expect(result.status).toBe('done'); + expect(result.steps).toHaveLength(1); + expect(result.steps[0]?.outcome).toBe('done'); + }); + + test('presses the chosen element and records the phase timings apart', async () => { + const device = deviceOf([welcome, codeScreen]); + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + provider: providerOf(decision()), + device, + }); + expect(device.calls).toEqual(['press @e6']); + expect(result.steps[0]).toMatchObject({ + outcome: 'acted', + snapshotMs: 400, + decideMs: 200, + actionMs: 900, + performed: { action: 'press', target: '@e6' }, + }); + expect(result.totals.costUsd).toBeCloseTo(0.0000168, 9); + }); + + test('re-reads once before filing an action as dead, so a slow transition still counts', async () => { + // Two unchanged reads, then the transition lands: immediate read stale, grace read fresh. + const device = deviceOf([welcome, welcome, codeScreen]); + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + provider: providerOf(decision()), + device, + }); + expect(result.steps[0]?.outcome).toBe('acted'); + }); + + test('records an action that left the screen unchanged as a dead action', async () => { + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + provider: providerOf(decision()), + device: deviceOf([welcome]), + }); + expect(result.steps[0]?.outcome).toBe('dead-action'); + expect(result.steps[0]?.reason).toBe('screen unchanged after press'); + expect(result.totals.deadActions).toBe(1); + }); + + test('ends as escalated after three unproductive steps in a row', async () => { + const result = await runPolicyActLoop({ + ...baseOptions, + provider: providerOf(decision()), + device: deviceOf([welcome]), + }); + expect(result.status).toBe('escalated'); + expect(result.steps).toHaveLength(3); + }); + + test('escalates rather than acting below the confidence floor', async () => { + const result = await runPolicyActLoop({ + ...baseOptions, + minConfidence: 0.8, + provider: providerOf(decision({ confidence: 0.3 })), + device: deviceOf([welcome]), + }); + expect(result.status).toBe('escalated'); + expect(result.steps[0]?.reason).toContain('below --min-confidence'); + }); + + test('acts through a blocked flag only when confidence is very high', async () => { + const acting = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + provider: providerOf(decision({ blocked: true, confidence: 0.99 })), + device: deviceOf([welcome, codeScreen]), + }); + expect(acting.steps[0]?.outcome).toBe('acted'); + + const halted = await runPolicyActLoop({ + ...baseOptions, + provider: providerOf(decision({ blocked: true, confidence: 0.7 })), + device: deviceOf([welcome]), + }); + expect(halted.status).toBe('blocked'); + }); + + test('never invents text: a field with no supplied value escalates', async () => { + const device = deviceOf([codeScreen]); + const result = await runPolicyActLoop({ + ...baseOptions, + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device, + }); + expect(device.calls).toEqual([]); + expect(result.status).toBe('escalated'); + expect(result.steps[0]?.reason).toBe('no text supplied for codeField'); + }); + + test('fills from an input entry matched to the field identifier', async () => { + const device = deviceOf([codeScreen, welcome]); + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', 'abcdef']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device, + }); + expect(device.calls).toEqual(['fill @e7 abcdef']); + expect(result.steps[0]?.performed).toEqual({ + action: 'fill', + target: '@e7', + entry: 'fill', + inputKey: 'code', + }); + }); + + test('records the input key but never the text it sent', async () => { + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', '482493']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device: deviceOf([codeScreen, welcome]), + }); + expect(JSON.stringify(result)).not.toContain('482493'); + }); + + test('enters digits on the keypad when fill leaves the field value unchanged', async () => { + const keypad: PolicySnapshotNode[] = [ + ...codeScreen, + { index: 9, type: 'Key', label: '4', ref: 'e30' }, + { index: 10, type: 'Key', label: '8', ref: 'e31' }, + ]; + const device = deviceOf([keypad]); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', '48']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device, + }); + expect(device.calls).toEqual(['fill @e7 48', 'press @e30', 'press @e31']); + }); + + test('keeps a fill that did land off the keypad path', async () => { + const landed: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: 'Enter the code', ref: 'e2' }, + { + index: 1, + type: 'TextField', + label: 'Code', + identifier: 'codeField', + value: '482493', + ref: 'e7', + }, + { index: 9, type: 'Key', label: '4', ref: 'e30' }, + ]; + const device = deviceOf([codeScreen, landed]); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', '482493']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device, + }); + expect(device.calls).toEqual(['fill @e7 482493']); + }); + + test('rewrites a field whose value is not the supplied text', async () => { + const prefilled: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: 'Welcome', ref: 'e2' }, + { + index: 1, + type: 'TextField', + label: 'Phone number', + identifier: 'phoneField', + value: '(555) 555-0100', + ref: 'e7', + }, + ]; + const device = deviceOf([prefilled, welcome]); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['phone', '5555550700']]), + provider: providerOf(decision({ action: 'press', target: '@e7' })), + device, + }); + expect(device.calls).toEqual(['fill @e7 5555550700']); + }); + + test('presses rather than rewrites a field that already holds the supplied text', async () => { + const satisfied: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: 'Welcome', ref: 'e2' }, + { + index: 1, + type: 'TextField', + label: 'Phone number', + identifier: 'phoneField', + value: '(555) 555-0700', + ref: 'e7', + }, + ]; + const device = deviceOf([satisfied, welcome]); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['phone', '5555550700']]), + provider: providerOf(decision({ action: 'press', target: '@e7' })), + device, + }); + expect(device.calls).toEqual(['press @e7']); + }); + + test('presses a field that already holds text when the caller supplied none', async () => { + const prefilled: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: 'Welcome', ref: 'e2' }, + { + index: 1, + type: 'TextField', + label: 'Phone number', + identifier: 'phoneField', + value: '(555) 555-0100', + ref: 'e7', + }, + ]; + const device = deviceOf([prefilled, welcome]); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + provider: providerOf(decision({ action: 'press', target: '@e7' })), + device, + }); + expect(device.calls).toEqual(['press @e7']); + }); + + test('reaches a digit-box widget through the keypad even when the write itself failed', async () => { + const boxes: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: '0 of 2 entered', ref: 'e2' }, + { index: 1, type: 'TextField', label: 'Code', identifier: 'codeField', value: '', ref: 'e7' }, + { index: 9, type: 'Key', label: '4', ref: 'e30' }, + { index: 10, type: 'Key', label: '8', ref: 'e31' }, + ]; + const device = deviceOf([boxes]); + const failing: PolicyDevicePort = { + ...device, + fill: async () => { + throw Object.assign(new Error('runner recorded a failure'), { + code: 'XCTEST_RECORDED_FAILURE', + }); + }, + }; + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', '48']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device: failing, + }); + expect(device.calls).toEqual(['press @e30', 'press @e31']); + expect(result.steps[0]?.performed?.entry).toBe('keypad'); + expect(result.steps[0]?.outcome).toBe('acted'); + }); + + test('accepts a hidden-value widget when the screen moved on after the write', async () => { + // A digit-box code field never reflects its value; the next screen is the only evidence. + const device = deviceOf([codeScreen, codeScreen, welcome]); + const failing: PolicyDevicePort = { + ...device, + fill: async () => { + throw Object.assign(new Error('recorded failure'), { code: 'XCTEST_RECORDED_FAILURE' }); + }, + }; + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', 'abcdef']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device: failing, + }); + expect(result.steps[0]?.outcome).toBe('acted'); + }); + + test('carries the write failure code when neither the write nor the keypad delivered', async () => { + const device = deviceOf([codeScreen]); + const failing: PolicyDevicePort = { + ...device, + fill: async () => { + throw Object.assign(new Error('mismatch'), { code: 'TEXT_ENTRY_MISMATCH' }); + }, + }; + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', '48']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device: failing, + }); + expect(result.steps[0]?.outcome).toBe('dead-action'); + expect(result.steps[0]?.reason).toContain('TEXT_ENTRY_MISMATCH'); + }); + + test('accepts a masked field that reformatted the supplied text', async () => { + const masked: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: 'Welcome', ref: 'e2' }, + { + index: 1, + type: 'TextField', + label: 'Phone number', + identifier: 'phoneField', + value: '(500) 555-0700', + ref: 'e7', + }, + ]; + const unformatted: PolicySnapshotNode[] = [ + { index: 0, type: 'StaticText', label: 'Welcome', ref: 'e2' }, + { + index: 1, + type: 'TextField', + label: 'Phone number', + identifier: 'phoneField', + value: '', + ref: 'e7', + }, + ]; + const device = deviceOf([unformatted, masked]); + const failing: PolicyDevicePort = { + ...device, + fill: async (ref, text) => { + device.calls.push(`fill ${ref} ${text}`); + throw Object.assign(new Error('mismatch'), { code: 'TEXT_ENTRY_MISMATCH' }); + }, + }; + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['phone', '5005550700']]), + provider: providerOf(decision({ action: 'fill', target: '@e7', needsText: true })), + device: failing, + }); + expect(device.calls).toEqual(['fill @e7 5005550700']); + expect(result.steps[0]?.outcome).toBe('acted'); + expect(result.steps[0]?.performed?.entry).toBe('fill'); + }); + + test('tells the policy which fields it already holds text for', async () => { + const provider = providerOf(decision({ target: '@e7', action: 'fill' })); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 1, + inputs: new Map([['code', 'abc']]), + provider, + device: deviceOf([codeScreen, welcome]), + }); + expect(provider.seen[0]?.textReadyRefs).toEqual(['@e7']); + }); + + test('gives the policy the history of what it already did', async () => { + const provider = providerOf(decision()); + await runPolicyActLoop({ + ...baseOptions, + maxSteps: 2, + provider, + device: deviceOf([welcome, codeScreen, welcome, codeScreen]), + }); + expect(provider.seen[0]?.history).toEqual([]); + expect(provider.seen[1]?.history).toEqual(['pressed Send code']); + }); + + test('stops at the step budget', async () => { + const result = await runPolicyActLoop({ + ...baseOptions, + maxSteps: 2, + provider: providerOf(decision()), + device: deviceOf([welcome, codeScreen, welcome, codeScreen]), + }); + expect(result.status).toBe('max-steps'); + expect(result.totals.steps).toBe(2); + }); +}); diff --git a/src/commands/policy/act-loop.ts b/src/commands/policy/act-loop.ts new file mode 100644 index 0000000000..c081e2e70d --- /dev/null +++ b/src/commands/policy/act-loop.ts @@ -0,0 +1,369 @@ +import { + findKeypadDigit, + isTextEntryRole, + screenDigest, + screenLabel, + toPolicyCandidates, + type PolicySnapshotNode, +} from './candidate-elements.ts'; +import type { + PolicyActResult, + PolicyActStep, + PolicyActStatus, + PolicyCandidate, + PolicyDecision, + PolicyProvider, +} from './policy-contract.ts'; +import { resolveTextInput, type PolicyTextInputs, type ResolvedTextInput } from './text-inputs.ts'; + +/** The three device operations the loop needs, named so a test can supply them directly. */ +export type PolicyDevicePort = { + snapshot: () => Promise<{ nodes: PolicySnapshotNode[]; ms: number }>; + press: (ref: string) => Promise; + fill: (ref: string, text: string) => Promise; +}; + +export type PolicyActOptions = { + goal: string; + provider: PolicyProvider; + device: PolicyDevicePort; + inputs: PolicyTextInputs; + env: Readonly>; + maxSteps: number; + minConfidence: number; + /** Called once per completed step so a CLI can stream progress. */ + onStep?: (step: PolicyActStep) => void; + /** Injected so a test never waits out the grace re-read. */ + sleep?: (ms: number) => Promise; +}; + +export const DEFAULT_POLICY_MAX_STEPS = 12; +export const DEFAULT_POLICY_MIN_CONFIDENCE = 0.4; + +/** + * A provider that reports `blocked` while also naming a concrete target at very high confidence is + * describing the screen, not the goal: an unauthenticated sign-in screen is literally a login wall, + * and every credential it needs is on it. Acting on that target is what keeps the loop from + * stalling on step one of any sign-in flow. Below this confidence the flag is taken at face value. + */ +const ACT_DESPITE_BLOCKED_CONFIDENCE = 0.9; + +/** Consecutive unproductive steps tolerated before the loop hands back. */ +const ESCALATION_LIMIT = 3; + +/** How long to wait before re-reading a screen that has not changed yet. */ +const TRANSITION_GRACE_MS = 800; + +export async function runPolicyActLoop(options: PolicyActOptions): Promise { + const steps: PolicyActStep[] = []; + const history: string[] = []; + let status: PolicyActStatus = 'max-steps'; + let consecutiveUnproductive = 0; + + for (let index = 0; index < options.maxSteps; index++) { + const before = await options.device.snapshot(); + const candidates = toPolicyCandidates(before.nodes); + const screen = screenLabel(candidates); + const digestBefore = screenDigest(candidates); + + const decision = await options.provider.decide({ + goal: options.goal, + candidates, + // Copied, not shared: a provider that holds the request must not observe later steps. + history: [...history], + screen, + textReadyRefs: textReadyRefs(candidates, options.inputs, options.env), + }); + + const record = ( + outcome: PolicyActStep['outcome'], + extra: Partial = {}, + ): PolicyActStep => { + const step: PolicyActStep = { + step: index + 1, + screen, + decision, + outcome, + snapshotMs: before.ms, + decideMs: decision.decideMs, + actionMs: 0, + ...extra, + }; + steps.push(step); + options.onStep?.(step); + return step; + }; + + if (decision.done) { + record('done'); + status = 'done'; + break; + } + + const halt = haltReason(decision, options.minConfidence); + if (halt) { + record(halt.outcome, { reason: halt.reason }); + if (halt.outcome === 'blocked') { + status = 'blocked'; + break; + } + if (++consecutiveUnproductive >= ESCALATION_LIMIT) { + status = 'escalated'; + break; + } + continue; + } + + const target = decision.target as string; + const candidate = candidates.find((entry) => entry.ref === target); + const textEntry = resolveTextEntry(candidate, options.inputs, options.env); + + if (textEntry.kind === 'unsupplied') { + record('escalated', { + reason: `no text supplied for ${candidate?.identifier ?? candidate?.name ?? target}`, + }); + if (++consecutiveUnproductive >= ESCALATION_LIMIT) { + status = 'escalated'; + break; + } + continue; + } + + if (textEntry.kind === 'write') { + const input = textEntry.input; + const filled = await enterText(options.device, target, input.text, candidate); + // A digit-box widget never reflects its value, so a confirmed value is sufficient evidence + // but not necessary: a screen that moved on is the other half of it. + const landed = filled.valueConfirmed || (await screenChanged(options, digestBefore)); + history.push( + landed + ? `filled ${candidate?.name ?? target} from input ${input.key}` + : `could not write into ${candidate?.name ?? target}`, + ); + record(landed ? 'acted' : 'dead-action', { + actionMs: filled.ms, + performed: { + action: 'fill', + target, + entry: filled.entry, + inputKey: input.key, + }, + ...(landed + ? {} + : { + reason: `the field did not take the supplied text (${filled.failure ?? 'value unchanged'})`, + }), + }); + consecutiveUnproductive = landed ? 0 : consecutiveUnproductive + 1; + if (consecutiveUnproductive >= ESCALATION_LIMIT) { + status = 'escalated'; + break; + } + continue; + } + + const actionMs = await options.device.press(target); + history.push(`pressed ${candidate?.name ?? target}`); + const changed = await screenChanged(options, digestBefore); + record(changed ? 'acted' : 'dead-action', { + actionMs, + performed: { action: 'press', target }, + ...(changed ? {} : { reason: 'screen unchanged after press' }), + }); + consecutiveUnproductive = changed ? 0 : consecutiveUnproductive + 1; + if (consecutiveUnproductive >= ESCALATION_LIMIT) { + status = 'escalated'; + break; + } + } + + return { goal: options.goal, status, steps, totals: totalsFor(steps) }; +} + +/** Refs of text fields the caller has text for that is not already in them. */ +function textReadyRefs( + candidates: readonly PolicyCandidate[], + inputs: PolicyTextInputs, + env: Readonly>, +): string[] { + return candidates + .filter((candidate) => resolveTextEntry(candidate, inputs, env).kind === 'write') + .map((candidate) => candidate.ref); +} + +/** + * Whether the screen moved on from the digest taken before the action. + * + * A settled observation only proves the local UI went quiet. A transition waiting on the network — + * request a code, verify a code — lands after that, so an immediate read still shows the old + * screen and a real action would be filed as dead. One grace re-read separates a slow transition + * from an action that genuinely did nothing. + */ +async function screenChanged( + options: Pick, + digestBefore: string, +): Promise { + const digestOf = (nodes: PolicySnapshotNode[]): string => screenDigest(toPolicyCandidates(nodes)); + const immediate = await options.device.snapshot(); + if (digestOf(immediate.nodes) !== digestBefore) return true; + await (options.sleep ?? defaultSleep)(TRANSITION_GRACE_MS); + const settled = await options.device.snapshot(); + return digestOf(settled.nodes) !== digestBefore; +} + +async function defaultSleep(ms: number): Promise { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Decide whether the chosen element needs text written into it. + * + * `press` on a text field only focuses it, so the question is not whether the field is empty but + * whether the caller named text for it that is not already there. A field carrying a development + * default the caller overrode has to be rewritten, and a field already holding the supplied value + * must not be rewritten, or the loop would spend its whole budget refilling one field. + */ +function resolveTextEntry( + candidate: PolicyCandidate | undefined, + inputs: PolicyTextInputs, + env: Readonly>, +): { kind: 'press' } | { kind: 'write'; input: ResolvedTextInput } | { kind: 'unsupplied' } { + if (!candidate || !isTextEntryRole(candidate.role)) return { kind: 'press' }; + const input = resolveTextInput(candidate, inputs, env); + if (!input) return candidate.value ? { kind: 'press' } : { kind: 'unsupplied' }; + return sameText(candidate.value, input.text) ? { kind: 'press' } : { kind: 'write', input }; +} + +/** Compares a field value with supplied text ignoring the formatting a field applies itself. */ +function sameText(value: string | undefined, text: string): boolean { + const normalize = (input: string): string => input.replaceAll(/[^a-z0-9]/giu, '').toLowerCase(); + if (value === undefined) return false; + const current = normalize(value); + const wanted = normalize(text); + return current === wanted || (wanted.length > 0 && current.endsWith(wanted)); +} + +function haltReason( + decision: PolicyDecision, + minConfidence: number, +): { outcome: 'blocked' | 'escalated'; reason: string } | undefined { + if (decision.target === null) { + return { outcome: 'escalated', reason: 'policy chose no element on this screen' }; + } + if (decision.blocked && decision.confidence < ACT_DESPITE_BLOCKED_CONFIDENCE) { + return { outcome: 'blocked', reason: 'policy reports progress blocked on this screen' }; + } + if (decision.confidence < minConfidence) { + return { + outcome: 'escalated', + reason: `confidence ${decision.confidence.toFixed(2)} below --min-confidence ${minConfidence}`, + }; + } + return undefined; +} + +/** + * Write text into a field, then verify from the screen that it arrived. + * + * Two device realities make the write itself an unreliable signal. + * + * A field with an input mask reformats what it receives, so the runner's own text-entry + * verification reports `TEXT_ENTRY_MISMATCH` comparing "5005550700" against "(500) 555-0700". The + * text did arrive. Keyed on that typed code, the loop re-reads the field and decides from the value + * rather than from the comparison. + * + * A code or PIN field is often a row of single-character boxes over a hidden input: the write + * succeeds, the accessibility value never changes, and the submit button stays disabled. When the + * text is all digits, the on-screen keypad is the only path that registers them, one key press at a + * time, each needing a fresh snapshot because refs are reissued after every press. + */ +async function enterText( + device: PolicyDevicePort, + ref: string, + text: string, + candidate: PolicyCandidate | undefined, +): Promise<{ ms: number; entry: 'fill' | 'keypad'; valueConfirmed: boolean; failure?: string }> { + const startedAt = Date.now(); + const failure = await writeFailure(device, ref, text); + const filledMs = Date.now() - startedAt; + + // The write's own verdict is not the evidence; the field's value is. Both realities above report + // failure on a write that arrived, and one reports success on a write that did not. + const verification = await device.snapshot(); + if (valueLanded(verification.nodes, candidate, text)) { + return { ms: filledMs, entry: 'fill', valueConfirmed: true }; + } + + const digits = text.replaceAll(/\D/gu, ''); + if (digits.length !== text.length || digits.length === 0) { + return { ms: filledMs, entry: 'fill', valueConfirmed: false, ...(failure ? { failure } : {}) }; + } + + let keypadMs = 0; + let entered = 0; + for (const digit of digits) { + const snapshot = await device.snapshot(); + const key = findKeypadDigit(snapshot.nodes, digit); + if (!key?.ref) break; + keypadMs += await device.press(key.ref.startsWith('@') ? key.ref : `@${key.ref}`); + entered++; + } + return { + ms: filledMs + keypadMs, + entry: entered > 0 ? 'keypad' : 'fill', + valueConfirmed: entered === digits.length, + ...(failure ? { failure } : {}), + }; +} + +/** + * Attempt the write and report its typed failure rather than raising it. + * + * A masked field reports `TEXT_ENTRY_MISMATCH` on text that did arrive, and a custom digit-box + * widget can make the runner record a failure whose documented recovery is a fresh snapshot — which + * is the next thing this path does. Either way the screen decides, so the write's verdict is + * carried as evidence instead of ending the run. + */ +async function writeFailure( + device: PolicyDevicePort, + ref: string, + text: string, +): Promise { + try { + await device.fill(ref, text); + return undefined; + } catch (error) { + const code = (error as { code?: unknown } | null)?.code; + return typeof code === 'string' ? code : 'the write did not complete'; + } +} + +function valueLanded( + nodes: readonly PolicySnapshotNode[], + candidate: PolicyCandidate | undefined, + text: string, +): boolean { + const node = nodes.find((entry) => + candidate?.identifier + ? entry.identifier === candidate.identifier + : entry.label === candidate?.name, + ); + return sameText(node?.value, text); +} + +function totalsFor(steps: readonly PolicyActStep[]): PolicyActResult['totals'] { + return { + steps: steps.length, + snapshotMs: sum(steps, (step) => step.snapshotMs), + decideMs: sum(steps, (step) => step.decideMs), + actionMs: sum(steps, (step) => step.actionMs), + inputTokens: sum(steps, (step) => step.decision.inputTokens), + costUsd: sum(steps, (step) => step.decision.costUsd), + deadActions: steps.filter((step) => step.outcome === 'dead-action').length, + escalations: steps.filter((step) => step.outcome === 'escalated').length, + }; +} + +function sum(steps: readonly PolicyActStep[], read: (step: PolicyActStep) => number): number { + return steps.reduce((total, step) => total + read(step), 0); +} From 3d04c603981824fab3bc778e254ed57309df57e8 Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 18:06:42 -0400 Subject: [PATCH 05/17] feat(cli): add the suggest and act commands `suggest ` prints one typed decision without acting. `act ` runs the loop. Both are local-cli commands: they compose snapshot, press, and fill from the client process and own no daemon route, which is what keeps them out of the public catalog that requires one. Every nested call is an ordinary daemon command, so a policy-driven run carries the same claims, ref frames, and recording behaviour as a run an agent drives by hand, and each mutation is pinned to the generation of the snapshot that issued its ref. Co-Authored-By: Claude Opus 5 (1M context) --- packages/command-registry/src/registry.ts | 29 ++++ src/agent-device-client.ts | 37 +++++ src/client/client-policy.ts | 147 +++++++++++++++++++ src/client/client-types.ts | 6 + src/commands/family/registry.ts | 2 + src/commands/policy/index.test.ts | 70 +++++++++ src/commands/policy/index.ts | 167 ++++++++++++++++++++++ src/commands/policy/output.ts | 64 +++++++++ 8 files changed, 522 insertions(+) create mode 100644 src/client/client-policy.ts create mode 100644 src/commands/policy/index.test.ts create mode 100644 src/commands/policy/index.ts create mode 100644 src/commands/policy/output.ts diff --git a/packages/command-registry/src/registry.ts b/packages/command-registry/src/registry.ts index d213c386aa..c88e436381 100644 --- a/packages/command-registry/src/registry.ts +++ b/packages/command-registry/src/registry.ts @@ -1538,6 +1538,35 @@ export const RAW_COMMAND_DESCRIPTORS = [ }, // -- local client-backed CLI/MCP commands (no daemon route/capability) -- + // The policy commands compose `snapshot`, `press`, and `fill` from the client process, so they + // own no daemon route of their own and stay out of the public catalog, which requires one. + { + name: 'suggest', + deviceClaimPolicy: 'none', + ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/policy/index.ts'] as const } : {}), + catalog: { group: 'local-cli' }, + recordsSessionAction: false, + timeoutPolicy: DEFAULT_TIMEOUT_POLICY, + batchable: false, + platformExecution: NO_PLATFORM_EXECUTION, + }, + { + name: 'act', + deviceClaimPolicy: 'none', + ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/policy/index.ts'] as const } : {}), + catalog: { group: 'local-cli' }, + recordsSessionAction: false, + // A run is a sequence of ordinary commands, each already under its own envelope; a client + // envelope over the whole loop would abort a run that is still making progress. `--max-steps` + // is the budget that bounds it. + timeoutPolicy: { + ...DEFAULT_TIMEOUT_POLICY, + envelopeMs: 'unbounded', + onTimeout: 'preserve-daemon', + }, + batchable: false, + platformExecution: NO_PLATFORM_EXECUTION, + }, { name: 'debug', deviceClaimPolicy: 'none', diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index e3a0cd60af..b2d15fd746 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -55,6 +55,11 @@ import { resolveSessionName, } from './client/client-normalizers.ts'; import type { AgentDeviceClient, MetroPrepareResult } from './client/client-types.ts'; +import { + runPolicyAct, + suggestPolicyAction, + type PolicyClientCalls, +} from './client/client-policy.ts'; import { INTERNAL_COMMANDS } from '@agent-device/command-registry/catalog'; import { buildRequestFlags } from './commands/command-flags.ts'; import { @@ -135,6 +140,24 @@ export function createAgentDeviceClient( const resolveRequestSession = (options: InternalRequestOptions = {}) => resolveSessionName(mergeClientOptions(config, options).session); + /** + * The three commands the policy loop composes. They are the ordinary daemon commands, so a + * policy-driven run carries the same claims, ref frames, and recording behaviour as a run an + * agent drives by hand. + */ + const policyClientCalls: PolicyClientCalls = { + snapshot: async (options) => { + const session = resolveRequestSession(options); + const data = await executeCommand>('snapshot', options); + const result = normalizeSnapshotResult(data, session); + return { nodes: result.nodes, refsGeneration: result.refsGeneration }; + }, + press: async ({ ref, ...options }) => + await executeCommand('press', { ...options, ...refTarget(ref) }), + fill: async ({ ref, text, ...options }) => + await executeCommand('fill', { ...options, ...refTarget(ref, { text }) }), + }; + return { command: { wait: async (options) => await executeCommand>('wait', options), @@ -423,6 +446,11 @@ export function createAgentDeviceClient( return symbolicateCrashArtifact({ cwd: options.cwd ?? config.cwd, ...options }); }, }, + policy: { + suggest: async (options) => + await suggestPolicyAction(policyClientCalls, options, process.env), + act: async (options) => await runPolicyAct(policyClientCalls, options, process.env), + }, recording: { record: async (options) => await executeCommand>('record', options), trace: async (options) => await executeCommand>('trace', options), @@ -433,6 +461,15 @@ export function createAgentDeviceClient( }; } +/** + * An `@ref` interaction target plus any command-specific fields, in the shape `executeCommand` + * forwards to the daemon. The request options type is deliberately narrow; interaction fields + * reach the wire through the same untyped bag every other interaction call uses. + */ +function refTarget(ref: string, extra: Record = {}): InternalRequestOptions { + return { ref, ...extra } as InternalRequestOptions; +} + function panGestureInput(options: PanOptions): InternalRequestOptions & Record { const { x, y, dx, dy, ...common } = options; return { ...common, kind: 'pan', origin: { x, y }, delta: { x: dx, y: dy } }; diff --git a/src/client/client-policy.ts b/src/client/client-policy.ts new file mode 100644 index 0000000000..610e0befa3 --- /dev/null +++ b/src/client/client-policy.ts @@ -0,0 +1,147 @@ +import type { + AgentDeviceRequestOverrides, + AgentDeviceSelectionOptions, +} from '@agent-device/contracts/client'; +import { + DEFAULT_POLICY_MAX_STEPS, + DEFAULT_POLICY_MIN_CONFIDENCE, + runPolicyActLoop, + type PolicyDevicePort, +} from '../commands/policy/act-loop.ts'; +import { + screenLabel, + toPolicyCandidates, + type PolicySnapshotNode, +} from '../commands/policy/candidate-elements.ts'; +import type { + PolicyActResult, + PolicyActStep, + PolicySuggestResult, +} from '../commands/policy/policy-contract.ts'; +import { + createPolicyProvider, + DEFAULT_POLICY_PROVIDER, +} from '../commands/policy/policy-provider.ts'; +import { parsePolicyTextInputs } from '../commands/policy/text-inputs.ts'; + +type PolicyCommonOptions = AgentDeviceRequestOverrides & AgentDeviceSelectionOptions; + +export type PolicySuggestClientOptions = PolicyCommonOptions & { + goal: string; + policy?: string; +}; + +export type PolicyActClientOptions = PolicySuggestClientOptions & { + maxSteps?: number; + minConfidence?: number; + /** Repeated `key=value` tokens naming text the loop may enter. */ + inputs?: readonly string[]; + onStep?: (step: PolicyActStep) => void; +}; + +/** + * The client calls the loop needs, narrowed to three. Narrowing here rather than passing the whole + * client keeps the loop's device surface visible at the seam where it is granted. + */ +export type PolicyClientCalls = { + snapshot: (options: PolicyCommonOptions & { interactiveOnly: true }) => Promise<{ + nodes: PolicySnapshotNode[]; + refsGeneration?: number; + }>; + press: (options: PolicyCommonOptions & { ref: string; settle?: boolean }) => Promise; + fill: ( + options: PolicyCommonOptions & { ref: string; text: string; settle?: boolean }, + ) => Promise; +}; + +export async function suggestPolicyAction( + calls: PolicyClientCalls, + options: PolicySuggestClientOptions, + env: Readonly>, +): Promise { + const provider = createPolicyProvider(options.policy ?? DEFAULT_POLICY_PROVIDER, env); + const common = commonOf(options); + const startedAt = Date.now(); + const snapshot = await calls.snapshot({ ...common, interactiveOnly: true }); + const snapshotMs = Date.now() - startedAt; + + const candidates = toPolicyCandidates(snapshot.nodes); + const screen = screenLabel(candidates); + const decision = await provider.decide({ + goal: options.goal, + candidates, + history: [], + screen, + // `suggest` reports what the screen affords; it enters no text, so it claims none is ready. + textReadyRefs: [], + }); + return { goal: options.goal, decision, candidates, snapshotMs, screen }; +} + +export async function runPolicyAct( + calls: PolicyClientCalls, + options: PolicyActClientOptions, + env: Readonly>, +): Promise { + const provider = createPolicyProvider(options.policy ?? DEFAULT_POLICY_PROVIDER, env); + return await runPolicyActLoop({ + goal: options.goal, + provider, + device: createPolicyDevicePort(calls, commonOf(options)), + inputs: parsePolicyTextInputs(options.inputs ?? []), + env, + maxSteps: options.maxSteps ?? DEFAULT_POLICY_MAX_STEPS, + minConfidence: options.minConfidence ?? DEFAULT_POLICY_MIN_CONFIDENCE, + ...(options.onStep ? { onStep: options.onStep } : {}), + }); +} + +/** + * Bind the loop to a live session. + * + * Every mutation is pinned to the generation of the snapshot that issued its ref (ADR 0014), so a + * decision made against a screen that has since changed is refused by the daemon rather than + * landing on whatever element inherited the ref. + */ +function createPolicyDevicePort( + calls: PolicyClientCalls, + common: PolicyCommonOptions, +): PolicyDevicePort { + let generation: number | undefined; + const pinned = (ref: string): string => { + const body = ref.startsWith('@') ? ref.slice(1) : ref; + return generation === undefined ? `@${body}` : `@${body}~s${generation}`; + }; + const timed = async (run: () => Promise): Promise => { + const startedAt = Date.now(); + await run(); + return Date.now() - startedAt; + }; + + return { + snapshot: async () => { + const startedAt = Date.now(); + const result = await calls.snapshot({ ...common, interactiveOnly: true }); + generation = result.refsGeneration; + return { nodes: result.nodes, ms: Date.now() - startedAt }; + }, + press: async (ref) => + await timed(async () => await calls.press({ ...common, ref: pinned(ref), settle: true })), + fill: async (ref, text) => + await timed( + async () => await calls.fill({ ...common, ref: pinned(ref), text, settle: true }), + ), + }; +} + +function commonOf(options: PolicySuggestClientOptions): PolicyCommonOptions { + const { goal: _goal, policy: _policy, ...common } = options as PolicyActClientOptions; + const { + maxSteps: _maxSteps, + minConfidence: _minConfidence, + inputs: _inputs, + onStep: _onStep, + ...rest + } = common as PolicyActClientOptions; + return rest as PolicyCommonOptions; +} diff --git a/src/client/client-types.ts b/src/client/client-types.ts index 05523a35b3..48e4421795 100644 --- a/src/client/client-types.ts +++ b/src/client/client-types.ts @@ -139,6 +139,8 @@ import type { } from '@agent-device/contracts/remote'; import type { BatchRunResult } from '@agent-device/command-registry/batch'; +import type { PolicyActClientOptions, PolicySuggestClientOptions } from './client-policy.ts'; +import type { PolicyActResult, PolicySuggestResult } from '../commands/policy/policy-contract.ts'; import type { AgentArtifactsResult, @@ -287,6 +289,10 @@ export type AgentDeviceClient = { debug: { symbols: (options: DebugSymbolsOptions) => Promise; }; + policy: { + suggest: (options: PolicySuggestClientOptions) => Promise; + act: (options: PolicyActClientOptions) => Promise; + }; recording: { record: (options: RecordOptions) => Promise>; trace: (options: TraceOptions) => Promise>; diff --git a/src/commands/family/registry.ts b/src/commands/family/registry.ts index bfd564c7f7..fd4785e448 100644 --- a/src/commands/family/registry.ts +++ b/src/commands/family/registry.ts @@ -8,6 +8,7 @@ import { metroCommandFamily } from '../metro/index.ts'; import { observabilityCommandFamily } from '../observability/index.ts'; import type { CliOutputFormatter } from '../output-common.ts'; import { perfCommandFamily } from '../perf/index.ts'; +import { policyCommandFamily } from '../policy/index.ts'; import { reactNativeCommandFamily } from '../react-native/index.ts'; import { recordingCommandFamily } from '../recording/index.ts'; import { replayCommandFamily } from '../replay/index.ts'; @@ -35,6 +36,7 @@ export const commandFamilies = [ recordingCommandFamily, metroCommandFamily, batchCommandFamily, + policyCommandFamily, ] as const satisfies readonly CommandFamilyFacet[]; export type CommandFamilyCommandName = (typeof commandFamilies)[number]['metadata'][number]['name']; diff --git a/src/commands/policy/index.test.ts b/src/commands/policy/index.test.ts new file mode 100644 index 0000000000..687b7b70c6 --- /dev/null +++ b/src/commands/policy/index.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, test } from 'vitest'; +import type { CliFlags } from '@agent-device/contracts/command'; +import { + actCliReader, + actCommandMetadata, + suggestCliReader, + suggestCommandMetadata, +} from './index.ts'; +import { createPolicyProvider } from './policy-provider.ts'; + +describe('policy command interface', () => { + test('owns suggest and act public metadata', () => { + expect(suggestCommandMetadata.name).toBe('suggest'); + expect(actCommandMetadata.name).toBe('act'); + }); + + test('reads a goal positional and the policy flag', () => { + expect(suggestCliReader(['sign in'], { policy: 'jev' } as CliFlags)).toMatchObject({ + goal: 'sign in', + policy: 'jev', + }); + }); + + test('reads the loop budget, confidence floor, and input map', () => { + expect( + actCliReader(['sign in'], { + batchMaxSteps: 8, + minConfidence: 0.6, + policyInput: ['phone=5555550100'], + } as unknown as CliFlags), + ).toMatchObject({ + goal: 'sign in', + maxSteps: 8, + minConfidence: 0.6, + inputs: ['phone=5555550100'], + }); + }); + + test('refuses an empty goal', () => { + for (const read of [suggestCliReader, actCliReader]) { + expect(() => read([' '], {} as CliFlags)).toThrow( + expect.objectContaining({ + code: 'INVALID_ARGS', + message: expect.stringContaining('needs a goal'), + }), + ); + } + }); +}); + +describe('policy provider resolution', () => { + test('refuses to run without a key, naming the environment entry and the fallback', () => { + expect(() => createPolicyProvider('jev', {})).toThrow( + expect.objectContaining({ + code: 'INVALID_ARGS', + message: expect.stringContaining('TYPESAFE_API_KEY is not set'), + }), + ); + }); + + test('refuses an unknown provider name', () => { + expect(() => createPolicyProvider('oracle', { TYPESAFE_API_KEY: 'k' })).toThrow( + expect.objectContaining({ message: expect.stringContaining('Unknown policy provider') }), + ); + }); + + test('builds the jev provider when the key is present', () => { + expect(createPolicyProvider('jev', { TYPESAFE_API_KEY: 'k' }).name).toBe('jev'); + }); +}); diff --git a/src/commands/policy/index.ts b/src/commands/policy/index.ts new file mode 100644 index 0000000000..d670115698 --- /dev/null +++ b/src/commands/policy/index.ts @@ -0,0 +1,167 @@ +import type { CommandSchemaOverride } from '@agent-device/command-registry/command-schema'; +import { AppError } from '@agent-device/kernel/errors'; +import { commonInputFromFlags } from '../cli-grammar/common.ts'; +import type { CliReader } from '../cli-grammar/types.ts'; +import { + enumField, + numberField, + requiredField, + stringArrayField, + stringField, +} from '../command-input.ts'; +import { commonToClientOptions, type CommonCommandInput } from '../common-input-fields.ts'; +import { defineCommandFacet, defineCommandFamilyFromFacets } from '../family/types.ts'; +import { defineFieldCommandMetadata } from '../field-command-contract.ts'; +import { DEFAULT_POLICY_MAX_STEPS, DEFAULT_POLICY_MIN_CONFIDENCE } from './act-loop.ts'; +import { policyCliOutputFormatters } from './output.ts'; +import { + DEFAULT_POLICY_PROVIDER, + JEV_API_KEY_ENV, + POLICY_PROVIDER_NAMES, +} from './policy-provider.ts'; +import { POLICY_INPUT_ENV_PREFIX } from './text-inputs.ts'; + +const SUGGEST_COMMAND_NAME = 'suggest'; +const ACT_COMMAND_NAME = 'act'; + +const POLICY_PREAMBLE = `A policy head answers one question — which visible element advances the goal — as a typed decision with calibrated probabilities, in roughly the time a snapshot takes. It is optional: with no ${JEV_API_KEY_ENV} in the environment both commands refuse before touching the device, and nothing else changes.`; + +const suggestDescription = `Ask a policy head which element to act on next, and print the decision without acting. ${POLICY_PREAMBLE}`; + +const actDescription = `Run snapshot, decide, act until the policy reports the goal done, reports it blocked, or loses confidence. ${POLICY_PREAMBLE} Text is never generated. A field the policy chooses is filled from a caller-supplied entry keyed by that field's identifier or label, or from the matching environment entry when the value is a secret; a field with no entry escalates instead of being filled with a guess.`; + +const goalField = requiredField( + stringField('What the run should achieve, in one sentence, as the policy sees it.'), +); +const policyField = enumField( + POLICY_PROVIDER_NAMES, + `Policy head to ask. Default ${DEFAULT_POLICY_PROVIDER}.`, +); + +export const suggestCommandMetadata = defineFieldCommandMetadata( + SUGGEST_COMMAND_NAME, + suggestDescription, + { goal: goalField, policy: policyField }, +); + +export const actCommandMetadata = defineFieldCommandMetadata(ACT_COMMAND_NAME, actDescription, { + goal: goalField, + policy: policyField, + maxSteps: numberField(`Stop after this many steps. Default ${DEFAULT_POLICY_MAX_STEPS}.`, { + min: 1, + max: 100, + }), + minConfidence: numberField( + `Escalate instead of acting below this confidence. Default ${DEFAULT_POLICY_MIN_CONFIDENCE}.`, + { min: 0, max: 1 }, + ), + inputs: stringArrayField( + 'Text the loop may enter, as key=value. The key matches a field identifier or label.', + ), +}); + +const suggestCliSchema = { + usageOverride: 'suggest [--policy ]', + usageFlags: [], + listUsageOverride: 'suggest ', + positionalArgs: ['goal'], + allowedFlags: ['policy'], +} as const satisfies CommandSchemaOverride; + +const actCliSchema = { + usageOverride: + 'act [--policy ] [--max-steps ] [--min-confidence ] [--input ]', + usageFlags: [], + listUsageOverride: 'act ', + positionalArgs: ['goal'], + allowedFlags: ['policy', 'batchMaxSteps', 'minConfidence', 'policyInput'], +} as const satisfies CommandSchemaOverride; + +export const suggestCliReader: CliReader = (positionals, flags) => ({ + ...commonInputFromFlags(flags), + goal: readGoal(positionals[0], SUGGEST_COMMAND_NAME), + policy: flags.policy, +}); + +export const actCliReader: CliReader = (positionals, flags) => ({ + ...commonInputFromFlags(flags), + goal: readGoal(positionals[0], ACT_COMMAND_NAME), + policy: flags.policy, + maxSteps: flags.batchMaxSteps, + minConfidence: flags.minConfidence, + inputs: flags.policyInput, +}); + +type SuggestInput = CommonCommandInput & { goal: string; policy?: string }; +type ActInput = SuggestInput & { + maxSteps?: number; + minConfidence?: number; + inputs?: string[]; +}; + +const suggestCommandFacet = defineCommandFacet({ + name: SUGGEST_COMMAND_NAME, + text: { + summary: 'Ask a policy head for the next element to act on', + cliDetail: [ + `Prints the chosen ref, the probability it carries, and whether the policy considers the goal done or blocked. It never touches the device beyond one interactive snapshot, so it is safe to run repeatedly while checking a flow by hand.`, + '', + 'Examples:', + ` agent-device suggest "sign in with the test phone number"`, + ` agent-device suggest "open the first item" --json`, + ].join('\n'), + }, + metadata: suggestCommandMetadata, + run: (client, input) => client.policy.suggest(toSuggestOptions(input as SuggestInput)), + cliSchema: suggestCliSchema, + cliReader: suggestCliReader, + cliOutputFormatter: policyCliOutputFormatters.suggest, +}); + +const actCommandFacet = defineCommandFacet({ + name: ACT_COMMAND_NAME, + text: { + summary: 'Drive a goal with a policy head until it is done or blocked', + cliDetail: [ + `Each step snapshots, asks the policy, acts, and re-snapshots. An action that leaves the screen digest unchanged is recorded as a dead action; three unproductive steps in a row end the run as escalated. Per-step snapshot, decide, and action milliseconds are in the result, as is the policy token cost. Supply text with --input key=value, or keep a secret out of argv by exporting ${POLICY_INPUT_ENV_PREFIX} instead.`, + '', + 'Examples:', + ` agent-device act "sign in and reach the main list" --input phone=5555550100 --max-steps 10`, + ` agent-device act "open the first item" --min-confidence 0.6 --json`, + ].join('\n'), + }, + metadata: actCommandMetadata, + run: (client, input) => client.policy.act(toActOptions(input as ActInput)), + cliSchema: actCliSchema, + cliReader: actCliReader, + cliOutputFormatter: policyCliOutputFormatters.act, +}); + +export const policyCommandFamily = defineCommandFamilyFromFacets({ + name: 'policy', + commands: [suggestCommandFacet, actCommandFacet], +}); + +function toSuggestOptions(input: SuggestInput) { + return { ...commonToClientOptions(input), goal: input.goal, policy: input.policy }; +} + +function toActOptions(input: ActInput) { + return { + ...toSuggestOptions(input), + maxSteps: input.maxSteps, + minConfidence: input.minConfidence, + inputs: input.inputs, + }; +} + +function readGoal(value: string | undefined, command: string): string { + const goal = value?.trim(); + if (!goal) { + throw new AppError( + 'INVALID_ARGS', + `${command} needs a goal, for example: ${command} "sign in"`, + ); + } + return goal; +} diff --git a/src/commands/policy/output.ts b/src/commands/policy/output.ts new file mode 100644 index 0000000000..39360a70d1 --- /dev/null +++ b/src/commands/policy/output.ts @@ -0,0 +1,64 @@ +import type { CliOutput } from '../command-contract.ts'; +import { resultOutput, type CliOutputFormatter } from '../output-common.ts'; +import type { PolicyActResult, PolicyDecision, PolicySuggestResult } from './policy-contract.ts'; + +function suggestCliOutput(result: PolicySuggestResult): CliOutput { + const lines = [ + `Goal: ${result.goal}`, + `Screen: ${result.screen}`, + decisionLine(result.decision), + topProbabilities(result.decision), + `Timing: snapshot ${ms(result.snapshotMs)} decide ${ms(result.decision.decideMs)}`, + `Cost: ${usd(result.decision.costUsd)} for ${result.decision.inputTokens} input tokens`, + ]; + return { data: result, text: lines.filter(Boolean).join('\n') }; +} + +function actCliOutput(result: PolicyActResult): CliOutput { + const lines = [`Goal: ${result.goal}`, `Status: ${result.status}`]; + for (const step of result.steps) { + const performed = step.performed + ? ` ${step.performed.action} ${step.performed.target}${step.performed.entry === 'keypad' ? ' via keypad' : ''}` + : ''; + lines.push( + `${String(step.step).padStart(2)} ${step.outcome.padEnd(12)}${performed} conf ${step.decision.confidence.toFixed(2)} | ${step.screen}`, + ); + if (step.reason) lines.push(` ${step.reason}`); + } + const totals = result.totals; + lines.push( + `Steps: ${totals.steps}, dead actions ${totals.deadActions}, escalations ${totals.escalations}`, + `Timing: snapshot ${ms(totals.snapshotMs)} decide ${ms(totals.decideMs)} action ${ms(totals.actionMs)}`, + `Cost: ${usd(totals.costUsd)} for ${totals.inputTokens} input tokens`, + ); + return { data: result, text: lines.join('\n') }; +} + +export const policyCliOutputFormatters = { + suggest: resultOutput(suggestCliOutput), + act: resultOutput(actCliOutput), +} as const satisfies Record; + +function decisionLine(decision: PolicyDecision): string { + const target = decision.target ?? 'none'; + const flags = [decision.done ? 'done' : '', decision.blocked ? 'blocked' : ''] + .filter(Boolean) + .join(' '); + return `Next: ${decision.action} ${target} at confidence ${decision.confidence.toFixed(2)}${flags ? ` (${flags})` : ''}`; +} + +function topProbabilities(decision: PolicyDecision): string { + const ranked = Object.entries(decision.probabilities) + .sort(([, a], [, b]) => b - a) + .slice(0, 3) + .map(([label, probability]) => `${label} ${(probability * 100).toFixed(1)}%`); + return ranked.length > 0 ? `Probabilities: ${ranked.join(', ')}` : ''; +} + +function ms(value: number): string { + return `${Math.round(value)}ms`; +} + +function usd(value: number): string { + return `$${value.toFixed(6)}`; +} From fc66d635ef4d010131c05c63c5da24f9c201cf85 Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 18:06:50 -0400 Subject: [PATCH 06/17] test: record the policy commands in the reviewed descriptor ledgers Both are pure delegators, so they join the `none` device-claim list. `act` joins the daemon-preserving and unbounded-envelope lists: a run is a sequence of ordinary commands that each carry their own envelope, and an outer one would abort a run still making progress while a reset would destroy the session the remaining steps need. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/__tests__/device-claim-policy.test.ts | 2 ++ src/__tests__/cli-client-commands.test.ts | 1 + src/__tests__/command-descriptor-timeout-policy.test.ts | 7 +++++++ src/__tests__/remote-connection.fixtures.ts | 1 + 4 files changed, 11 insertions(+) diff --git a/packages/command-registry/src/__tests__/device-claim-policy.test.ts b/packages/command-registry/src/__tests__/device-claim-policy.test.ts index 97a40dc68f..2266ffa873 100644 --- a/packages/command-registry/src/__tests__/device-claim-policy.test.ts +++ b/packages/command-registry/src/__tests__/device-claim-policy.test.ts @@ -52,6 +52,7 @@ test('every command that deviates from require-owner is a reviewed, diffable set ], observe: ['apps', 'appstate', 'capabilities', 'device', 'devices', 'doctor', 'takeover'], none: [ + 'act', 'artifacts', 'auth', 'batch', @@ -74,6 +75,7 @@ test('every command that deviates from require-owner is a reviewed, diffable set 'session', 'session_list', 'session_save_script', + 'suggest', 'web', ], }); diff --git a/src/__tests__/cli-client-commands.test.ts b/src/__tests__/cli-client-commands.test.ts index b3382c6ba9..dfe131c986 100644 --- a/src/__tests__/cli-client-commands.test.ts +++ b/src/__tests__/cli-client-commands.test.ts @@ -1272,6 +1272,7 @@ function createStubClient(params: { events: params.events ?? unexpectedCommandCall, }, debug: createThrowingMethodGroup(), + policy: createThrowingMethodGroup(), recording: createThrowingMethodGroup(), settings: { update: params.updateSettings ?? unexpectedCommandCall, diff --git a/src/__tests__/command-descriptor-timeout-policy.test.ts b/src/__tests__/command-descriptor-timeout-policy.test.ts index d12d682520..4b9c38b207 100644 --- a/src/__tests__/command-descriptor-timeout-policy.test.ts +++ b/src/__tests__/command-descriptor-timeout-policy.test.ts @@ -67,10 +67,14 @@ test('daemon-preserving timeout commands are a bounded, reviewed set', () => { // sessions the daemon owns, so a client-side timeout must not SIGKILL the // daemon mid-create/mid-release and orphan them (and every other provider // session held). + // `act` joined with the policy head: a run is a sequence of ordinary commands, each already + // under its own envelope, so a client-side timeout over the whole loop must not reset the + // daemon and destroy the session the remaining steps still need. const preserving = commandDescriptors .filter((descriptor) => descriptor.timeoutPolicy.onTimeout === 'preserve-daemon') .map((descriptor) => descriptor.name); assert.deepEqual(preserving.sort(), [ + 'act', 'back', 'click', 'fill', @@ -146,6 +150,9 @@ test('request envelopes deviating from the default are bounded, reviewed sets', // #1774: base allocation budget (300s) + client/daemon race margin (30s). lease_allocate: 330_000, test: 'unbounded', + // A policy run is bounded by --max-steps, not by wall clock; each step it issues carries its + // own envelope, and an outer one would abort a run that is still making progress. + act: 'unbounded', }; for (const descriptor of commandDescriptors) { const expected = EXPECTED_ENVELOPES[descriptor.name] ?? 90_000; diff --git a/src/__tests__/remote-connection.fixtures.ts b/src/__tests__/remote-connection.fixtures.ts index b121ac06cd..7fdfd4a041 100644 --- a/src/__tests__/remote-connection.fixtures.ts +++ b/src/__tests__/remote-connection.fixtures.ts @@ -116,6 +116,7 @@ export function createTestClient( batch: createThrowingMethodGroup(), observability: createThrowingMethodGroup(), debug: createThrowingMethodGroup(), + policy: createThrowingMethodGroup(), recording: createThrowingMethodGroup(), settings: createThrowingMethodGroup(), }; From db96be32b7e4dc820a365e6651bab88dccca5efc Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 18:06:50 -0400 Subject: [PATCH 07/17] docs: document the policy head A command-reference section, a reference page covering the decision shape, the text-supply rule, the post-action checks, and the limitations, plus one paragraph on the skill routing card. Co-Authored-By: Claude Opus 5 (1M context) --- skills/agent-device/SKILL.md | 2 + website/docs/docs/_meta.json | 5 ++ website/docs/docs/commands.md | 27 +++++++ website/docs/docs/policy-head.md | 127 +++++++++++++++++++++++++++++++ 4 files changed, 161 insertions(+) create mode 100644 website/docs/docs/policy-head.md diff --git a/skills/agent-device/SKILL.md b/skills/agent-device/SKILL.md index a2447b3ed5..de243b483e 100644 --- a/skills/agent-device/SKILL.md +++ b/skills/agent-device/SKILL.md @@ -19,4 +19,6 @@ Reaching an off-screen target is one command, not a scroll-and-check loop: `scro Copy refs byte-for-byte: `@e12`, `@e12~s4` — keep the `@` and any `~sN`. Prefer current refs, then `id`/`label`/`role` selectors; coordinates are a last resort. If snapshot reports sparse/AX-unavailable, its refs and selectors are invalid: run `agent-device screenshot`, inspect the image, use coordinates, then retry `snapshot -i` after navigating. Otherwise run `snapshot -i` only when the diff lacks the next target. +With `TYPESAFE_API_KEY` set, `suggest ""` names the next element to act on in roughly the time a snapshot takes, and `act "" --input key=value` runs the whole snapshot-decide-act loop. Text is never generated; a field with no `--input` entry escalates. Without the key both refuse, and the loop above is the path. On `Jev unavailable: `, fall back to that loop. + Error output includes corrective hints; follow them instead of re-planning. Only when the task is specialized (for example gestures, scripting, TV, macOS, remote, or debugging) or a command shape is unclear, run `agent-device help `. `agent-device --help` lists topics, but is not a startup step. diff --git a/website/docs/docs/_meta.json b/website/docs/docs/_meta.json index 68655a6001..c69962ab84 100644 --- a/website/docs/docs/_meta.json +++ b/website/docs/docs/_meta.json @@ -56,6 +56,11 @@ "label": "Batching", "link": "/docs/batching" }, + { + "type": "custom-link", + "label": "Policy head", + "link": "/docs/policy-head" + }, { "type": "custom-link", "label": "Snapshots", diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 18a4aefd42..d430ec9b0f 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -592,6 +592,33 @@ agent-device batch --steps '[{"command":"open","input":{"app":"settings"}}]' See [Batching](/docs/batching) for payload format, response shape, and usage guidelines. +## Policy head (optional) + +```bash +agent-device suggest "sign in and reach the main list" +agent-device act "sign in and reach the main list" --input phone=5555550100 --max-steps 10 +agent-device act "open the first item" --min-confidence 0.6 --json +``` + +- A policy head answers one question — which visible element advances the goal — as a typed + decision with calibrated probabilities. `suggest` prints that decision; `act` loops + snapshot, decide, press or fill until the goal is done, blocked, or the policy loses confidence. +- Both commands are gated on `TYPESAFE_API_KEY`. Without it they refuse before touching the + device, and nothing else in the CLI changes. +- `--policy ` selects the head; `jev` is the only one today and the default. +- `act` never generates text. A field the policy chooses is filled from `--input key=value`, or + from `AGENT_DEVICE_INPUT_` in the environment when the value is a secret. A field with no + matching entry escalates instead of being filled with a guess. +- After every action the loop re-snapshots and compares a content digest of the screen. An action + that changed nothing is recorded as a dead action, and three unproductive steps in a row end the + run as `escalated`. +- A code or PIN field that ignores `fill` is entered digit by digit on the on-screen keypad. +- When the head is unreachable the error names the status and the recovery: `Jev unavailable: + ; fall back to agent-driven policy`. The agent's own snapshot-and-choose loop is always + available; the policy head is an accelerator, not a dependency. + +See [Policy head](/docs/policy-head) for the decision shape, timings, and limitations. + ## App install (in-place) ```bash diff --git a/website/docs/docs/policy-head.md b/website/docs/docs/policy-head.md new file mode 100644 index 0000000000..81a1040be3 --- /dev/null +++ b/website/docs/docs/policy-head.md @@ -0,0 +1,127 @@ +--- +title: Policy head +--- + +# Policy head + +An agent driving a device spends most of its wall clock deciding, not acting. A snapshot returns in +roughly 400 ms and a press settles in one to ten seconds, but the model reading that snapshot and +choosing a ref usually takes several seconds and costs a full inference. Over a twelve-step sign-in +flow that is the largest single line in the bill and the longest single phase of the run. + +A policy head answers only the narrow question — which of these elements advances this goal — as a +typed decision with calibrated probabilities. It does not write text, plan, or explain. `suggest` +asks it once. `act` puts it in a loop. + +The feature is optional and off by default. Without `TYPESAFE_API_KEY` in the environment both +commands refuse before touching the device, and nothing else in the CLI behaves differently. + +## Asking once + +```bash +agent-device suggest "sign in and reach the main list" +``` + +``` +Goal: sign in and reach the main list +Screen: Sign in with your phone. | Phone number | Send code +Next: fill @e7 at confidence 0.99 (blocked) +Probabilities: @e7 99.0%, @e9 0.6%, __none__ 0.4% +Timing: snapshot 412ms decide 198ms +Cost: $0.000017 for 412 input tokens +``` + +`suggest` takes one interactive snapshot and prints the decision. It performs no action, so it is +safe to run repeatedly while working through a flow by hand. With `--json` the same decision comes +back as a typed object: `target`, `action`, `confidence`, `probabilities`, `done`, `blocked`, +`needsText`, plus the timing and token cost. + +## Running the loop + +```bash +agent-device act "sign in and reach the main list" \ + --input phone=5555550100 \ + --max-steps 10 +``` + +Each step snapshots, asks the policy, presses or fills, and snapshots again. The run ends when the +policy reports the goal done, reports it blocked, or three unproductive steps happen in a row. + +| Flag | Meaning | +| --- | --- | +| `--policy ` | Which head to ask. `jev` is the default and the only one today. | +| `--max-steps ` | Step budget for the run. Default 12. | +| `--min-confidence ` | Escalate instead of acting below this confidence. Default 0.4. | +| `--input ` | Repeatable. Text the loop may enter. | + +## Text is supplied, never generated + +The policy chooses *where* text goes. It never chooses *what* the text is. A field the policy picks +is filled from an `--input` entry whose key matches the field's accessibility identifier or label, +first exactly and then as a substring. + +A press on a text field only focuses it, so the loop reads the supplied entry rather than the field's +emptiness to decide what to do. A field whose value is not the supplied text is rewritten, which is +what reaches past a development default the caller overrode. A field already holding that text is +pressed, so the loop cannot spend its whole budget refilling one field. A field that is empty with no +matching entry escalates. + +For a value that should not appear in a command line or a shell history, set +`AGENT_DEVICE_INPUT_` in the environment instead. A one-time code fetched from a test backend +belongs there: + +```bash +AGENT_DEVICE_INPUT_CODE=$(fetch-test-otp) agent-device act "finish signing in" +``` + +The step record names the input key that was used. It never records the value. + +## What the loop checks after it acts + +**Same-screen check.** After every action the loop re-snapshots and compares a content digest of the +screen: roles, names, values, and disabled states, with refs deliberately excluded because they are +reissued per snapshot generation. An action that leaves the digest unchanged is recorded as a dead +action rather than counted as progress, and three in a row end the run. + +**Digit fields.** A code or PIN field is often a row of single-character boxes over a hidden input. +`fill` reports success, the accessibility value never changes, and the submit button stays disabled. +When the loop sees a fill of all-digit text that did not land, it enters the digits one at a time on +the on-screen keypad, taking a fresh snapshot per key because refs are reissued after every press. + +**Ref pinning.** Every mutation is pinned to the generation of the snapshot that issued its ref, so a +decision made against a screen that has since changed is refused rather than landing on whatever +element inherited the ref. + +## Blocked is about the screen, not the goal + +A head asked "is progress blocked" will say yes on any sign-in screen, because an unauthenticated +screen is literally a login wall. It says so while also naming the phone field at very high +confidence. The loop acts on a target named at 0.9 confidence or above even when `blocked` is set, +and takes the flag at face value below that. Without this a sign-in flow stalls on step one. + +## When the head is unreachable + +Failures are typed, carry the HTTP status, and name the recovery: + +``` +Jev unavailable: 429 rate limit exceeded; fall back to agent-driven policy +``` + +| Status | `reason` | Retriable | +| --- | --- | --- | +| 401, 403 | `policy-provider-unauthorized` | no | +| 402 | `policy-provider-payment-required` | no | +| 429 | `policy-provider-rate-limited` | yes | +| 5xx, transport | `policy-provider-http`, `policy-provider-transport` | yes | + +The documented fallback is the agent's own loop: `snapshot`, choose an element, `press` or `fill`. +That path is always available and is what the CLI does today. The policy head accelerates it; it is +never a dependency. + +## Limitations + +- Needs an API key. There is no offline or local head. +- Decides between elements only. It writes no text and plans no multi-screen strategy. +- Keeps concurrency low. More than about four parallel calls invites rate limiting. +- Reads the accessibility tree, so an element the snapshot cannot see is an element the policy + cannot choose. From 8dca87daeaf5f9b00525372374399554f444a141 Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 18:14:38 -0400 Subject: [PATCH 08/17] refactor(policy): split the loop and the candidate projection The changed-code quality gate flagged the step loop as critically complex and six exports as unconsumed. The loop now reads as snapshot, decide, resolve step, record, with the write and press branches and the keypad walk as their own functions, and the candidate projection resolves a node's role separately from building the candidate. The internal constants are module-local. Co-Authored-By: Claude Opus 5 (1M context) --- src/commands/policy/act-loop.ts | 237 +++++++++++---------- src/commands/policy/candidate-elements.ts | 55 ++--- src/commands/policy/jev-composer.test.ts | 9 +- src/commands/policy/jev-policy-provider.ts | 4 +- src/commands/policy/policy-provider.ts | 6 +- 5 files changed, 172 insertions(+), 139 deletions(-) diff --git a/src/commands/policy/act-loop.ts b/src/commands/policy/act-loop.ts index c081e2e70d..741385e6e5 100644 --- a/src/commands/policy/act-loop.ts +++ b/src/commands/policy/act-loop.ts @@ -13,6 +13,7 @@ import type { PolicyCandidate, PolicyDecision, PolicyProvider, + PolicyStepOutcome, } from './policy-contract.ts'; import { resolveTextInput, type PolicyTextInputs, type ResolvedTextInput } from './text-inputs.ts'; @@ -54,130 +55,141 @@ const ESCALATION_LIMIT = 3; /** How long to wait before re-reading a screen that has not changed yet. */ const TRANSITION_GRACE_MS = 800; +/** The per-step inputs a step decision and action both read. */ +type StepContext = { + decision: PolicyDecision; + candidates: PolicyCandidate[]; + digestBefore: string; + history: string[]; +}; + +/** What one step concluded, before the loop turns it into a recorded step. */ +type StepResult = { outcome: PolicyStepOutcome; extra?: Partial }; + export async function runPolicyActLoop(options: PolicyActOptions): Promise { const steps: PolicyActStep[] = []; const history: string[] = []; let status: PolicyActStatus = 'max-steps'; - let consecutiveUnproductive = 0; + let unproductive = 0; for (let index = 0; index < options.maxSteps; index++) { const before = await options.device.snapshot(); const candidates = toPolicyCandidates(before.nodes); const screen = screenLabel(candidates); - const digestBefore = screenDigest(candidates); - - const decision = await options.provider.decide({ - goal: options.goal, + const context: StepContext = { + decision: await options.provider.decide({ + goal: options.goal, + candidates, + // Copied, not shared: a provider that holds the request must not observe later steps. + history: [...history], + screen, + textReadyRefs: textReadyRefs(candidates, options.inputs, options.env), + }), candidates, - // Copied, not shared: a provider that holds the request must not observe later steps. - history: [...history], + digestBefore: screenDigest(candidates), + history, + }; + + const result = await resolveStep(options, context); + const step: PolicyActStep = { + step: index + 1, screen, - textReadyRefs: textReadyRefs(candidates, options.inputs, options.env), - }); - - const record = ( - outcome: PolicyActStep['outcome'], - extra: Partial = {}, - ): PolicyActStep => { - const step: PolicyActStep = { - step: index + 1, - screen, - decision, - outcome, - snapshotMs: before.ms, - decideMs: decision.decideMs, - actionMs: 0, - ...extra, - }; - steps.push(step); - options.onStep?.(step); - return step; + decision: context.decision, + outcome: result.outcome, + snapshotMs: before.ms, + decideMs: context.decision.decideMs, + actionMs: 0, + ...result.extra, }; + steps.push(step); + options.onStep?.(step); - if (decision.done) { - record('done'); - status = 'done'; + unproductive = result.outcome === 'acted' ? 0 : unproductive + 1; + const ended = endStatus(result.outcome, unproductive); + if (ended) { + status = ended; break; } + } - const halt = haltReason(decision, options.minConfidence); - if (halt) { - record(halt.outcome, { reason: halt.reason }); - if (halt.outcome === 'blocked') { - status = 'blocked'; - break; - } - if (++consecutiveUnproductive >= ESCALATION_LIMIT) { - status = 'escalated'; - break; - } - continue; - } + return { goal: options.goal, status, steps, totals: totalsFor(steps) }; +} - const target = decision.target as string; - const candidate = candidates.find((entry) => entry.ref === target); - const textEntry = resolveTextEntry(candidate, options.inputs, options.env); +/** The status a run ends with after this outcome, or undefined to keep going. */ +function endStatus(outcome: PolicyStepOutcome, unproductive: number): PolicyActStatus | undefined { + if (outcome === 'done') return 'done'; + if (outcome === 'blocked') return 'blocked'; + return unproductive >= ESCALATION_LIMIT ? 'escalated' : undefined; +} + +/** Judge the decision, and act on it when it is actionable. */ +async function resolveStep(options: PolicyActOptions, context: StepContext): Promise { + if (context.decision.done) return { outcome: 'done' }; + + const halt = haltReason(context.decision, options.minConfidence); + if (halt) return { outcome: halt.outcome, extra: { reason: halt.reason } }; - if (textEntry.kind === 'unsupplied') { - record('escalated', { + const target = context.decision.target as string; + const candidate = context.candidates.find((entry) => entry.ref === target); + const textEntry = resolveTextEntry(candidate, options.inputs, options.env); + + if (textEntry.kind === 'unsupplied') { + return { + outcome: 'escalated', + extra: { reason: `no text supplied for ${candidate?.identifier ?? candidate?.name ?? target}`, - }); - if (++consecutiveUnproductive >= ESCALATION_LIMIT) { - status = 'escalated'; - break; - } - continue; - } + }, + }; + } + if (textEntry.kind === 'write') { + return await writeStep(options, context, target, candidate, textEntry.input); + } + return await pressStep(options, context, target, candidate); +} - if (textEntry.kind === 'write') { - const input = textEntry.input; - const filled = await enterText(options.device, target, input.text, candidate); - // A digit-box widget never reflects its value, so a confirmed value is sufficient evidence - // but not necessary: a screen that moved on is the other half of it. - const landed = filled.valueConfirmed || (await screenChanged(options, digestBefore)); - history.push( - landed - ? `filled ${candidate?.name ?? target} from input ${input.key}` - : `could not write into ${candidate?.name ?? target}`, - ); - record(landed ? 'acted' : 'dead-action', { - actionMs: filled.ms, - performed: { - action: 'fill', - target, - entry: filled.entry, - inputKey: input.key, - }, - ...(landed - ? {} - : { - reason: `the field did not take the supplied text (${filled.failure ?? 'value unchanged'})`, - }), - }); - consecutiveUnproductive = landed ? 0 : consecutiveUnproductive + 1; - if (consecutiveUnproductive >= ESCALATION_LIMIT) { - status = 'escalated'; - break; - } - continue; - } +async function writeStep( + options: PolicyActOptions, + context: StepContext, + target: string, + candidate: PolicyCandidate | undefined, + input: ResolvedTextInput, +): Promise { + const name = candidate?.name ?? target; + const filled = await enterText(options.device, target, input.text, candidate); + // A digit-box widget never reflects its value, so a confirmed value is sufficient evidence but + // not necessary: a screen that moved on is the other half of it. + const landed = filled.valueConfirmed || (await screenChanged(options, context.digestBefore)); + context.history.push( + landed ? `filled ${name} from input ${input.key}` : `could not write into ${name}`, + ); + const reason = `the field did not take the supplied text (${filled.failure ?? 'value unchanged'})`; + return { + outcome: landed ? 'acted' : 'dead-action', + extra: { + actionMs: filled.ms, + performed: { action: 'fill', target, entry: filled.entry, inputKey: input.key }, + ...(landed ? {} : { reason }), + }, + }; +} - const actionMs = await options.device.press(target); - history.push(`pressed ${candidate?.name ?? target}`); - const changed = await screenChanged(options, digestBefore); - record(changed ? 'acted' : 'dead-action', { +async function pressStep( + options: PolicyActOptions, + context: StepContext, + target: string, + candidate: PolicyCandidate | undefined, +): Promise { + const actionMs = await options.device.press(target); + context.history.push(`pressed ${candidate?.name ?? target}`); + const changed = await screenChanged(options, context.digestBefore); + return { + outcome: changed ? 'acted' : 'dead-action', + extra: { actionMs, performed: { action: 'press', target }, ...(changed ? {} : { reason: 'screen unchanged after press' }), - }); - consecutiveUnproductive = changed ? 0 : consecutiveUnproductive + 1; - if (consecutiveUnproductive >= ESCALATION_LIMIT) { - status = 'escalated'; - break; - } - } - - return { goal: options.goal, status, steps, totals: totalsFor(steps) }; + }, + }; } /** Refs of text fields the caller has text for that is not already in them. */ @@ -299,21 +311,30 @@ async function enterText( return { ms: filledMs, entry: 'fill', valueConfirmed: false, ...(failure ? { failure } : {}) }; } - let keypadMs = 0; + const keypad = await enterDigitsOnKeypad(device, digits); + return { + ms: filledMs + keypad.ms, + entry: keypad.entered > 0 ? 'keypad' : 'fill', + valueConfirmed: keypad.entered === digits.length, + ...(failure ? { failure } : {}), + }; +} + +/** Tap each digit on the on-screen keypad, re-reading because refs are reissued per press. */ +async function enterDigitsOnKeypad( + device: PolicyDevicePort, + digits: string, +): Promise<{ ms: number; entered: number }> { + let ms = 0; let entered = 0; for (const digit of digits) { const snapshot = await device.snapshot(); const key = findKeypadDigit(snapshot.nodes, digit); if (!key?.ref) break; - keypadMs += await device.press(key.ref.startsWith('@') ? key.ref : `@${key.ref}`); + ms += await device.press(key.ref.startsWith('@') ? key.ref : `@${key.ref}`); entered++; } - return { - ms: filledMs + keypadMs, - entry: entered > 0 ? 'keypad' : 'fill', - valueConfirmed: entered === digits.length, - ...(failure ? { failure } : {}), - }; + return { ms, entered }; } /** diff --git a/src/commands/policy/candidate-elements.ts b/src/commands/policy/candidate-elements.ts index eaf6bdf5e4..57ce7d6296 100644 --- a/src/commands/policy/candidate-elements.ts +++ b/src/commands/policy/candidate-elements.ts @@ -57,35 +57,15 @@ export function isTextEntryRole(role: string): boolean { * what distinguishes "Enter the code" from "Welcome" when every button on both screens is generic. */ export function toPolicyCandidates(nodes: readonly PolicySnapshotNode[]): PolicyCandidate[] { - const childrenOf = new Map(); - for (const node of nodes) { - if (node.parentIndex === undefined) continue; - const siblings = childrenOf.get(node.parentIndex); - if (siblings) siblings.push(node); - else childrenOf.set(node.parentIndex, [node]); - } - + const childrenOf = groupByParent(nodes); const candidates: PolicyCandidate[] = []; for (const node of nodes) { - const type = node.type ?? ''; - if (!node.ref || STRUCTURAL_TYPES.has(type)) continue; - - let role: string; - if (LEAF_ROLES[type]) { - role = LEAF_ROLES[type]; - } else if (type === 'Cell') { - if (node.index !== undefined && hasActionableDescendant(childrenOf, node.index)) continue; - role = 'button'; - } else if (type === 'StaticText') { - role = TEXT_ROLE; - } else { - continue; - } - + const role = roleForNode(node, childrenOf); + if (role === undefined || !node.ref) continue; candidates.push({ ref: node.ref.startsWith('@') ? node.ref : `@${node.ref}`, role, - name: node.label || node.identifier || type, + name: node.label || node.identifier || node.type || 'element', ...(node.identifier === undefined ? {} : { identifier: node.identifier }), ...(node.value === undefined ? {} : { value: node.value }), ...(node.enabled === false ? { disabled: true } : {}), @@ -94,6 +74,33 @@ export function toPolicyCandidates(nodes: readonly PolicySnapshotNode[]): Policy return candidates; } +/** The candidate role for one node, or undefined when it carries no decision value. */ +function roleForNode( + node: PolicySnapshotNode, + childrenOf: ReadonlyMap, +): string | undefined { + const type = node.type ?? ''; + if (!node.ref || STRUCTURAL_TYPES.has(type)) return undefined; + const leafRole = LEAF_ROLES[type]; + if (leafRole) return leafRole; + if (type === 'StaticText') return TEXT_ROLE; + if (type !== 'Cell') return undefined; + // A row wrapping its own control would make the policy choose between the two. + if (node.index !== undefined && hasActionableDescendant(childrenOf, node.index)) return undefined; + return 'button'; +} + +function groupByParent(nodes: readonly PolicySnapshotNode[]): Map { + const childrenOf = new Map(); + for (const node of nodes) { + if (node.parentIndex === undefined) continue; + const siblings = childrenOf.get(node.parentIndex); + if (siblings) siblings.push(node); + else childrenOf.set(node.parentIndex, [node]); + } + return childrenOf; +} + /** The subset a policy may choose: actionable, enabled, and not plain text. */ export function selectableCandidates(candidates: readonly PolicyCandidate[]): PolicyCandidate[] { return candidates.filter( diff --git a/src/commands/policy/jev-composer.test.ts b/src/commands/policy/jev-composer.test.ts index 520a712f4c..004bbe9c8b 100644 --- a/src/commands/policy/jev-composer.test.ts +++ b/src/commands/policy/jev-composer.test.ts @@ -1,5 +1,10 @@ import { describe, expect, test } from 'vitest'; -import { buildJevRequest, DECLINE_LABEL, readJevDecision } from './jev-composer.ts'; +import { + buildJevRequest, + DECLINE_LABEL, + JEV_USD_PER_INPUT_TOKEN, + readJevDecision, +} from './jev-composer.ts'; import type { PolicyCandidate, PolicyRequest } from './policy-contract.ts'; const candidates: PolicyCandidate[] = [ @@ -82,7 +87,7 @@ describe('jev decision parsing', () => { model: 'jev-1.13.0', decideMs: 187, inputTokens: 500, - costUsd: 500 * (42 / 1e9), + costUsd: 500 * JEV_USD_PER_INPUT_TOKEN, }); }); diff --git a/src/commands/policy/jev-policy-provider.ts b/src/commands/policy/jev-policy-provider.ts index f85d4b9caa..31d4ff82a1 100644 --- a/src/commands/policy/jev-policy-provider.ts +++ b/src/commands/policy/jev-policy-provider.ts @@ -3,8 +3,8 @@ import { buildJevRequest, readJevDecision } from './jev-composer.ts'; import type { PolicyDecision, PolicyProvider, PolicyRequest } from './policy-contract.ts'; export const JEV_PROVIDER_NAME = 'jev'; -export const JEV_DEFAULT_ENDPOINT = 'https://api.typesafe.ai/v1/systemone'; -export const JEV_DEFAULT_MODEL = 'jev-latest'; +const JEV_DEFAULT_ENDPOINT = 'https://api.typesafe.ai/v1/systemone'; +const JEV_DEFAULT_MODEL = 'jev-latest'; const JEV_TIMEOUT_MS = 20_000; /** diff --git a/src/commands/policy/policy-provider.ts b/src/commands/policy/policy-provider.ts index 59bce6fe5b..892b0f6382 100644 --- a/src/commands/policy/policy-provider.ts +++ b/src/commands/policy/policy-provider.ts @@ -13,12 +13,12 @@ export const DEFAULT_POLICY_PROVIDER: PolicyProviderName = JEV_PROVIDER_NAME; /** Environment keys the jev provider reads. The key is never accepted as a CLI flag. */ export const JEV_API_KEY_ENV = 'TYPESAFE_API_KEY'; -export const JEV_MODEL_ENV = 'AGENT_DEVICE_POLICY_MODEL'; -export const JEV_ENDPOINT_ENV = 'AGENT_DEVICE_POLICY_ENDPOINT'; +const JEV_MODEL_ENV = 'AGENT_DEVICE_POLICY_MODEL'; +const JEV_ENDPOINT_ENV = 'AGENT_DEVICE_POLICY_ENDPOINT'; export type PolicyProviderEnvironment = Readonly>; -export function isPolicyProviderName(value: string): value is PolicyProviderName { +function isPolicyProviderName(value: string): value is PolicyProviderName { return (POLICY_PROVIDER_NAMES as readonly string[]).includes(value); } From 35147e9878b240b60d5e45ed3493aeedacd01949 Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 18:29:02 -0400 Subject: [PATCH 09/17] fix(policy): compose the fallback hint as one sentence The hint read "Without it, Fall back to agent-driven policy" wherever it was interpolated. The clause now lives in its own module, lower-cased, so both the provider and the composer can name it without one importing the other. Co-Authored-By: Claude Opus 5 (1M context) --- src/commands/policy/jev-composer.ts | 3 ++- src/commands/policy/jev-policy-provider.ts | 22 ++++++++++------------ src/commands/policy/policy-fallback.ts | 9 +++++++++ src/commands/policy/policy-provider.ts | 9 +++------ 4 files changed, 24 insertions(+), 19 deletions(-) create mode 100644 src/commands/policy/policy-fallback.ts diff --git a/src/commands/policy/jev-composer.ts b/src/commands/policy/jev-composer.ts index 8300848d7f..f25474c5f0 100644 --- a/src/commands/policy/jev-composer.ts +++ b/src/commands/policy/jev-composer.ts @@ -1,5 +1,6 @@ import { AppError } from '@agent-device/kernel/errors'; import { isTextEntryRole, selectableCandidates } from './candidate-elements.ts'; +import { POLICY_FALLBACK_HINT } from './policy-fallback.ts'; import type { PolicyCandidate, PolicyDecision, PolicyRequest } from './policy-contract.ts'; /** @@ -167,6 +168,6 @@ function isRecord(value: unknown): value is Record { function providerFault(message: string): AppError { return new AppError('COMMAND_FAILED', `Jev returned an unusable decision: ${message}`, { reason: 'policy-provider-response', - hint: 'Fall back to agent-driven policy: snapshot, choose an element yourself, then press or fill it.', + hint: `Retry, or ${POLICY_FALLBACK_HINT}`, }); } diff --git a/src/commands/policy/jev-policy-provider.ts b/src/commands/policy/jev-policy-provider.ts index 31d4ff82a1..523f02d85d 100644 --- a/src/commands/policy/jev-policy-provider.ts +++ b/src/commands/policy/jev-policy-provider.ts @@ -1,5 +1,6 @@ import { AppError } from '@agent-device/kernel/errors'; import { buildJevRequest, readJevDecision } from './jev-composer.ts'; +import { POLICY_FALLBACK_HINT } from './policy-fallback.ts'; import type { PolicyDecision, PolicyProvider, PolicyRequest } from './policy-contract.ts'; export const JEV_PROVIDER_NAME = 'jev'; @@ -7,13 +8,6 @@ const JEV_DEFAULT_ENDPOINT = 'https://api.typesafe.ai/v1/systemone'; const JEV_DEFAULT_MODEL = 'jev-latest'; const JEV_TIMEOUT_MS = 20_000; -/** - * The documented recovery when the policy head is unreachable. It names the agent's own loop - * because that loop is always available: the policy head is an accelerator, never a dependency. - */ -export const POLICY_FALLBACK_HINT = - 'Fall back to agent-driven policy: snapshot, choose an element yourself, then press or fill it.'; - export type JevProviderOptions = { apiKey: string; model?: string; @@ -69,7 +63,11 @@ async function callJev( throw new AppError( 'COMMAND_FAILED', `Jev unavailable: ${error instanceof Error ? error.message : 'request failed'}`, - { reason: 'policy-provider-transport', retriable: true, hint: POLICY_FALLBACK_HINT }, + { + reason: 'policy-provider-transport', + retriable: true, + hint: `Retry, or ${POLICY_FALLBACK_HINT}`, + }, error, ); } @@ -102,13 +100,13 @@ function reasonForStatus(status: number): string { function hintForStatus(status: number): string { if (status === 401 || status === 403) { - return `TYPESAFE_API_KEY is missing or rejected. ${POLICY_FALLBACK_HINT}`; + return `TYPESAFE_API_KEY is missing or rejected; ${POLICY_FALLBACK_HINT}`; } - if (status === 402) return `The TypeSafe account has no credit. ${POLICY_FALLBACK_HINT}`; + if (status === 402) return `The TypeSafe account has no credit; ${POLICY_FALLBACK_HINT}`; if (status === 429) { - return `Rate limited; keep policy concurrency at or below 4. ${POLICY_FALLBACK_HINT}`; + return `Rate limited; keep policy concurrency at or below 4, or ${POLICY_FALLBACK_HINT}`; } - return POLICY_FALLBACK_HINT; + return `Retry, or ${POLICY_FALLBACK_HINT}`; } async function readErrorText(response: Response): Promise { diff --git a/src/commands/policy/policy-fallback.ts b/src/commands/policy/policy-fallback.ts new file mode 100644 index 0000000000..a99c87606d --- /dev/null +++ b/src/commands/policy/policy-fallback.ts @@ -0,0 +1,9 @@ +/** + * The documented recovery when the policy head cannot answer. It names the agent's own loop + * because that loop is always available: the policy head is an accelerator, never a dependency. + * + * It lives apart from both the provider and the composer so either can name it without one + * importing the other. + */ +export const POLICY_FALLBACK_HINT = + 'fall back to agent-driven policy: snapshot, choose an element yourself, then press or fill it.'; diff --git a/src/commands/policy/policy-provider.ts b/src/commands/policy/policy-provider.ts index 892b0f6382..33891f5429 100644 --- a/src/commands/policy/policy-provider.ts +++ b/src/commands/policy/policy-provider.ts @@ -1,9 +1,6 @@ import { AppError } from '@agent-device/kernel/errors'; -import { - createJevPolicyProvider, - JEV_PROVIDER_NAME, - POLICY_FALLBACK_HINT, -} from './jev-policy-provider.ts'; +import { createJevPolicyProvider, JEV_PROVIDER_NAME } from './jev-policy-provider.ts'; +import { POLICY_FALLBACK_HINT } from './policy-fallback.ts'; import type { PolicyProvider } from './policy-contract.ts'; /** Every policy head the CLI can resolve. One ships today. */ @@ -47,7 +44,7 @@ export function createPolicyProvider( `${JEV_API_KEY_ENV} is not set, so policy ${name} cannot decide`, { reason: 'policy-provider-unconfigured', - hint: `Export ${JEV_API_KEY_ENV} to enable the policy head. Without it, ${POLICY_FALLBACK_HINT}`, + hint: `Export ${JEV_API_KEY_ENV} to enable the policy head, or ${POLICY_FALLBACK_HINT}`, }, ); } From 533fafded155dd2f9650d57ed8fd8e74fd058516 Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 18:54:29 -0400 Subject: [PATCH 10/17] fix(policy): make a blocked screen terminal only with no way on A policy asked whether progress is blocked says yes on anything that looks like a wall. In live runs on an iOS sign-in flow the flag was set twice while the policy also named the element that led onward: the phone field on the sign-in screen, and "I'll start fresh" on an onboarding page offering a file import. The previous rule acted through the flag only above 0.9 confidence, so the second one ended the run three screens short of the goal. The flag is now terminal only together with nothing worth acting on: no target, or one below --min-confidence. A named target above the floor is acted on. Being wrong there costs one step that the same-screen check catches and the step budget bounds; believing the flag costs the run. Co-Authored-By: Claude Opus 5 (1M context) --- src/commands/policy/act-loop.test.ts | 19 +++++++++++--- src/commands/policy/act-loop.ts | 39 +++++++++++++++------------- website/docs/docs/policy-head.md | 16 ++++++++---- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/commands/policy/act-loop.test.ts b/src/commands/policy/act-loop.test.ts index 1ef54ffdfb..49a5d27e9c 100644 --- a/src/commands/policy/act-loop.test.ts +++ b/src/commands/policy/act-loop.test.ts @@ -154,18 +154,31 @@ describe('policy act loop', () => { expect(result.steps[0]?.reason).toContain('below --min-confidence'); }); - test('acts through a blocked flag only when confidence is very high', async () => { + test('acts on a named target even when the policy flags the screen blocked', async () => { const acting = await runPolicyActLoop({ ...baseOptions, maxSteps: 1, - provider: providerOf(decision({ blocked: true, confidence: 0.99 })), + provider: providerOf(decision({ blocked: true, confidence: 0.52 })), device: deviceOf([welcome, codeScreen]), }); expect(acting.steps[0]?.outcome).toBe('acted'); + }); + + test('stops as blocked when the policy flags the screen and names no way on', async () => { + const halted = await runPolicyActLoop({ + ...baseOptions, + provider: providerOf(decision({ blocked: true, target: null, action: 'none' })), + device: deviceOf([welcome]), + }); + expect(halted.status).toBe('blocked'); + expect(halted.steps[0]?.reason).toBe('policy reports progress blocked and names no way on'); + }); + test('stops as blocked when the flag comes with a target below the floor', async () => { const halted = await runPolicyActLoop({ ...baseOptions, - provider: providerOf(decision({ blocked: true, confidence: 0.7 })), + minConfidence: 0.6, + provider: providerOf(decision({ blocked: true, confidence: 0.2 })), device: deviceOf([welcome]), }); expect(halted.status).toBe('blocked'); diff --git a/src/commands/policy/act-loop.ts b/src/commands/policy/act-loop.ts index 741385e6e5..cda13961e9 100644 --- a/src/commands/policy/act-loop.ts +++ b/src/commands/policy/act-loop.ts @@ -41,14 +41,6 @@ export type PolicyActOptions = { export const DEFAULT_POLICY_MAX_STEPS = 12; export const DEFAULT_POLICY_MIN_CONFIDENCE = 0.4; -/** - * A provider that reports `blocked` while also naming a concrete target at very high confidence is - * describing the screen, not the goal: an unauthenticated sign-in screen is literally a login wall, - * and every credential it needs is on it. Acting on that target is what keeps the loop from - * stalling on step one of any sign-in flow. Below this confidence the flag is taken at face value. - */ -const ACT_DESPITE_BLOCKED_CONFIDENCE = 0.9; - /** Consecutive unproductive steps tolerated before the loop hands back. */ const ESCALATION_LIMIT = 3; @@ -255,23 +247,34 @@ function sameText(value: string | undefined, text: string): boolean { return current === wanted || (wanted.length > 0 && current.endsWith(wanted)); } +/** + * Whether to stop instead of acting. + * + * `blocked` answers a question about the screen, not about the goal, and a policy asked it says yes + * on anything that looks like a wall: an unauthenticated sign-in screen, or an onboarding page + * offering to import a file. Twice in live runs it was set while the policy also named the one + * element that led onward. So `blocked` is terminal only together with nothing worth acting on — + * no target, or one below the confidence floor. A named target above the floor is acted on, because + * the cost of being wrong is one step the same-screen check catches and the step budget bounds, + * while the cost of believing the flag is the whole run. + */ function haltReason( decision: PolicyDecision, minConfidence: number, ): { outcome: 'blocked' | 'escalated'; reason: string } | undefined { + const usable = + decision.target !== null && decision.confidence >= minConfidence ? decision.target : undefined; + if (usable !== undefined) return undefined; + if (decision.blocked) { + return { outcome: 'blocked', reason: 'policy reports progress blocked and names no way on' }; + } if (decision.target === null) { return { outcome: 'escalated', reason: 'policy chose no element on this screen' }; } - if (decision.blocked && decision.confidence < ACT_DESPITE_BLOCKED_CONFIDENCE) { - return { outcome: 'blocked', reason: 'policy reports progress blocked on this screen' }; - } - if (decision.confidence < minConfidence) { - return { - outcome: 'escalated', - reason: `confidence ${decision.confidence.toFixed(2)} below --min-confidence ${minConfidence}`, - }; - } - return undefined; + return { + outcome: 'escalated', + reason: `confidence ${decision.confidence.toFixed(2)} below --min-confidence ${minConfidence}`, + }; } /** diff --git a/website/docs/docs/policy-head.md b/website/docs/docs/policy-head.md index 81a1040be3..a9a3c85bc4 100644 --- a/website/docs/docs/policy-head.md +++ b/website/docs/docs/policy-head.md @@ -45,7 +45,8 @@ agent-device act "sign in and reach the main list" \ ``` Each step snapshots, asks the policy, presses or fills, and snapshots again. The run ends when the -policy reports the goal done, reports it blocked, or three unproductive steps happen in a row. +policy reports the goal done, reports it blocked with no element worth acting on, or three +unproductive steps happen in a row. | Flag | Meaning | | --- | --- | @@ -94,10 +95,15 @@ element inherited the ref. ## Blocked is about the screen, not the goal -A head asked "is progress blocked" will say yes on any sign-in screen, because an unauthenticated -screen is literally a login wall. It says so while also naming the phone field at very high -confidence. The loop acts on a target named at 0.9 confidence or above even when `blocked` is set, -and takes the flag at face value below that. Without this a sign-in flow stalls on step one. +A policy asked whether progress is blocked says yes on anything that looks like a wall. An +unauthenticated sign-in screen is literally a login wall. An onboarding page offering to import a +file is blocked on a file. In live runs the flag was set twice while the policy also named the one +element that led onward: the phone field, and an "I'll start fresh" button. + +So the flag is terminal only together with nothing worth acting on: no target, or one below +`--min-confidence`. A named target above the floor is acted on. The cost of being wrong there is one +step, which the same-screen check catches and the step budget bounds. The cost of believing the flag +is the whole run. ## When the head is unreachable From 32d3fe31394660cbe423a28c92504f49720177c6 Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 19:22:43 -0400 Subject: [PATCH 11/17] perf(policy): load the policy client on demand The CLI startup closure must not grow, and a static edge from the client facade pulled the whole policy subtree into every invocation. The policy head is optional, so its client loads inside the two methods that use it, the way debug symbols already does. Also replaces a `as string` on the chosen target with a discriminated judgment, and reaches the client fixture's device group through the spread idiom that file already uses for observability, so the oversized-test-file ratchet holds. Co-Authored-By: Claude Opus 5 (1M context) --- src/__tests__/cli-client-commands.test.ts | 4 +- src/agent-device-client.ts | 19 +++--- src/client/client-policy.test.ts | 80 +++++++++++++++++++++++ src/commands/policy/act-loop.ts | 33 +++++++--- 4 files changed, 115 insertions(+), 21 deletions(-) create mode 100644 src/client/client-policy.test.ts diff --git a/src/__tests__/cli-client-commands.test.ts b/src/__tests__/cli-client-commands.test.ts index dfe131c986..6766a22947 100644 --- a/src/__tests__/cli-client-commands.test.ts +++ b/src/__tests__/cli-client-commands.test.ts @@ -1168,10 +1168,8 @@ function createStubClient(params: { return { command, devices: { + ...createThrowingMethodGroup(), list: async () => [], - capabilities: unexpectedCommandCall, - boot: unexpectedCommandCall, - shutdown: unexpectedCommandCall, }, sessions: { list: async () => [], diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index b2d15fd746..737187b7a6 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -55,11 +55,7 @@ import { resolveSessionName, } from './client/client-normalizers.ts'; import type { AgentDeviceClient, MetroPrepareResult } from './client/client-types.ts'; -import { - runPolicyAct, - suggestPolicyAction, - type PolicyClientCalls, -} from './client/client-policy.ts'; +import type { PolicyClientCalls } from './client/client-policy.ts'; import { INTERNAL_COMMANDS } from '@agent-device/command-registry/catalog'; import { buildRequestFlags } from './commands/command-flags.ts'; import { @@ -447,9 +443,16 @@ export function createAgentDeviceClient( }, }, policy: { - suggest: async (options) => - await suggestPolicyAction(policyClientCalls, options, process.env), - act: async (options) => await runPolicyAct(policyClientCalls, options, process.env), + // Loaded on demand: the policy head is optional, and nothing about it belongs in the + // module closure every CLI invocation evaluates at startup. + suggest: async (options) => { + const { suggestPolicyAction } = await import('./client/client-policy.ts'); + return await suggestPolicyAction(policyClientCalls, options, process.env); + }, + act: async (options) => { + const { runPolicyAct } = await import('./client/client-policy.ts'); + return await runPolicyAct(policyClientCalls, options, process.env); + }, }, recording: { record: async (options) => await executeCommand>('record', options), diff --git a/src/client/client-policy.test.ts b/src/client/client-policy.test.ts new file mode 100644 index 0000000000..02682acc8a --- /dev/null +++ b/src/client/client-policy.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, test } from 'vitest'; +import { runPolicyAct, suggestPolicyAction, type PolicyClientCalls } from './client-policy.ts'; + +/** A device surface that fails the test if the commands reach it. */ +function forbiddenDevice(): PolicyClientCalls & { touched: string[] } { + const touched: string[] = []; + return { + touched, + snapshot: async () => { + touched.push('snapshot'); + return { nodes: [] }; + }, + press: async () => { + touched.push('press'); + }, + fill: async () => { + touched.push('fill'); + }, + }; +} + +describe('policy commands without a key', () => { + test('suggest refuses before reading the screen', async () => { + const device = forbiddenDevice(); + await expect(suggestPolicyAction(device, { goal: 'sign in' }, {})).rejects.toThrow( + expect.objectContaining({ + code: 'INVALID_ARGS', + message: expect.stringContaining('TYPESAFE_API_KEY is not set'), + }), + ); + expect(device.touched).toEqual([]); + }); + + test('act refuses before reading the screen', async () => { + const device = forbiddenDevice(); + await expect(runPolicyAct(device, { goal: 'sign in' }, {})).rejects.toThrow( + expect.objectContaining({ code: 'INVALID_ARGS' }), + ); + expect(device.touched).toEqual([]); + }); + + test('the refusal names both the way to enable it and the way without it', async () => { + const device = forbiddenDevice(); + await expect(suggestPolicyAction(device, { goal: 'sign in' }, {})).rejects.toMatchObject({ + details: { + reason: 'policy-provider-unconfigured', + hint: expect.stringContaining('Export TYPESAFE_API_KEY'), + }, + }); + await expect(suggestPolicyAction(device, { goal: 'sign in' }, {})).rejects.toMatchObject({ + details: { hint: expect.stringContaining('fall back to agent-driven policy') }, + }); + }); + + test('an empty key is treated as no key', async () => { + const device = forbiddenDevice(); + await expect( + suggestPolicyAction(device, { goal: 'sign in' }, { TYPESAFE_API_KEY: ' ' }), + ).rejects.toThrow(expect.objectContaining({ code: 'INVALID_ARGS' })); + expect(device.touched).toEqual([]); + }); +}); + +describe('the rest of the command surface without a key', () => { + test('every other command keeps its descriptor untouched by the policy feature', async () => { + const { commandDescriptors } = await import('@agent-device/command-registry/registry'); + const policyNames = new Set(['suggest', 'act']); + const others = commandDescriptors.filter((descriptor) => !policyNames.has(descriptor.name)); + + // The feature adds two descriptors and changes none: anything that reads the registry at + // startup behaves identically whether or not a key is present, because no existing command + // gained a policy field. + for (const descriptor of others) { + expect(JSON.stringify(descriptor)).not.toContain('policy'); + } + expect( + commandDescriptors.filter((descriptor) => policyNames.has(descriptor.name)), + ).toHaveLength(2); + }); +}); diff --git a/src/commands/policy/act-loop.ts b/src/commands/policy/act-loop.ts index cda13961e9..0ff20dea83 100644 --- a/src/commands/policy/act-loop.ts +++ b/src/commands/policy/act-loop.ts @@ -118,10 +118,12 @@ function endStatus(outcome: PolicyStepOutcome, unproductive: number): PolicyActS async function resolveStep(options: PolicyActOptions, context: StepContext): Promise { if (context.decision.done) return { outcome: 'done' }; - const halt = haltReason(context.decision, options.minConfidence); - if (halt) return { outcome: halt.outcome, extra: { reason: halt.reason } }; + const judgment = judge(context.decision, options.minConfidence); + if (judgment.kind === 'halt') { + return { outcome: judgment.outcome, extra: { reason: judgment.reason } }; + } - const target = context.decision.target as string; + const target = judgment.target; const candidate = context.candidates.find((entry) => entry.ref === target); const textEntry = resolveTextEntry(candidate, options.inputs, options.env); @@ -258,20 +260,31 @@ function sameText(value: string | undefined, text: string): boolean { * the cost of being wrong is one step the same-screen check catches and the step budget bounds, * while the cost of believing the flag is the whole run. */ -function haltReason( +function judge( decision: PolicyDecision, minConfidence: number, -): { outcome: 'blocked' | 'escalated'; reason: string } | undefined { - const usable = - decision.target !== null && decision.confidence >= minConfidence ? decision.target : undefined; - if (usable !== undefined) return undefined; +): + | { kind: 'act'; target: string } + | { kind: 'halt'; outcome: 'blocked' | 'escalated'; reason: string } { + if (decision.target !== null && decision.confidence >= minConfidence) { + return { kind: 'act', target: decision.target }; + } if (decision.blocked) { - return { outcome: 'blocked', reason: 'policy reports progress blocked and names no way on' }; + return { + kind: 'halt', + outcome: 'blocked', + reason: 'policy reports progress blocked and names no way on', + }; } if (decision.target === null) { - return { outcome: 'escalated', reason: 'policy chose no element on this screen' }; + return { + kind: 'halt', + outcome: 'escalated', + reason: 'policy chose no element on this screen', + }; } return { + kind: 'halt', outcome: 'escalated', reason: `confidence ${decision.confidence.toFixed(2)} below --min-confidence ${minConfidence}`, }; From f343125e1848290471dd9a2de63fdff72840a796 Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 19:28:12 -0400 Subject: [PATCH 12/17] refactor(policy): home the CLI surface where the startup closure already reaches The eager-closure gate holds src/cli.ts to the module count at the merge-base, and a command family is reached from it through the family registry, so a new family directory grows it by every module the family index evaluates. The gate names the remedy: give the new code a home in a module the closure already evaluates. suggest and act join the interaction family, whose vocabulary they already share: find, get, and is are reads that name an element, and act is a loop over the verbs beside it. Their CLI defaults move next to the flags that publish them, so the help prose and the loop read one declaration. The policy runtime stays under commands/policy and is still reached only through the client's on-demand import. Co-Authored-By: Claude Opus 5 (1M context) --- packages/command-registry/package.json | 4 + .../src/flag-definitions-workflow.ts | 24 ++- src/client/client-policy.ts | 11 +- src/commands/family/registry.ts | 2 - src/commands/interaction/index.ts | 168 +++++++++++++++++- src/commands/interaction/output.ts | 97 ++++++++-- .../policy-commands.test.ts} | 2 +- src/commands/policy/act-loop.ts | 3 - src/commands/policy/index.ts | 167 ----------------- src/commands/policy/output.ts | 64 ------- src/commands/policy/policy-provider.ts | 18 +- src/commands/policy/text-inputs.ts | 4 +- 12 files changed, 289 insertions(+), 275 deletions(-) rename src/commands/{policy/index.test.ts => interaction/policy-commands.test.ts} (96%) delete mode 100644 src/commands/policy/index.ts delete mode 100644 src/commands/policy/output.ts diff --git a/packages/command-registry/package.json b/packages/command-registry/package.json index 3fef336010..871043cc3d 100644 --- a/packages/command-registry/package.json +++ b/packages/command-registry/package.json @@ -83,6 +83,10 @@ "types": "./src/flag-groups.ts", "default": "./src/flag-groups.ts" }, + "./flag-definitions-workflow": { + "types": "./src/flag-definitions-workflow.ts", + "default": "./src/flag-definitions-workflow.ts" + }, "./command-text": { "types": "./src/command-text.ts", "default": "./src/command-text.ts" diff --git a/packages/command-registry/src/flag-definitions-workflow.ts b/packages/command-registry/src/flag-definitions-workflow.ts index e20c8c14a1..5ee836f998 100644 --- a/packages/command-registry/src/flag-definitions-workflow.ts +++ b/packages/command-registry/src/flag-definitions-workflow.ts @@ -1,6 +1,24 @@ import { SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS } from '@agent-device/contracts/capture'; import type { FlagDefinition } from './flag-types.ts'; +/** + * The policy head's CLI vocabulary and defaults. + * + * They live beside the flags that publish them so the prose and the runtime read one declaration: + * `--max-steps` and `--min-confidence` state these numbers in their help, and the act loop applies + * them. Kept here rather than in the policy modules because the CLI surface is evaluated at + * startup while the policy runtime loads on demand, and a constant may not drag the second into + * the first. + */ +export const POLICY_PROVIDER_NAMES = ['jev'] as const; +export const DEFAULT_POLICY_PROVIDER: (typeof POLICY_PROVIDER_NAMES)[number] = 'jev'; +export const DEFAULT_POLICY_MAX_STEPS = 12; +export const DEFAULT_POLICY_MIN_CONFIDENCE = 0.4; +/** Environment entry the jev provider reads its credential from; never a flag. */ +export const POLICY_API_KEY_ENV = 'TYPESAFE_API_KEY'; +/** Prefix for per-key text entries, so a secret never has to appear in argv. */ +export const POLICY_INPUT_ENV_PREFIX = 'AGENT_DEVICE_INPUT_'; + export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ { key: 'replayUpdate', @@ -195,7 +213,7 @@ export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ min: 1, max: 1000, usageLabel: '--max-steps ', - usageDescription: 'Batch: maximum allowed steps; act: stop the loop after this many steps', + usageDescription: `Batch: maximum allowed steps; act: stop the loop after this many steps (default ${DEFAULT_POLICY_MAX_STEPS})`, projectConfig: true, recorded: false, }, @@ -327,7 +345,7 @@ export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ names: ['--policy'], type: 'string', usageLabel: '--policy ', - usageDescription: 'suggest/act: policy head that decides the next element (default jev)', + usageDescription: `suggest/act: policy head that decides the next element (default ${DEFAULT_POLICY_PROVIDER})`, projectConfig: true, recorded: false, }, @@ -336,7 +354,7 @@ export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ names: ['--min-confidence'], type: 'number', usageLabel: '--min-confidence ', - usageDescription: 'act: escalate instead of acting below this policy confidence (default 0.4)', + usageDescription: `act: escalate instead of acting below this policy confidence (default ${DEFAULT_POLICY_MIN_CONFIDENCE})`, projectConfig: false, recorded: false, }, diff --git a/src/client/client-policy.ts b/src/client/client-policy.ts index 610e0befa3..0c47d25a88 100644 --- a/src/client/client-policy.ts +++ b/src/client/client-policy.ts @@ -5,9 +5,9 @@ import type { import { DEFAULT_POLICY_MAX_STEPS, DEFAULT_POLICY_MIN_CONFIDENCE, - runPolicyActLoop, - type PolicyDevicePort, -} from '../commands/policy/act-loop.ts'; + DEFAULT_POLICY_PROVIDER, +} from '@agent-device/command-registry/flag-definitions-workflow'; +import { runPolicyActLoop, type PolicyDevicePort } from '../commands/policy/act-loop.ts'; import { screenLabel, toPolicyCandidates, @@ -18,10 +18,7 @@ import type { PolicyActStep, PolicySuggestResult, } from '../commands/policy/policy-contract.ts'; -import { - createPolicyProvider, - DEFAULT_POLICY_PROVIDER, -} from '../commands/policy/policy-provider.ts'; +import { createPolicyProvider } from '../commands/policy/policy-provider.ts'; import { parsePolicyTextInputs } from '../commands/policy/text-inputs.ts'; type PolicyCommonOptions = AgentDeviceRequestOverrides & AgentDeviceSelectionOptions; diff --git a/src/commands/family/registry.ts b/src/commands/family/registry.ts index fd4785e448..bfd564c7f7 100644 --- a/src/commands/family/registry.ts +++ b/src/commands/family/registry.ts @@ -8,7 +8,6 @@ import { metroCommandFamily } from '../metro/index.ts'; import { observabilityCommandFamily } from '../observability/index.ts'; import type { CliOutputFormatter } from '../output-common.ts'; import { perfCommandFamily } from '../perf/index.ts'; -import { policyCommandFamily } from '../policy/index.ts'; import { reactNativeCommandFamily } from '../react-native/index.ts'; import { recordingCommandFamily } from '../recording/index.ts'; import { replayCommandFamily } from '../replay/index.ts'; @@ -36,7 +35,6 @@ export const commandFamilies = [ recordingCommandFamily, metroCommandFamily, batchCommandFamily, - policyCommandFamily, ] as const satisfies readonly CommandFamilyFacet[]; export type CommandFamilyCommandName = (typeof commandFamilies)[number]['metadata'][number]['name']; diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index 52ff2d5391..3656cc5edb 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -20,18 +20,35 @@ import type { TypeTextOptions, } from '@agent-device/contracts/client'; import type { CommandSchemaOverride } from '@agent-device/command-registry/command-schema'; +import { + DEFAULT_POLICY_MAX_STEPS, + DEFAULT_POLICY_MIN_CONFIDENCE, + DEFAULT_POLICY_PROVIDER, + POLICY_API_KEY_ENV, + POLICY_INPUT_ENV_PREFIX, + POLICY_PROVIDER_NAMES, +} from '@agent-device/command-registry/flag-definitions-workflow'; +import { AppError } from '@agent-device/kernel/errors'; import { REPEATED_TOUCH_FLAGS, SELECTOR_SNAPSHOT_FLAGS, } from '@agent-device/command-registry/flag-groups'; import { postActionObservationCliFlags } from '../post-action-observation-grammar.ts'; import { + enumField, + numberField, + requiredField, + stringArrayField, + stringField, toClientElementTarget, toClientInteractionTarget, toRepeatedOptions, toSelectorSnapshotOptions, } from '../command-input.ts'; -import { commonToClientOptions } from '../common-input-fields.ts'; +import { commonInputFromFlags } from '../cli-grammar/common.ts'; +import type { CliReader } from '../cli-grammar/types.ts'; +import { commonToClientOptions, type CommonCommandInput } from '../common-input-fields.ts'; +import { defineFieldCommandMetadata } from '../field-command-contract.ts'; import { defineCommandFacet, defineCommandFamilyFromFacets } from '../family/types.ts'; import { gestureCliReaders, gestureDaemonWriters } from './gesture.ts'; import { interactionCliReaders, interactionDaemonWriters } from './interactions.ts'; @@ -347,6 +364,153 @@ const gestureCommandFacet = defineCommandFacet({ daemonWriter: gestureDaemonWriters.gesture, }); +// --- policy-backed commands ----------------------------------------------------------------- +// `suggest` names the next element the way `find` does, and `act` is a loop over the verbs above, +// so both belong to this family's vocabulary. They are declared here rather than in their own +// module because the CLI surface is part of the closure every invocation evaluates at startup, +// and a new module there would grow it; the policy runtime they call lives under commands/policy +// and is loaded on demand by the client. + +const SUGGEST_COMMAND_NAME = 'suggest'; +const ACT_COMMAND_NAME = 'act'; + +const POLICY_PREAMBLE = `A policy head answers one question — which visible element advances the goal — as a typed decision with calibrated probabilities, in roughly the time a snapshot takes. It is optional: with no ${POLICY_API_KEY_ENV} in the environment both commands refuse before touching the device, and nothing else changes.`; + +const suggestDescription = `Ask a policy head which element to act on next, and print the decision without acting. ${POLICY_PREAMBLE}`; + +const actDescription = `Run snapshot, decide, act until the policy reports the goal done, reports it blocked, or loses confidence. ${POLICY_PREAMBLE} Text is never generated. A field the policy chooses is filled from a caller-supplied entry keyed by that field's identifier or label, or from the matching environment entry when the value is a secret; a field with no entry escalates instead of being filled with a guess.`; + +const goalField = requiredField( + stringField('What the run should achieve, in one sentence, as the policy sees it.'), +); +const policyField = enumField( + POLICY_PROVIDER_NAMES, + `Policy head to ask. Default ${DEFAULT_POLICY_PROVIDER}.`, +); + +export const suggestCommandMetadata = defineFieldCommandMetadata( + SUGGEST_COMMAND_NAME, + suggestDescription, + { goal: goalField, policy: policyField }, +); + +export const actCommandMetadata = defineFieldCommandMetadata(ACT_COMMAND_NAME, actDescription, { + goal: goalField, + policy: policyField, + maxSteps: numberField(`Stop after this many steps. Default ${DEFAULT_POLICY_MAX_STEPS}.`, { + min: 1, + max: 100, + }), + minConfidence: numberField( + `Escalate instead of acting below this confidence. Default ${DEFAULT_POLICY_MIN_CONFIDENCE}.`, + { min: 0, max: 1 }, + ), + inputs: stringArrayField( + 'Text the loop may enter, as key=value. The key matches a field identifier or label.', + ), +}); + +const suggestCliSchema = { + usageOverride: 'suggest [--policy ]', + usageFlags: [], + listUsageOverride: 'suggest ', + positionalArgs: ['goal'], + allowedFlags: ['policy'], +} as const satisfies CommandSchemaOverride; + +const actCliSchema = { + usageOverride: + 'act [--policy ] [--max-steps ] [--min-confidence ] [--input ]', + usageFlags: [], + listUsageOverride: 'act ', + positionalArgs: ['goal'], + allowedFlags: ['policy', 'batchMaxSteps', 'minConfidence', 'policyInput'], +} as const satisfies CommandSchemaOverride; + +export const suggestCliReader: CliReader = (positionals, flags) => ({ + ...commonInputFromFlags(flags), + goal: readGoal(positionals[0], SUGGEST_COMMAND_NAME), + policy: flags.policy, +}); + +export const actCliReader: CliReader = (positionals, flags) => ({ + ...commonInputFromFlags(flags), + goal: readGoal(positionals[0], ACT_COMMAND_NAME), + policy: flags.policy, + maxSteps: flags.batchMaxSteps, + minConfidence: flags.minConfidence, + inputs: flags.policyInput, +}); + +type SuggestInput = CommonCommandInput & { goal: string; policy?: string }; +type ActInput = SuggestInput & { + maxSteps?: number; + minConfidence?: number; + inputs?: string[]; +}; + +export const suggestCommandFacet = defineCommandFacet({ + name: SUGGEST_COMMAND_NAME, + text: { + summary: 'Ask a policy head for the next element to act on', + cliDetail: [ + `Prints the chosen ref, the probability it carries, and whether the policy considers the goal done or blocked. It never touches the device beyond one interactive snapshot, so it is safe to run repeatedly while checking a flow by hand.`, + '', + 'Examples:', + ` agent-device suggest "sign in with the test phone number"`, + ` agent-device suggest "open the first item" --json`, + ].join('\n'), + }, + metadata: suggestCommandMetadata, + run: (client, input) => client.policy.suggest(toSuggestOptions(input as SuggestInput)), + cliSchema: suggestCliSchema, + cliReader: suggestCliReader, + cliOutputFormatter: interactionCliOutputFormatters.suggest, +}); + +export const actCommandFacet = defineCommandFacet({ + name: ACT_COMMAND_NAME, + text: { + summary: 'Drive a goal with a policy head until it is done or blocked', + cliDetail: [ + `Each step snapshots, asks the policy, acts, and re-snapshots. An action that leaves the screen digest unchanged is recorded as a dead action; three unproductive steps in a row end the run as escalated. Per-step snapshot, decide, and action milliseconds are in the result, as is the policy token cost. Supply text with --input key=value, or keep a secret out of argv by exporting ${POLICY_INPUT_ENV_PREFIX} instead.`, + '', + 'Examples:', + ` agent-device act "sign in and reach the main list" --input phone=5555550100 --max-steps 10`, + ` agent-device act "open the first item" --min-confidence 0.6 --json`, + ].join('\n'), + }, + metadata: actCommandMetadata, + run: (client, input) => client.policy.act(toActOptions(input as ActInput)), + cliSchema: actCliSchema, + cliReader: actCliReader, + cliOutputFormatter: interactionCliOutputFormatters.act, +}); + +function toSuggestOptions(input: SuggestInput) { + return { ...commonToClientOptions(input), goal: input.goal, policy: input.policy }; +} + +function toActOptions(input: ActInput) { + return { + ...toSuggestOptions(input), + maxSteps: input.maxSteps, + minConfidence: input.minConfidence, + inputs: input.inputs, + }; +} + +function readGoal(value: string | undefined, command: string): string { + const goal = value?.trim(); + if (!goal) { + throw new AppError( + 'INVALID_ARGS', + `${command} needs a goal, for example: ${command} "sign in"`, + ); + } + return goal; +} + export const interactionCommandFamily = defineCommandFamilyFromFacets({ name: 'interaction', clientSurface: false, @@ -364,6 +528,8 @@ export const interactionCommandFamily = defineCommandFamilyFromFacets({ isCommandFacet, findCommandFacet, gestureCommandFacet, + suggestCommandFacet, + actCommandFacet, ], }); diff --git a/src/commands/interaction/output.ts b/src/commands/interaction/output.ts index 5bb19b46c8..0c631d1fa0 100644 --- a/src/commands/interaction/output.ts +++ b/src/commands/interaction/output.ts @@ -1,6 +1,12 @@ import type { CommandRequestResult } from '@agent-device/contracts/client'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; import type { CliOutput } from '../command-contract.ts'; +// Type-only, so the startup closure never reaches the policy runtime through this module. +import type { + PolicyActResult, + PolicyDecision, + PolicySuggestResult, +} from '../policy/policy-contract.ts'; import { displayLabel, formatRole } from '@agent-device/capture-kit/snapshot-lines'; import { readCommandMessage } from '@agent-device/kernel/success-text'; import { @@ -82,22 +88,83 @@ function tapCliOutput(result: CommandRequestResult): CliOutput { // #1652: settle-capable entries (click, press, fill, longpress, hover, scroll) // get the warning/settle notes appended by the trait-derived wrapper; the rest // of the map is returned untouched. -export const interactionCliOutputFormatters = withSettleCapableNotes({ - click: resultOutput(tapCliOutput), - press: resultOutput(tapCliOutput), - fill: messageOutput, - longpress: messageOutput, - hover: messageOutput, - scroll: messageOutput, - get: ({ input, result }) => - getCliOutput({ - result: result as CommandRequestResult, - format: input.format as Parameters[0]['format'], - }), - is: resultOutput(isCliOutput), - find: resultOutput(findCliOutput), -} satisfies Record); +export const interactionCliOutputFormatters = { + ...withSettleCapableNotes({ + click: resultOutput(tapCliOutput), + press: resultOutput(tapCliOutput), + fill: messageOutput, + longpress: messageOutput, + hover: messageOutput, + scroll: messageOutput, + get: ({ input, result }) => + getCliOutput({ + result: result as CommandRequestResult, + format: input.format as Parameters[0]['format'], + }), + is: resultOutput(isCliOutput), + find: resultOutput(findCliOutput), + // Policy results carry no settle diff, so they are added outside the settle-note wrapper. + } satisfies Record), + suggest: resultOutput(suggestCliOutput), + act: resultOutput(actCliOutput), +} satisfies Record; function defaultCommandCliOutput(result: CommandRequestResult): CliOutput { return messageCliOutput(result as Record); } + +function suggestCliOutput(result: PolicySuggestResult): CliOutput { + const lines = [ + `Goal: ${result.goal}`, + `Screen: ${result.screen}`, + decisionLine(result.decision), + topProbabilities(result.decision), + `Timing: snapshot ${ms(result.snapshotMs)} decide ${ms(result.decision.decideMs)}`, + `Cost: ${usd(result.decision.costUsd)} for ${result.decision.inputTokens} input tokens`, + ]; + return { data: result, text: lines.filter(Boolean).join('\n') }; +} + +function actCliOutput(result: PolicyActResult): CliOutput { + const lines = [`Goal: ${result.goal}`, `Status: ${result.status}`]; + for (const step of result.steps) { + const performed = step.performed + ? ` ${step.performed.action} ${step.performed.target}${step.performed.entry === 'keypad' ? ' via keypad' : ''}` + : ''; + lines.push( + `${String(step.step).padStart(2)} ${step.outcome.padEnd(12)}${performed} conf ${step.decision.confidence.toFixed(2)} | ${step.screen}`, + ); + if (step.reason) lines.push(` ${step.reason}`); + } + const totals = result.totals; + lines.push( + `Steps: ${totals.steps}, dead actions ${totals.deadActions}, escalations ${totals.escalations}`, + `Timing: snapshot ${ms(totals.snapshotMs)} decide ${ms(totals.decideMs)} action ${ms(totals.actionMs)}`, + `Cost: ${usd(totals.costUsd)} for ${totals.inputTokens} input tokens`, + ); + return { data: result, text: lines.join('\n') }; +} + +function decisionLine(decision: PolicyDecision): string { + const target = decision.target ?? 'none'; + const flags = [decision.done ? 'done' : '', decision.blocked ? 'blocked' : ''] + .filter(Boolean) + .join(' '); + return `Next: ${decision.action} ${target} at confidence ${decision.confidence.toFixed(2)}${flags ? ` (${flags})` : ''}`; +} + +function topProbabilities(decision: PolicyDecision): string { + const ranked = Object.entries(decision.probabilities) + .sort(([, a], [, b]) => b - a) + .slice(0, 3) + .map(([label, probability]) => `${label} ${(probability * 100).toFixed(1)}%`); + return ranked.length > 0 ? `Probabilities: ${ranked.join(', ')}` : ''; +} + +function ms(value: number): string { + return `${Math.round(value)}ms`; +} + +function usd(value: number): string { + return `$${value.toFixed(6)}`; +} diff --git a/src/commands/policy/index.test.ts b/src/commands/interaction/policy-commands.test.ts similarity index 96% rename from src/commands/policy/index.test.ts rename to src/commands/interaction/policy-commands.test.ts index 687b7b70c6..f31d0dc07d 100644 --- a/src/commands/policy/index.test.ts +++ b/src/commands/interaction/policy-commands.test.ts @@ -6,7 +6,7 @@ import { suggestCliReader, suggestCommandMetadata, } from './index.ts'; -import { createPolicyProvider } from './policy-provider.ts'; +import { createPolicyProvider } from '../policy/policy-provider.ts'; describe('policy command interface', () => { test('owns suggest and act public metadata', () => { diff --git a/src/commands/policy/act-loop.ts b/src/commands/policy/act-loop.ts index 0ff20dea83..4bcf200f07 100644 --- a/src/commands/policy/act-loop.ts +++ b/src/commands/policy/act-loop.ts @@ -38,9 +38,6 @@ export type PolicyActOptions = { sleep?: (ms: number) => Promise; }; -export const DEFAULT_POLICY_MAX_STEPS = 12; -export const DEFAULT_POLICY_MIN_CONFIDENCE = 0.4; - /** Consecutive unproductive steps tolerated before the loop hands back. */ const ESCALATION_LIMIT = 3; diff --git a/src/commands/policy/index.ts b/src/commands/policy/index.ts deleted file mode 100644 index d670115698..0000000000 --- a/src/commands/policy/index.ts +++ /dev/null @@ -1,167 +0,0 @@ -import type { CommandSchemaOverride } from '@agent-device/command-registry/command-schema'; -import { AppError } from '@agent-device/kernel/errors'; -import { commonInputFromFlags } from '../cli-grammar/common.ts'; -import type { CliReader } from '../cli-grammar/types.ts'; -import { - enumField, - numberField, - requiredField, - stringArrayField, - stringField, -} from '../command-input.ts'; -import { commonToClientOptions, type CommonCommandInput } from '../common-input-fields.ts'; -import { defineCommandFacet, defineCommandFamilyFromFacets } from '../family/types.ts'; -import { defineFieldCommandMetadata } from '../field-command-contract.ts'; -import { DEFAULT_POLICY_MAX_STEPS, DEFAULT_POLICY_MIN_CONFIDENCE } from './act-loop.ts'; -import { policyCliOutputFormatters } from './output.ts'; -import { - DEFAULT_POLICY_PROVIDER, - JEV_API_KEY_ENV, - POLICY_PROVIDER_NAMES, -} from './policy-provider.ts'; -import { POLICY_INPUT_ENV_PREFIX } from './text-inputs.ts'; - -const SUGGEST_COMMAND_NAME = 'suggest'; -const ACT_COMMAND_NAME = 'act'; - -const POLICY_PREAMBLE = `A policy head answers one question — which visible element advances the goal — as a typed decision with calibrated probabilities, in roughly the time a snapshot takes. It is optional: with no ${JEV_API_KEY_ENV} in the environment both commands refuse before touching the device, and nothing else changes.`; - -const suggestDescription = `Ask a policy head which element to act on next, and print the decision without acting. ${POLICY_PREAMBLE}`; - -const actDescription = `Run snapshot, decide, act until the policy reports the goal done, reports it blocked, or loses confidence. ${POLICY_PREAMBLE} Text is never generated. A field the policy chooses is filled from a caller-supplied entry keyed by that field's identifier or label, or from the matching environment entry when the value is a secret; a field with no entry escalates instead of being filled with a guess.`; - -const goalField = requiredField( - stringField('What the run should achieve, in one sentence, as the policy sees it.'), -); -const policyField = enumField( - POLICY_PROVIDER_NAMES, - `Policy head to ask. Default ${DEFAULT_POLICY_PROVIDER}.`, -); - -export const suggestCommandMetadata = defineFieldCommandMetadata( - SUGGEST_COMMAND_NAME, - suggestDescription, - { goal: goalField, policy: policyField }, -); - -export const actCommandMetadata = defineFieldCommandMetadata(ACT_COMMAND_NAME, actDescription, { - goal: goalField, - policy: policyField, - maxSteps: numberField(`Stop after this many steps. Default ${DEFAULT_POLICY_MAX_STEPS}.`, { - min: 1, - max: 100, - }), - minConfidence: numberField( - `Escalate instead of acting below this confidence. Default ${DEFAULT_POLICY_MIN_CONFIDENCE}.`, - { min: 0, max: 1 }, - ), - inputs: stringArrayField( - 'Text the loop may enter, as key=value. The key matches a field identifier or label.', - ), -}); - -const suggestCliSchema = { - usageOverride: 'suggest [--policy ]', - usageFlags: [], - listUsageOverride: 'suggest ', - positionalArgs: ['goal'], - allowedFlags: ['policy'], -} as const satisfies CommandSchemaOverride; - -const actCliSchema = { - usageOverride: - 'act [--policy ] [--max-steps ] [--min-confidence ] [--input ]', - usageFlags: [], - listUsageOverride: 'act ', - positionalArgs: ['goal'], - allowedFlags: ['policy', 'batchMaxSteps', 'minConfidence', 'policyInput'], -} as const satisfies CommandSchemaOverride; - -export const suggestCliReader: CliReader = (positionals, flags) => ({ - ...commonInputFromFlags(flags), - goal: readGoal(positionals[0], SUGGEST_COMMAND_NAME), - policy: flags.policy, -}); - -export const actCliReader: CliReader = (positionals, flags) => ({ - ...commonInputFromFlags(flags), - goal: readGoal(positionals[0], ACT_COMMAND_NAME), - policy: flags.policy, - maxSteps: flags.batchMaxSteps, - minConfidence: flags.minConfidence, - inputs: flags.policyInput, -}); - -type SuggestInput = CommonCommandInput & { goal: string; policy?: string }; -type ActInput = SuggestInput & { - maxSteps?: number; - minConfidence?: number; - inputs?: string[]; -}; - -const suggestCommandFacet = defineCommandFacet({ - name: SUGGEST_COMMAND_NAME, - text: { - summary: 'Ask a policy head for the next element to act on', - cliDetail: [ - `Prints the chosen ref, the probability it carries, and whether the policy considers the goal done or blocked. It never touches the device beyond one interactive snapshot, so it is safe to run repeatedly while checking a flow by hand.`, - '', - 'Examples:', - ` agent-device suggest "sign in with the test phone number"`, - ` agent-device suggest "open the first item" --json`, - ].join('\n'), - }, - metadata: suggestCommandMetadata, - run: (client, input) => client.policy.suggest(toSuggestOptions(input as SuggestInput)), - cliSchema: suggestCliSchema, - cliReader: suggestCliReader, - cliOutputFormatter: policyCliOutputFormatters.suggest, -}); - -const actCommandFacet = defineCommandFacet({ - name: ACT_COMMAND_NAME, - text: { - summary: 'Drive a goal with a policy head until it is done or blocked', - cliDetail: [ - `Each step snapshots, asks the policy, acts, and re-snapshots. An action that leaves the screen digest unchanged is recorded as a dead action; three unproductive steps in a row end the run as escalated. Per-step snapshot, decide, and action milliseconds are in the result, as is the policy token cost. Supply text with --input key=value, or keep a secret out of argv by exporting ${POLICY_INPUT_ENV_PREFIX} instead.`, - '', - 'Examples:', - ` agent-device act "sign in and reach the main list" --input phone=5555550100 --max-steps 10`, - ` agent-device act "open the first item" --min-confidence 0.6 --json`, - ].join('\n'), - }, - metadata: actCommandMetadata, - run: (client, input) => client.policy.act(toActOptions(input as ActInput)), - cliSchema: actCliSchema, - cliReader: actCliReader, - cliOutputFormatter: policyCliOutputFormatters.act, -}); - -export const policyCommandFamily = defineCommandFamilyFromFacets({ - name: 'policy', - commands: [suggestCommandFacet, actCommandFacet], -}); - -function toSuggestOptions(input: SuggestInput) { - return { ...commonToClientOptions(input), goal: input.goal, policy: input.policy }; -} - -function toActOptions(input: ActInput) { - return { - ...toSuggestOptions(input), - maxSteps: input.maxSteps, - minConfidence: input.minConfidence, - inputs: input.inputs, - }; -} - -function readGoal(value: string | undefined, command: string): string { - const goal = value?.trim(); - if (!goal) { - throw new AppError( - 'INVALID_ARGS', - `${command} needs a goal, for example: ${command} "sign in"`, - ); - } - return goal; -} diff --git a/src/commands/policy/output.ts b/src/commands/policy/output.ts deleted file mode 100644 index 39360a70d1..0000000000 --- a/src/commands/policy/output.ts +++ /dev/null @@ -1,64 +0,0 @@ -import type { CliOutput } from '../command-contract.ts'; -import { resultOutput, type CliOutputFormatter } from '../output-common.ts'; -import type { PolicyActResult, PolicyDecision, PolicySuggestResult } from './policy-contract.ts'; - -function suggestCliOutput(result: PolicySuggestResult): CliOutput { - const lines = [ - `Goal: ${result.goal}`, - `Screen: ${result.screen}`, - decisionLine(result.decision), - topProbabilities(result.decision), - `Timing: snapshot ${ms(result.snapshotMs)} decide ${ms(result.decision.decideMs)}`, - `Cost: ${usd(result.decision.costUsd)} for ${result.decision.inputTokens} input tokens`, - ]; - return { data: result, text: lines.filter(Boolean).join('\n') }; -} - -function actCliOutput(result: PolicyActResult): CliOutput { - const lines = [`Goal: ${result.goal}`, `Status: ${result.status}`]; - for (const step of result.steps) { - const performed = step.performed - ? ` ${step.performed.action} ${step.performed.target}${step.performed.entry === 'keypad' ? ' via keypad' : ''}` - : ''; - lines.push( - `${String(step.step).padStart(2)} ${step.outcome.padEnd(12)}${performed} conf ${step.decision.confidence.toFixed(2)} | ${step.screen}`, - ); - if (step.reason) lines.push(` ${step.reason}`); - } - const totals = result.totals; - lines.push( - `Steps: ${totals.steps}, dead actions ${totals.deadActions}, escalations ${totals.escalations}`, - `Timing: snapshot ${ms(totals.snapshotMs)} decide ${ms(totals.decideMs)} action ${ms(totals.actionMs)}`, - `Cost: ${usd(totals.costUsd)} for ${totals.inputTokens} input tokens`, - ); - return { data: result, text: lines.join('\n') }; -} - -export const policyCliOutputFormatters = { - suggest: resultOutput(suggestCliOutput), - act: resultOutput(actCliOutput), -} as const satisfies Record; - -function decisionLine(decision: PolicyDecision): string { - const target = decision.target ?? 'none'; - const flags = [decision.done ? 'done' : '', decision.blocked ? 'blocked' : ''] - .filter(Boolean) - .join(' '); - return `Next: ${decision.action} ${target} at confidence ${decision.confidence.toFixed(2)}${flags ? ` (${flags})` : ''}`; -} - -function topProbabilities(decision: PolicyDecision): string { - const ranked = Object.entries(decision.probabilities) - .sort(([, a], [, b]) => b - a) - .slice(0, 3) - .map(([label, probability]) => `${label} ${(probability * 100).toFixed(1)}%`); - return ranked.length > 0 ? `Probabilities: ${ranked.join(', ')}` : ''; -} - -function ms(value: number): string { - return `${Math.round(value)}ms`; -} - -function usd(value: number): string { - return `$${value.toFixed(6)}`; -} diff --git a/src/commands/policy/policy-provider.ts b/src/commands/policy/policy-provider.ts index 33891f5429..479e27b517 100644 --- a/src/commands/policy/policy-provider.ts +++ b/src/commands/policy/policy-provider.ts @@ -1,15 +1,15 @@ import { AppError } from '@agent-device/kernel/errors'; -import { createJevPolicyProvider, JEV_PROVIDER_NAME } from './jev-policy-provider.ts'; +import { + POLICY_API_KEY_ENV, + POLICY_PROVIDER_NAMES, +} from '@agent-device/command-registry/flag-definitions-workflow'; +import { createJevPolicyProvider } from './jev-policy-provider.ts'; import { POLICY_FALLBACK_HINT } from './policy-fallback.ts'; import type { PolicyProvider } from './policy-contract.ts'; -/** Every policy head the CLI can resolve. One ships today. */ -export const POLICY_PROVIDER_NAMES = [JEV_PROVIDER_NAME] as const; export type PolicyProviderName = (typeof POLICY_PROVIDER_NAMES)[number]; -export const DEFAULT_POLICY_PROVIDER: PolicyProviderName = JEV_PROVIDER_NAME; -/** Environment keys the jev provider reads. The key is never accepted as a CLI flag. */ -export const JEV_API_KEY_ENV = 'TYPESAFE_API_KEY'; +/** Environment keys the jev provider reads beyond its credential. */ const JEV_MODEL_ENV = 'AGENT_DEVICE_POLICY_MODEL'; const JEV_ENDPOINT_ENV = 'AGENT_DEVICE_POLICY_ENDPOINT'; @@ -37,14 +37,14 @@ export function createPolicyProvider( `Unknown policy provider ${name}; available: ${POLICY_PROVIDER_NAMES.join(', ')}`, ); } - const apiKey = env[JEV_API_KEY_ENV]?.trim(); + const apiKey = env[POLICY_API_KEY_ENV]?.trim(); if (!apiKey) { throw new AppError( 'INVALID_ARGS', - `${JEV_API_KEY_ENV} is not set, so policy ${name} cannot decide`, + `${POLICY_API_KEY_ENV} is not set, so policy ${name} cannot decide`, { reason: 'policy-provider-unconfigured', - hint: `Export ${JEV_API_KEY_ENV} to enable the policy head, or ${POLICY_FALLBACK_HINT}`, + hint: `Export ${POLICY_API_KEY_ENV} to enable the policy head, or ${POLICY_FALLBACK_HINT}`, }, ); } diff --git a/src/commands/policy/text-inputs.ts b/src/commands/policy/text-inputs.ts index a9a3f01d8c..2879cdfdc7 100644 --- a/src/commands/policy/text-inputs.ts +++ b/src/commands/policy/text-inputs.ts @@ -1,3 +1,4 @@ +import { POLICY_INPUT_ENV_PREFIX } from '@agent-device/command-registry/flag-definitions-workflow'; import { AppError } from '@agent-device/kernel/errors'; import type { PolicyCandidate } from './policy-contract.ts'; @@ -7,9 +8,6 @@ import type { PolicyCandidate } from './policy-contract.ts'; */ export type PolicyTextInputs = ReadonlyMap; -/** Prefix for per-key environment entries, so a secret never has to appear in argv. */ -export const POLICY_INPUT_ENV_PREFIX = 'AGENT_DEVICE_INPUT_'; - /** Parse repeated `--input key=value` tokens. An empty value is allowed; an empty key is not. */ export function parsePolicyTextInputs(tokens: readonly string[]): Map { const inputs = new Map(); From 9af1dc0065a477a87c5367e635aa340e3da88f3a Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 19:33:30 -0400 Subject: [PATCH 13/17] fix(policy): point the command owner files at their new home The move left both descriptors naming a deleted module, which the explain-command table caught, and left three exports with no consumers. Co-Authored-By: Claude Opus 5 (1M context) --- packages/command-registry/src/registry.ts | 11 +++++++++-- src/commands/interaction/index.ts | 4 ++-- src/commands/policy/jev-policy-provider.ts | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/command-registry/src/registry.ts b/packages/command-registry/src/registry.ts index c88e436381..8b138b462d 100644 --- a/packages/command-registry/src/registry.ts +++ b/packages/command-registry/src/registry.ts @@ -1543,7 +1543,7 @@ export const RAW_COMMAND_DESCRIPTORS = [ { name: 'suggest', deviceClaimPolicy: 'none', - ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/policy/index.ts'] as const } : {}), + ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/interaction/index.ts'] as const } : {}), catalog: { group: 'local-cli' }, recordsSessionAction: false, timeoutPolicy: DEFAULT_TIMEOUT_POLICY, @@ -1553,7 +1553,14 @@ export const RAW_COMMAND_DESCRIPTORS = [ { name: 'act', deviceClaimPolicy: 'none', - ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/policy/index.ts'] as const } : {}), + ...(ownerFilesEnabled + ? { + ownerFiles: [ + 'src/commands/interaction/index.ts', + 'src/commands/policy/act-loop.ts', + ] as const, + } + : {}), catalog: { group: 'local-cli' }, recordsSessionAction: false, // A run is a sequence of ordinary commands, each already under its own envelope; a client diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index 3656cc5edb..742faa0eaa 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -449,7 +449,7 @@ type ActInput = SuggestInput & { inputs?: string[]; }; -export const suggestCommandFacet = defineCommandFacet({ +const suggestCommandFacet = defineCommandFacet({ name: SUGGEST_COMMAND_NAME, text: { summary: 'Ask a policy head for the next element to act on', @@ -468,7 +468,7 @@ export const suggestCommandFacet = defineCommandFacet({ cliOutputFormatter: interactionCliOutputFormatters.suggest, }); -export const actCommandFacet = defineCommandFacet({ +const actCommandFacet = defineCommandFacet({ name: ACT_COMMAND_NAME, text: { summary: 'Drive a goal with a policy head until it is done or blocked', diff --git a/src/commands/policy/jev-policy-provider.ts b/src/commands/policy/jev-policy-provider.ts index 523f02d85d..30b0cec691 100644 --- a/src/commands/policy/jev-policy-provider.ts +++ b/src/commands/policy/jev-policy-provider.ts @@ -3,7 +3,7 @@ import { buildJevRequest, readJevDecision } from './jev-composer.ts'; import { POLICY_FALLBACK_HINT } from './policy-fallback.ts'; import type { PolicyDecision, PolicyProvider, PolicyRequest } from './policy-contract.ts'; -export const JEV_PROVIDER_NAME = 'jev'; +const JEV_PROVIDER_NAME = 'jev'; const JEV_DEFAULT_ENDPOINT = 'https://api.typesafe.ai/v1/systemone'; const JEV_DEFAULT_MODEL = 'jev-latest'; const JEV_TIMEOUT_MS = 20_000; From 38d8edf8eda1bdf4a769f75bb29eef0189303c51 Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 19:37:45 -0400 Subject: [PATCH 14/17] docs(policy): explain the formatter placement where the structure changes Co-Authored-By: Claude Opus 5 (1M context) --- src/commands/interaction/output.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/interaction/output.ts b/src/commands/interaction/output.ts index 0c631d1fa0..43cf0ba66e 100644 --- a/src/commands/interaction/output.ts +++ b/src/commands/interaction/output.ts @@ -88,6 +88,7 @@ function tapCliOutput(result: CommandRequestResult): CliOutput { // #1652: settle-capable entries (click, press, fill, longpress, hover, scroll) // get the warning/settle notes appended by the trait-derived wrapper; the rest // of the map is returned untouched. +// The policy commands sit outside the settle-note wrapper: their results carry no settle diff. export const interactionCliOutputFormatters = { ...withSettleCapableNotes({ click: resultOutput(tapCliOutput), @@ -103,7 +104,6 @@ export const interactionCliOutputFormatters = { }), is: resultOutput(isCliOutput), find: resultOutput(findCliOutput), - // Policy results carry no settle diff, so they are added outside the settle-note wrapper. } satisfies Record), suggest: resultOutput(suggestCliOutput), act: resultOutput(actCliOutput), From d73b179fb10f24b1d6dc0072d6c5d8e61aede3cb Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 19:43:22 -0400 Subject: [PATCH 15/17] test(policy): cover the ref-pinning port and the two output formatters The changed-line coverage gate passed at 83%, but the two largest gaps were the seam that pins a mutation to its snapshot generation and the text both commands print. A run that loses the pin fails silently, on whichever element inherited the ref, so it is worth its own test rather than coverage by a live run. Co-Authored-By: Claude Opus 5 (1M context) --- src/client/client-policy.test.ts | 89 +++++++++++++++++- src/client/client-policy.ts | 6 +- src/commands/interaction/output.test.ts | 115 ++++++++++++++++++++++++ 3 files changed, 207 insertions(+), 3 deletions(-) diff --git a/src/client/client-policy.test.ts b/src/client/client-policy.test.ts index 02682acc8a..1eb95f6a2f 100644 --- a/src/client/client-policy.test.ts +++ b/src/client/client-policy.test.ts @@ -1,5 +1,10 @@ import { describe, expect, test } from 'vitest'; -import { runPolicyAct, suggestPolicyAction, type PolicyClientCalls } from './client-policy.ts'; +import { + createPolicyDevicePort, + runPolicyAct, + suggestPolicyAction, + type PolicyClientCalls, +} from './client-policy.ts'; /** A device surface that fails the test if the commands reach it. */ function forbiddenDevice(): PolicyClientCalls & { touched: string[] } { @@ -78,3 +83,85 @@ describe('the rest of the command surface without a key', () => { ).toHaveLength(2); }); }); + +describe("the loop's device port", () => { + /** Records what the port sent to the client, with the snapshot generations it was handed. */ + function recordingCalls(generations: readonly (number | undefined)[]): PolicyClientCalls & { + sent: string[]; + } { + const sent: string[] = []; + let index = 0; + return { + sent, + snapshot: async () => { + const refsGeneration = generations[Math.min(index++, generations.length - 1)]; + sent.push(`snapshot~${refsGeneration ?? 'none'}`); + return { nodes: [], ...(refsGeneration === undefined ? {} : { refsGeneration }) }; + }, + press: async ({ ref }) => { + sent.push(`press ${ref}`); + }, + fill: async ({ ref, text }) => { + sent.push(`fill ${ref} ${text}`); + }, + }; + } + + test('pins a mutation to the generation of the snapshot that issued its ref', async () => { + const calls = recordingCalls([7]); + const port = createPolicyDevicePort(calls, {}); + await port.snapshot(); + await port.press('@e12'); + await port.fill('@e12', 'hello'); + expect(calls.sent).toEqual(['snapshot~7', 'press @e12~s7', 'fill @e12~s7 hello']); + }); + + test('re-pins after a later snapshot reissues refs', async () => { + const calls = recordingCalls([7, 8]); + const port = createPolicyDevicePort(calls, {}); + await port.snapshot(); + await port.press('@e12'); + await port.snapshot(); + await port.press('@e12'); + expect(calls.sent).toEqual(['snapshot~7', 'press @e12~s7', 'snapshot~8', 'press @e12~s8']); + }); + + test('sends a plain ref when the daemon issued no generation', async () => { + const calls = recordingCalls([undefined]); + const port = createPolicyDevicePort(calls, {}); + await port.snapshot(); + await port.press('e12'); + expect(calls.sent).toEqual(['snapshot~none', 'press @e12']); + }); + + test('reports a duration for every operation', async () => { + const port = createPolicyDevicePort(recordingCalls([1]), {}); + const snapshot = await port.snapshot(); + expect(snapshot.ms).toBeGreaterThanOrEqual(0); + expect(await port.press('@e1')).toBeGreaterThanOrEqual(0); + expect(await port.fill('@e1', 'x')).toBeGreaterThanOrEqual(0); + }); + + test("carries the caller's device selection into every call", async () => { + const seen: Array> = []; + const port = createPolicyDevicePort( + { + snapshot: async (options) => { + seen.push(options); + return { nodes: [] }; + }, + press: async (options) => { + seen.push(options); + }, + fill: async (options) => { + seen.push(options); + }, + }, + { udid: 'SIM-1' }, + ); + await port.snapshot(); + await port.press('@e1'); + expect(seen.every((options) => options.udid === 'SIM-1')).toBe(true); + expect(seen[0]).toMatchObject({ interactiveOnly: true }); + }); +}); diff --git a/src/client/client-policy.ts b/src/client/client-policy.ts index 0c47d25a88..491a3a7896 100644 --- a/src/client/client-policy.ts +++ b/src/client/client-policy.ts @@ -98,9 +98,11 @@ export async function runPolicyAct( * * Every mutation is pinned to the generation of the snapshot that issued its ref (ADR 0014), so a * decision made against a screen that has since changed is refused by the daemon rather than - * landing on whatever element inherited the ref. + * landing on whatever element inherited the ref. Exported because that pinning rule is the seam + * worth testing on its own: a run that loses it fails silently, on whichever element inherited + * the ref. */ -function createPolicyDevicePort( +export function createPolicyDevicePort( calls: PolicyClientCalls, common: PolicyCommonOptions, ): PolicyDevicePort { diff --git a/src/commands/interaction/output.test.ts b/src/commands/interaction/output.test.ts index 1acbab2eef..168bc7bac2 100644 --- a/src/commands/interaction/output.test.ts +++ b/src/commands/interaction/output.test.ts @@ -309,3 +309,118 @@ describe('longpress CLI output', () => { ); }); }); + +describe('policy command output', () => { + const decision = { + action: 'fill' as const, + target: '@e7', + done: false, + blocked: true, + needsText: true, + confidence: 0.99, + probabilities: { '@e7': 0.99, '@e9': 0.006, __none__: 0.004 }, + provider: 'jev', + model: 'jev-1.13.0', + decideMs: 198, + inputTokens: 412, + costUsd: 0.000017304, + }; + + const formatSuggest = (result: unknown) => + interactionCliOutputFormatters.suggest({ input: {}, result } as never); + const formatAct = (result: unknown) => + interactionCliOutputFormatters.act({ input: {}, result } as never); + + test('suggest prints the decision, its top probabilities, timing, and cost', async () => { + const output = await formatSuggest({ + goal: 'sign in', + decision, + candidates: [], + snapshotMs: 412, + screen: 'Phone number | Send code', + }); + expect(output.text).toBe( + [ + 'Goal: sign in', + 'Screen: Phone number | Send code', + 'Next: fill @e7 at confidence 0.99 (blocked)', + 'Probabilities: @e7 99.0%, @e9 0.6%, __none__ 0.4%', + 'Timing: snapshot 412ms decide 198ms', + 'Cost: $0.000017 for 412 input tokens', + ].join('\n'), + ); + }); + + test('suggest names both flags when the policy calls the goal done and blocked', async () => { + const output = await formatSuggest({ + goal: 'sign in', + decision: { ...decision, done: true, blocked: true }, + candidates: [], + snapshotMs: 1, + screen: 'Shelf', + }); + expect(output.text).toContain('(done blocked)'); + }); + + test('suggest reports a decline as no target', async () => { + const output = await formatSuggest({ + goal: 'sign in', + decision: { ...decision, action: 'none', target: null, blocked: false }, + candidates: [], + snapshotMs: 1, + screen: 'Shelf', + }); + expect(output.text).toContain('Next: none none at confidence 0.99'); + }); + + test('act prints one line per step, the reason for a step that stalled, and the totals', async () => { + const output = await formatAct({ + goal: 'sign in', + status: 'escalated', + steps: [ + { + step: 1, + screen: 'Phone number', + decision, + outcome: 'acted', + performed: { action: 'fill', target: '@e7', entry: 'keypad', inputKey: 'phone' }, + snapshotMs: 400, + decideMs: 198, + actionMs: 900, + }, + { + step: 2, + screen: 'Phone number', + decision, + outcome: 'escalated', + snapshotMs: 400, + decideMs: 198, + actionMs: 0, + reason: 'policy chose no element on this screen', + }, + ], + totals: { + steps: 2, + snapshotMs: 800, + decideMs: 396, + actionMs: 900, + inputTokens: 824, + costUsd: 0.000034608, + deadActions: 0, + escalations: 1, + }, + }); + expect(output.text).toBe( + [ + 'Goal: sign in', + 'Status: escalated', + ' 1 acted fill @e7 via keypad conf 0.99 | Phone number', + ' 2 escalated conf 0.99 | Phone number', + ' policy chose no element on this screen', + 'Steps: 2, dead actions 0, escalations 1', + 'Timing: snapshot 800ms decide 396ms action 900ms', + 'Cost: $0.000035 for 824 input tokens', + ].join('\n'), + ); + }); +}); From 6862da66cc7e4cae5bebc0b9a79977f8046adfc3 Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 19:56:55 -0400 Subject: [PATCH 16/17] docs(policy): state what the credential actually gates "Gated on TYPESAFE_API_KEY" read as though the commands disappear without one. They do not: the catalog and the MCP tool list are the same either way, because neither reads the environment. What the credential gates is whether a run proceeds. The surface-identity test now asserts exactly that, by building both projections with the key set and unset and comparing them, and by pinning that neither command owns a daemon route. Co-Authored-By: Claude Opus 5 (1M context) --- skills/agent-device/SKILL.md | 2 +- src/client/client-policy.test.ts | 18 ------ src/commands/interaction/index.ts | 2 +- .../interaction/policy-commands.test.ts | 58 +++++++++++++++++++ website/docs/docs/commands.md | 6 +- website/docs/docs/policy-head.md | 7 ++- 6 files changed, 69 insertions(+), 24 deletions(-) diff --git a/skills/agent-device/SKILL.md b/skills/agent-device/SKILL.md index de243b483e..496588aba9 100644 --- a/skills/agent-device/SKILL.md +++ b/skills/agent-device/SKILL.md @@ -19,6 +19,6 @@ Reaching an off-screen target is one command, not a scroll-and-check loop: `scro Copy refs byte-for-byte: `@e12`, `@e12~s4` — keep the `@` and any `~sN`. Prefer current refs, then `id`/`label`/`role` selectors; coordinates are a last resort. If snapshot reports sparse/AX-unavailable, its refs and selectors are invalid: run `agent-device screenshot`, inspect the image, use coordinates, then retry `snapshot -i` after navigating. Otherwise run `snapshot -i` only when the diff lacks the next target. -With `TYPESAFE_API_KEY` set, `suggest ""` names the next element to act on in roughly the time a snapshot takes, and `act "" --input key=value` runs the whole snapshot-decide-act loop. Text is never generated; a field with no `--input` entry escalates. Without the key both refuse, and the loop above is the path. On `Jev unavailable: `, fall back to that loop. +With `TYPESAFE_API_KEY` set, `suggest ""` names the next element to act on in roughly the time a snapshot takes, and `act "" --input key=value` runs the whole snapshot-decide-act loop. Text is never generated; a field with no `--input` entry escalates. Both are always listed; without the key each refuses with a typed error and the loop above is the path. On `Jev unavailable: `, fall back to that loop. Error output includes corrective hints; follow them instead of re-planning. Only when the task is specialized (for example gestures, scripting, TV, macOS, remote, or debugging) or a command shape is unclear, run `agent-device help `. `agent-device --help` lists topics, but is not a startup step. diff --git a/src/client/client-policy.test.ts b/src/client/client-policy.test.ts index 1eb95f6a2f..3eb72a898d 100644 --- a/src/client/client-policy.test.ts +++ b/src/client/client-policy.test.ts @@ -66,24 +66,6 @@ describe('policy commands without a key', () => { }); }); -describe('the rest of the command surface without a key', () => { - test('every other command keeps its descriptor untouched by the policy feature', async () => { - const { commandDescriptors } = await import('@agent-device/command-registry/registry'); - const policyNames = new Set(['suggest', 'act']); - const others = commandDescriptors.filter((descriptor) => !policyNames.has(descriptor.name)); - - // The feature adds two descriptors and changes none: anything that reads the registry at - // startup behaves identically whether or not a key is present, because no existing command - // gained a policy field. - for (const descriptor of others) { - expect(JSON.stringify(descriptor)).not.toContain('policy'); - } - expect( - commandDescriptors.filter((descriptor) => policyNames.has(descriptor.name)), - ).toHaveLength(2); - }); -}); - describe("the loop's device port", () => { /** Records what the port sent to the client, with the snapshot generations it was handed. */ function recordingCalls(generations: readonly (number | undefined)[]): PolicyClientCalls & { diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index 742faa0eaa..7728b9432e 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -374,7 +374,7 @@ const gestureCommandFacet = defineCommandFacet({ const SUGGEST_COMMAND_NAME = 'suggest'; const ACT_COMMAND_NAME = 'act'; -const POLICY_PREAMBLE = `A policy head answers one question — which visible element advances the goal — as a typed decision with calibrated probabilities, in roughly the time a snapshot takes. It is optional: with no ${POLICY_API_KEY_ENV} in the environment both commands refuse before touching the device, and nothing else changes.`; +const POLICY_PREAMBLE = `A policy head answers one question — which visible element advances the goal — as a typed decision with calibrated probabilities, in roughly the time a snapshot takes. It is optional: with no ${POLICY_API_KEY_ENV} in the environment this command refuses before touching the device, and nothing else changes.`; const suggestDescription = `Ask a policy head which element to act on next, and print the decision without acting. ${POLICY_PREAMBLE}`; diff --git a/src/commands/interaction/policy-commands.test.ts b/src/commands/interaction/policy-commands.test.ts index f31d0dc07d..6dfc581af6 100644 --- a/src/commands/interaction/policy-commands.test.ts +++ b/src/commands/interaction/policy-commands.test.ts @@ -7,6 +7,12 @@ import { suggestCommandMetadata, } from './index.ts'; import { createPolicyProvider } from '../policy/policy-provider.ts'; +import { listCliCommandNames } from '@agent-device/command-registry/catalog'; +import { + commandDescriptors, + listMcpExposedCommandNames, +} from '@agent-device/command-registry/registry'; +import { POLICY_API_KEY_ENV } from '@agent-device/command-registry/flag-definitions-workflow'; describe('policy command interface', () => { test('owns suggest and act public metadata', () => { @@ -68,3 +74,55 @@ describe('policy provider resolution', () => { expect(createPolicyProvider('jev', { TYPESAFE_API_KEY: 'k' }).name).toBe('jev'); }); }); + +/** + * The surface is identical with and without a credential: the commands are always listed and + * always offered over MCP, and refuse at run time. Nothing here reads the environment, so the + * assertions run the projections twice with the key set and unset and compare. + */ +describe('the command surface does not depend on the key', () => { + const POLICY_COMMANDS = ['suggest', 'act'] as const; + + function withKey(value: string | undefined, read: () => T): T { + const previous = process.env[POLICY_API_KEY_ENV]; + if (value === undefined) delete process.env[POLICY_API_KEY_ENV]; + else process.env[POLICY_API_KEY_ENV] = value; + try { + return read(); + } finally { + if (previous === undefined) delete process.env[POLICY_API_KEY_ENV]; + else process.env[POLICY_API_KEY_ENV] = previous; + } + } + + test('both commands are in the CLI catalog whether or not the key is set', () => { + const withoutKey = withKey(undefined, listCliCommandNames); + const withKeySet = withKey('a-key', listCliCommandNames); + expect(withoutKey).toEqual(withKeySet); + for (const command of POLICY_COMMANDS) expect(withoutKey).toContain(command); + }); + + test('both commands are offered over MCP whether or not the key is set', () => { + const withoutKey = withKey(undefined, listMcpExposedCommandNames); + const withKeySet = withKey('a-key', listMcpExposedCommandNames); + expect(withoutKey).toEqual(withKeySet); + for (const command of POLICY_COMMANDS) expect(withoutKey).toContain(command); + }); + + test('neither command owns a daemon route', () => { + for (const command of POLICY_COMMANDS) { + const descriptor = commandDescriptors.find((entry) => entry.name === command); + expect(descriptor, `${command} is missing from the registry`).toBeDefined(); + expect(descriptor).not.toHaveProperty('daemon'); + } + }); + + test('no other command gained a policy field', () => { + const others = commandDescriptors.filter( + (descriptor) => !(POLICY_COMMANDS as readonly string[]).includes(descriptor.name), + ); + for (const descriptor of others) { + expect(JSON.stringify(descriptor), `${descriptor.name} changed`).not.toContain('policy'); + } + }); +}); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index d430ec9b0f..38dc685cc0 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -603,8 +603,10 @@ agent-device act "open the first item" --min-confidence 0.6 --json - A policy head answers one question — which visible element advances the goal — as a typed decision with calibrated probabilities. `suggest` prints that decision; `act` loops snapshot, decide, press or fill until the goal is done, blocked, or the policy loses confidence. -- Both commands are gated on `TYPESAFE_API_KEY`. Without it they refuse before touching the - device, and nothing else in the CLI changes. +- Both commands are always listed and always offered over MCP. Without `TYPESAFE_API_KEY` in the + environment they refuse to run, with a typed `INVALID_ARGS` naming the variable and the fallback, + before any snapshot, press or fill reaches the device. Nothing else in the CLI changes: neither + command owns a daemon route, and no existing command's schema moves. - `--policy ` selects the head; `jev` is the only one today and the default. - `act` never generates text. A field the policy chooses is filled from `--input key=value`, or from `AGENT_DEVICE_INPUT_` in the environment when the value is a secret. A field with no diff --git a/website/docs/docs/policy-head.md b/website/docs/docs/policy-head.md index a9a3c85bc4..24555c1e5b 100644 --- a/website/docs/docs/policy-head.md +++ b/website/docs/docs/policy-head.md @@ -13,8 +13,11 @@ A policy head answers only the narrow question — which of these elements advan typed decision with calibrated probabilities. It does not write text, plan, or explain. `suggest` asks it once. `act` puts it in a loop. -The feature is optional and off by default. Without `TYPESAFE_API_KEY` in the environment both -commands refuse before touching the device, and nothing else in the CLI behaves differently. +The feature is optional. Both commands are always listed by `agent-device help commands` and always +offered over MCP, because the command catalog is the same with or without a credential. What the +credential changes is whether they run: without `TYPESAFE_API_KEY` in the environment each refuses +with a typed `INVALID_ARGS` naming the variable and the fallback, before any snapshot, press or fill +reaches the device. Nothing else in the CLI behaves differently. ## Asking once From b4150c078930a8ab72d56fe2790b6a769af05258 Mon Sep 17 00:00:00 2001 From: alexneyman Date: Wed, 16 Sep 2026 20:13:56 -0400 Subject: [PATCH 17/17] test(policy): assert the rendered MCP tools, not just the exposed names The exposed-name list proves the commands are registered, not that a usable tool reaches a model. The assertion now reads the rendered tool: it takes a goal, its description names the variable to set, and no input in its schema can ever carry the credential's value. Co-Authored-By: Claude Opus 5 (1M context) --- .../interaction/policy-commands.test.ts | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/commands/interaction/policy-commands.test.ts b/src/commands/interaction/policy-commands.test.ts index 6dfc581af6..d5382cdbc6 100644 --- a/src/commands/interaction/policy-commands.test.ts +++ b/src/commands/interaction/policy-commands.test.ts @@ -8,10 +8,8 @@ import { } from './index.ts'; import { createPolicyProvider } from '../policy/policy-provider.ts'; import { listCliCommandNames } from '@agent-device/command-registry/catalog'; -import { - commandDescriptors, - listMcpExposedCommandNames, -} from '@agent-device/command-registry/registry'; +import { commandDescriptors } from '@agent-device/command-registry/registry'; +import { listMcpCommandMetadata } from '../command-metadata.ts'; import { POLICY_API_KEY_ENV } from '@agent-device/command-registry/flag-definitions-workflow'; describe('policy command interface', () => { @@ -103,10 +101,28 @@ describe('the command surface does not depend on the key', () => { }); test('both commands are offered over MCP whether or not the key is set', () => { - const withoutKey = withKey(undefined, listMcpExposedCommandNames); - const withKeySet = withKey('a-key', listMcpExposedCommandNames); - expect(withoutKey).toEqual(withKeySet); - for (const command of POLICY_COMMANDS) expect(withoutKey).toContain(command); + const toolNames = () => listMcpCommandMetadata().map((tool) => tool.name); + expect(withKey(undefined, toolNames)).toEqual(withKey('a-key', toolNames)); + for (const command of POLICY_COMMANDS) { + expect(withKey(undefined, toolNames)).toContain(command); + } + }); + + test('each MCP tool takes a goal and never takes the credential', () => { + for (const command of POLICY_COMMANDS) { + const tool = withKey(undefined, () => + listMcpCommandMetadata().find((entry) => entry.name === command), + ); + const schema = tool?.inputSchema; + expect(Object.keys(schema?.properties ?? {}), `${command} must accept a goal`).toContain( + 'goal', + ); + // The credential is environment-only: the description may name the variable so a model + // knows what to set, but no input may ever carry its value. + expect(JSON.stringify(schema)).not.toContain(POLICY_API_KEY_ENV); + expect(JSON.stringify(schema).toLowerCase()).not.toContain('apikey'); + expect(tool?.description).toContain(POLICY_API_KEY_ENV); + } }); test('neither command owns a daemon route', () => {