refactor(computer-use): enforce semantic-only actions and enable guarded keyboard input

apps/desktop/src/main/__tests__/computer-use-real-model-policy.test.ts

@@ -178,12 +178,12 @@ test('semantic mutations require an observation created by the owned fixture', a
   assert.deepEqual(calls, ['observe', 'click_element']);
 });
 
-test('wait and cursor_position do not require an impossible observation_id', async () => {
+test('wait does not require an impossible observation_id', async () => {
   const calls: string[] = [];
   const [wrapped] = applyComputerUseRealModelPolicy(toolSet(calls), {
-    allowedActions: ['wait', 'cursor_position'],
-    maxTotalActions: 2,
-    maxActionCounts: { wait: 1, cursor_position: 1 },
+    allowedActions: ['wait'],
+    maxTotalActions: 1,
+    maxActionCounts: { wait: 1 },
     allowedApps: ['Owned Fixture'],
   });
   const context = {
@@ -199,12 +199,6 @@ test('wait and cursor_position do not require an impossible observation_id', asy
     { action: 'wait', duration: 0.01 } as never,
     context,
   ) as { text: string };
-  const cursor = await wrapped.impl(
-    { action: 'cursor_position' } as never,
-    context,
-  ) as { text: string };
-
   assert.equal(wait.text, 'ok');
-  assert.equal(cursor.text, 'ok');
-  assert.deepEqual(calls, ['wait', 'cursor_position']);
+  assert.deepEqual(calls, ['wait']);
 });

apps/desktop/src/main/__tests__/runtime-host-native-capabilities.test.ts

@@ -127,10 +127,15 @@ test('publishes the real Computer Use schema through the Client Capability proto
       offers: provider.offers(),
     }),
   );
-  const coordinateSchema = provider.offers()[0]?.tools[0]?.inputSchema.properties as
-    | Record<string, { items?: unknown }>
+  const actionSchema = provider.offers()[0]?.tools[0]?.inputSchema.properties as
+    | Record<string, { enum?: unknown }>
     | undefined;
-  assert.equal(Array.isArray(coordinateSchema?.coordinate?.items), true);
+  assert.equal(
+    Array.isArray(actionSchema?.action?.enum) &&
+      actionSchema.action.enum.includes('click_element') &&
+      !actionSchema.action.enum.includes('left_click'),
+    true,
+  );
 });
 
 test('publishes every production Desktop-owned tool schema through the protocol', () => {

apps/desktop/src/main/computer-use-real-model-policy.ts

@@ -24,7 +24,6 @@ import type { MakaTool } from '@maka/runtime/tool-runtime';
 const ACTIONS_WITHOUT_OBSERVATION_OWNERSHIP = new Set([
   'list_apps',
   'wait',
-  'cursor_position',
 ]);
 
 export interface ComputerUseRealModelPolicy {

apps/desktop/src/main/computer-use/pip-feed.ts

@@ -61,7 +61,6 @@ export function withComputerUsePip<
 
 interface PipFeedResult {
   screenshot?: { base64: string; mimeType: 'image/png' | 'image/jpeg'; widthPx: number; heightPx: number };
-  resolvedScreenPoint?: { x: number; y: number };
   observation?: {
     windowTitle?: string;
     windowBounds?: { x: number; y: number; width: number; height: number };
@@ -92,14 +91,10 @@ function presentToPip(
   // pixels. Scale through the window rather than subtracting the origin alone,
   // so a Retina capture (wider than the window in points) still lands on the
   // right control instead of a quarter of the way into it.
-  // The executor reports a landing point only for the coordinate paths. An
-  // element action resolves to an element, not a pointer position, so the point
-  // it was addressed to — the element's own centre, already computed for the
-  // cursor's flight — is what the mirror draws. Without this fallback the
-  // mirror cleared its cursor at the end of every accessibility action, which
-  // is every action Maka dispatches by default: the window the user is watching
-  // showed the app being driven by nothing.
-  const point = result.resolvedScreenPoint ?? context?.presentationScreenPoint;
+  // Semantic actions resolve to elements, not pointer positions. The Runtime
+  // derives an element centre for presentation only; executor results carry no
+  // coordinate that could be mistaken for model or dispatch input.
+  const point = context?.presentationScreenPoint;
   const bounds = result.observation?.windowBounds;
   if (!point || !bounds || bounds.width <= 0 || bounds.height <= 0) {
     pip.setCursor({ sessionId });

apps/desktop/src/overlay/cursor-overlay.ts

@@ -31,11 +31,11 @@ interface MovePayload {
   actionId: string;
   x: number;
   y: number;
-  kind?: 'move' | 'click' | 'drag' | 'scroll';
+  kind?: 'click' | 'scroll';
   pressed?: boolean;
   instant?: boolean;
 }
-interface CompletePayload { actionId?: string; x: number; y: number; kind?: 'move' | 'click' | 'drag' | 'scroll'; pulse?: boolean }
+interface CompletePayload { actionId?: string; x: number; y: number; kind?: 'click' | 'scroll'; pulse?: boolean }
 interface CancelPayload { actionId: string }
 interface ResetPayload { sessionId: string; generation: number }
 declare global {

packages/computer-use/src/__tests__/computer-use-cumulative-e2e.test.ts

@@ -19,12 +19,12 @@
 
 import assert from 'node:assert/strict';
 import { describe, test } from 'node:test';
-import type { CuAction } from '@maka/core/computer-use';
 import { buildComputerUseTools } from '@maka/runtime/computer-use-tools';
 import {
   type CuDispatchBackend,
   type CuObservation,
   type CuRunContext,
+  type CuSemanticAction,
 } from '@maka/runtime/computer-use-types';
 import { parseObservationText } from '@maka/runtime/test-only/observation-text-reader';
 import {
@@ -73,6 +73,7 @@ function fixtureObservation(overrides: Partial<CuObservation> = {}): CuObservati
         elementId: '5',
         role: 'AXButton',
         label: 'Commit target',
+        frame: { x: 280, y: 185, width: 40, height: 30 },
         identity: {
           token: 'target-button-token',
           role: 'AXButton',
@@ -136,7 +137,7 @@ function overlayRecorder() {
 describe('Computer Use cross-layer deterministic contract', () => {
   test('bound target propagation, presentation order, fresh state, and duplicate rejection', async () => {
     const overlay = overlayRecorder();
-    const dispatches: Array<{ action: CuAction; context: CuRunContext }> = [];
+    const dispatches: Array<{ action: CuSemanticAction; context: CuRunContext }> = [];
     let revision = 0;
     const backend: CuDispatchBackend = {
       async preflight() {
@@ -146,7 +147,10 @@ describe('Computer Use cross-layer deterministic contract', () => {
         assert.equal(input.windowId, 10);
         return fixtureObservation();
       },
-      async run(action, _signal, runContext) {
+      async run() {
+        throw new Error('non-semantic dispatch is not expected');
+      },
+      async runSemantic(action, _signal, runContext) {
         dispatches.push({ action, context: runContext });
         assert.equal(runContext.boundAction?.target.pid, 100);
         assert.equal(runContext.boundAction?.target.windowId, 10);
@@ -158,7 +162,6 @@ describe('Computer Use cross-layer deterministic contract', () => {
             verified: true,
             evidence: { effect: 'confirmed' },
           },
-          resolvedScreenPoint: { x: 300, y: 200 },
           observation: fixtureObservation({
             observationId: `backend-observation-${revision + 1}`,
             contentFingerprint: `fixture-structure-${revision + 1}`,
@@ -182,9 +185,9 @@ describe('Computer Use cross-layer deterministic contract', () => {
     const observationId = observationIdOf(observed.modelText);
     const result = (await tool.impl(
       {
-        action: 'left_click',
+        action: 'click_element',
         observation_id: observationId,
-        coordinate: [400, 200],
+        element_id: '5',
       } as never,
       context({ toolCallId: 'click-target' }),
     )) as {
@@ -213,9 +216,9 @@ describe('Computer Use cross-layer deterministic contract', () => {
 
     const replay = (await tool.impl(
       {
-        action: 'left_click',
+        action: 'click_element',
         observation_id: observationId,
-        coordinate: [400, 200],
+        element_id: '5',
       } as never,
       context({ toolCallId: 'duplicate' }),
     )) as { text: string };
@@ -234,6 +237,9 @@ describe('Computer Use cross-layer deterministic contract', () => {
         return fixtureObservation();
       },
       async run() {
+        throw new Error('non-semantic dispatch is not expected');
+      },
+      async runSemantic() {
         return mode === 'target_change'
           ? {
               outcome: {
@@ -267,9 +273,9 @@ describe('Computer Use cross-layer deterministic contract', () => {
     )) as { modelText?: string };
     await tool.impl(
       {
-        action: 'left_click',
+        action: 'click_element',
         observation_id: observationIdOf(firstObservation.modelText),
-        coordinate: [400, 200],
+        element_id: '5',
       } as never,
       context({ toolCallId: 'target-change' }),
     );
@@ -287,9 +293,9 @@ describe('Computer Use cross-layer deterministic contract', () => {
     mode = 'unknown';
     await tool.impl(
       {
-        action: 'left_click',
+        action: 'click_element',
         observation_id: observationIdOf(secondObservation.modelText),
-        coordinate: [400, 200],
+        element_id: '5',
       } as never,
       context({ turnId: 'turn-2', toolCallId: 'unknown' }),
     );
@@ -315,6 +321,9 @@ describe('Computer Use cross-layer deterministic contract', () => {
         });
       },
       async run() {
+        throw new Error('non-semantic dispatch is not expected');
+      },
+      async runSemantic() {
         return {
           outcome: { ok: true, tier: 'ax', verified: true },
           observation: fixtureObservation({
@@ -340,9 +349,9 @@ describe('Computer Use cross-layer deterministic contract', () => {
     assert.equal(tools.sessionEvents.snapshot('session-1').status, 'user_stopped');
     const afterTurn = (await tool.impl(
       {
-        action: 'left_click',
+        action: 'click_element',
         observation_id: observationIdOf(observed.modelText),
-        coordinate: [400, 200],
+        element_id: '5',
       } as never,
       context({ toolCallId: 'late-action' }),
     )) as { text: string };

packages/computer-use/src/__tests__/computer-use-overlay-hook.test.ts

@@ -55,7 +55,7 @@ test('presentation starts from the Runtime-bound screen point', () => {
   const { controller, moves } = fakeController();
   const hook = createComputerUseOverlayHook(controller as never);
   hook.onActionBegin(
-    { type: 'left_click', coordinate: { x: 400, y: 300 } },
+    { type: 'click_element' },
     {
       sessionId: 's1',
       toolCallId: 'a1',
@@ -83,7 +83,7 @@ test('a window to order against replaces the level as what keeps the cursor visi
   const { controller, moves } = fakeController();
   const hook = createComputerUseOverlayHook(controller as never);
   hook.onActionBegin(
-    { type: 'left_click', coordinate: { x: 400, y: 300 } },
+    { type: 'click_element' },
     {
       sessionId: 's1',
       toolCallId: 'a1',
@@ -100,7 +100,7 @@ test('with no window to order against the cursor stays elevated', () => {
   const { controller, moves } = fakeController();
   const hook = createComputerUseOverlayHook(controller as never);
   hook.onActionBegin(
-    { type: 'left_click', coordinate: { x: 400, y: 300 } },
+    { type: 'click_element' },
     {
       sessionId: 's1',
       toolCallId: 'a1',
@@ -120,7 +120,7 @@ test('the landing carries the window the cursor has to come back down to', () =>
   const { controller, completions } = fakeController();
   const hook = createComputerUseOverlayHook(controller as never);
   hook.onActionEnd?.(
-    { type: 'left_click', coordinate: { x: 400, y: 300 } },
+    { type: 'click_element' },
     { outcome: { ok: true, tier: 'semantic-background', verified: true } },
     {
       sessionId: 's1',
@@ -132,16 +132,17 @@ test('the landing carries the window the cursor has to come back down to', () =>
   assert.equal((completions[0] as { targetWindowId?: number }).targetWindowId, 20);
 });
 
-test('the executor-resolved point wins when there is one', () => {
+test('the Runtime-bound presentation point completes the cursor animation', () => {
   const { controller, completions } = fakeController();
   const hook = createComputerUseOverlayHook(controller as never);
   hook.onActionEnd?.(
-    { type: 'left_click', coordinate: { x: 400, y: 300 } },
+    { type: 'click_element' },
+    { outcome: { ok: true, tier: 'semantic-background', verified: true } },
     {
-      outcome: { ok: true, tier: 'semantic-background', verified: true },
-      resolvedScreenPoint: { x: 202, y: 152 },
+      sessionId: 's1',
+      toolCallId: 'a1',
+      presentationScreenPoint: { x: 202, y: 152 },
     },
-    { sessionId: 's1', toolCallId: 'a1' },
   );
   assert.deepEqual(completions, [
     {
@@ -159,65 +160,27 @@ test('a successful action with no point anywhere still cancels', () => {
   const { controller, completions, cancellations } = fakeController();
   const hook = createComputerUseOverlayHook(controller as never);
   hook.onActionEnd?.(
-    { type: 'left_click', coordinate: { x: 400, y: 300 } },
+    { type: 'click_element' },
     { outcome: { ok: true, tier: 'ax', verified: false } },
     { sessionId: 's1', toolCallId: 'a1' },
   );
   assert.deepEqual(completions, []);
   assert.deepEqual(cancellations, [{ actionId: 'a1', sessionId: 's1' }]);
 });
 
-test('failed pointer action without a resolved point cancels presentation', () => {
+test('failed semantic action cancels presentation', () => {
   const { controller, completions, cancellations } = fakeController();
   const hook = createComputerUseOverlayHook(controller as never);
   hook.onActionEnd?.(
-    { type: 'left_click', coordinate: { x: 40, y: 30 } },
+    { type: 'click_element' },
     { outcome: { ok: false, error: 'capture_failed', message: 'no effect' } },
     { sessionId: 's1', toolCallId: 'a1' },
   );
   assert.deepEqual(completions, []);
   assert.deepEqual(cancellations, [{ actionId: 'a1', sessionId: 's1' }]);
 });
 
-test('failed pointer action with a diagnostic point still cancels', () => {
-  const { controller, completions, cancellations } = fakeController();
-  const hook = createComputerUseOverlayHook(controller as never);
-  hook.onActionEnd?.(
-    { type: 'left_click', coordinate: { x: 40, y: 30 } },
-    {
-      outcome: { ok: false, error: 'target_changed', message: 'moved' },
-      resolvedScreenPoint: { x: 140, y: 130 },
-    },
-    { sessionId: 's1', toolCallId: 'a1' },
-  );
-  assert.deepEqual(completions, []);
-  assert.deepEqual(cancellations, [{ actionId: 'a1', sessionId: 's1' }]);
-});
-
-test('mouse_move completion is reconciled from executor evidence', () => {
-  const { controller, completions } = fakeController();
-  const hook = createComputerUseOverlayHook(controller as never);
-  hook.onActionEnd?.(
-    { type: 'mouse_move', coordinate: { x: 40, y: 30 } },
-    {
-      outcome: { ok: true, tier: 'coordinate-background' },
-      resolvedScreenPoint: { x: 140, y: 130 },
-    },
-    { sessionId: 's1', toolCallId: 'move1' },
-  );
-  assert.deepEqual(completions, [
-    {
-      actionId: 'move1',
-      sessionId: 's1',
-      screenX: 140,
-      screenY: 130,
-      kind: 'move',
-      pulse: false,
-    },
-  ]);
-});
-
-test('non-pointer actions keep the session cursor without moving it', () => {
+test('non-presented actions keep the session cursor without moving it', () => {
   const { controller, moves, ensured } = fakeController();
   const hook = createComputerUseOverlayHook(controller as never);
   for (const action of [
@@ -369,18 +332,6 @@ test('a semantic action reaches the sink with the point it is aimed at', async (
   });
 });
 
-test('a coordinate action is unchanged', async () => {
-  const events = await driveRealTool({}, { action: 'left_click', coordinate: [400, 300] });
-  assert.deepEqual(
-    events.map((event) => event.call),
-    ['move', 'complete'],
-  );
-  // 400/800 and 300/600 of a 400x300 window at (100, 50).
-  const move = events[0]?.input as { screenX: number; screenY: number };
-  assert.equal(move.screenX, 300);
-  assert.equal(move.screenY, 200);
-});
-
 test('an element whose observed frame is outside its window is not aimed at', async () => {
   // A frame that no longer lies inside the window is stale or the element has
   // moved, which is what the executor refuses the action for. Flying the cursor