test(maestro): prove settle metric execution

Author: thymikee

packages/maestro/src/internal/engine-types.ts

@@ -105,6 +105,7 @@ export type MaestroRuntimeMetrics = {
   hierarchyCaptures: number;
   screenshotCaptures: number;
   tapRetries: number;
+  settleLatches: number;
   settleTimeouts: number;
 };
 

packages/maestro/src/internal/facade-execution.ts

@@ -7,6 +7,7 @@ import {
   isMaestroControlCommandDescriptor,
   type MaestroEngineEvent,
   type MaestroEngineObserver,
+  type MaestroRuntimeMetrics,
   type MaestroRuntimePort,
 } from './engine-types.ts';
 import { parseMaestroProgram } from './program-ir-parser.ts';
@@ -49,12 +50,7 @@ export type MaestroActionEvent = {
 
 export type MaestroCompletedActionEvent = MaestroActionEvent & {
   readonly durationMs: number;
-  readonly runtimeMetrics?: {
-    hierarchyCaptures: number;
-    screenshotCaptures: number;
-    tapRetries: number;
-    settleTimeouts: number;
-  };
+  readonly runtimeMetrics?: MaestroRuntimeMetrics;
   readonly data?: Record<string, unknown>;
 };
 

packages/maestro/src/internal/replay-plan-execution.ts

@@ -112,6 +112,7 @@ function runtimeMetricsDelta(
       hierarchyCaptures: after.hierarchyCaptures - before.hierarchyCaptures,
       screenshotCaptures: after.screenshotCaptures - before.screenshotCaptures,
       tapRetries: after.tapRetries - before.tapRetries,
+      settleLatches: after.settleLatches - before.settleLatches,
       settleTimeouts: after.settleTimeouts - before.settleTimeouts,
     },
   };

packages/maestro/test/conformance/differential/invariants.test.ts

@@ -7,6 +7,7 @@ import os from 'node:os';
 import path from 'node:path';
 import { test } from 'node:test';
 import { MAESTRO_DEFAULT_SETTLE_TIMEOUT_MS, parseMaestroConformanceSource } from '../harness.ts';
+import { parseMaestroProgram } from '../../../src/internal/program-ir-parser.ts';
 import { DIFFERENTIAL_SCENARIOS } from './scenarios.ts';
 import { type Invariant, evaluateInvariant, readTrace } from './invariants.ts';
 
@@ -33,29 +34,59 @@ const SETTLE_TIMEOUT_INVARIANT: Invariant = {
   because: 'test',
 };
 
-const settleStep = (command: string, durationMs: number, settleTimeouts: number, step = 1) => ({
+const SETTLE_LATCH_INVARIANT: Invariant = {
+  kind: 'metricAtLeast',
+  command: 'tapOn',
+  metric: 'settleLatches',
+  min: 1,
+  because: 'test',
+};
+
+const settleStep = (
+  command: string,
+  durationMs: number,
+  metrics: { settleLatches: number; settleTimeouts: number },
+  step = 1,
+) => ({
   type: 'replay_action_stop',
   step,
   command,
   ok: true,
   durationMs,
-  resultTiming: { hierarchyCaptures: 1, screenshotCaptures: 0, tapRetries: 0, settleTimeouts },
+  resultTiming: { hierarchyCaptures: 1, screenshotCaptures: 0, tapRetries: 0, ...metrics },
 });
 
 test('a stability loop that latched holds the settle invariant however slow the step', () => {
-  const result = evaluateInvariant([settleStep('tapOn', 3344, 0)], SETTLE_TIMEOUT_INVARIANT);
+  const result = evaluateInvariant(
+    [settleStep('tapOn', 3344, { settleLatches: 1, settleTimeouts: 0 })],
+    SETTLE_TIMEOUT_INVARIANT,
+  );
   assert.equal(result.status, 'held');
 });
 
 test('a stability loop that never latched violates it however fast the step', () => {
-  const result = evaluateInvariant([settleStep('tapOn', 120, 1)], SETTLE_TIMEOUT_INVARIANT);
+  const result = evaluateInvariant(
+    [settleStep('tapOn', 120, { settleLatches: 0, settleTimeouts: 1 })],
+    SETTLE_TIMEOUT_INVARIANT,
+  );
   assert.equal(result.status, 'violated');
   assert.match(result.detail, /settleTimeouts was 1/);
 });
 
+test('a tap that did not run the loop violates the settle proof-of-life invariant', () => {
+  const result = evaluateInvariant(
+    [settleStep('tapOn', 120, { settleLatches: 0, settleTimeouts: 0 })],
+    SETTLE_LATCH_INVARIANT,
+  );
+  assert.equal(result.status, 'violated');
+});
+
 test('another command running out of settle budget does not implicate the tap', () => {
   const result = evaluateInvariant(
-    [settleStep('scroll', 900, 2), settleStep('tapOn', 3344, 0, 2)],
+    [
+      settleStep('scroll', 900, { settleLatches: 0, settleTimeouts: 2 }),
+      settleStep('tapOn', 3344, { settleLatches: 1, settleTimeouts: 0 }, 2),
+    ],
     SETTLE_TIMEOUT_INVARIANT,
   );
   assert.equal(result.status, 'held');
@@ -122,21 +153,28 @@ test('readTrace on a missing file returns no events', () => {
 
 test('bug class 4 has a machine-checkable invariant, not just outcome parity', () => {
   const settle = DIFFERENTIAL_SCENARIOS.find((scenario) => scenario.bugClass === 4);
-  const invariant = settle?.engineInvariants?.[0];
-  assert.ok(invariant, 'settle scenario must carry an engine-side invariant');
-  assert.equal(invariant?.kind, 'metricAtMost');
+  const invariants = settle?.engineInvariants;
+  assert.ok(invariants, 'settle scenario must carry engine-side invariants');
   assert.deepEqual(
-    invariant?.kind === 'metricAtMost'
-      ? { metric: invariant.metric, max: invariant.max }
-      : undefined,
-    { metric: 'settleTimeouts', max: 0 },
+    invariants.map((invariant) =>
+      invariant.kind === 'metricAtLeast'
+        ? { kind: invariant.kind, metric: invariant.metric, min: invariant.min }
+        : invariant.kind === 'metricAtMost'
+          ? { kind: invariant.kind, metric: invariant.metric, max: invariant.max }
+          : { kind: invariant.kind },
+    ),
+    [
+      { kind: 'metricAtLeast', metric: 'settleLatches', min: 1 },
+      { kind: 'metricAtMost', metric: 'settleTimeouts', max: 0 },
+    ],
   );
 });
 
 const SETTLE_FLOW_PATH = path.join(import.meta.dirname, 'flows/settle-after-tap.yaml');
 
 function assertSettleFlowSemantics(source: string): void {
   const parsed = parseMaestroConformanceSource(source, SETTLE_FLOW_PATH);
+  const program = parseMaestroProgram(source, { sourcePath: SETTLE_FLOW_PATH });
   assert.equal(
     parsed.commands.some(
       (command) => command.kind === 'scroll' || command.kind === 'scrollUntilVisible',
@@ -147,7 +185,9 @@ function assertSettleFlowSemantics(source: string): void {
     parsed.commands.filter((command) => command.kind === 'tap'),
     [{ kind: 'tap', longPress: false, repeat: 1, target: { selector: { text: 'Settings' } } }],
   );
-  assert.match(source, /retryTapIfNoChange: true/);
+  const tap = program.commands.find((command) => command.kind === 'tapOn');
+  assert.equal(tap?.kind, 'tapOn');
+  assert.equal(tap?.retryTapIfNoChange, true);
   assert.equal(
     parsed.commands.some(
       (command) =>
@@ -163,9 +203,10 @@ test('the settle detector reaches its tap without an unrelated setup command', (
   assertSettleFlowSemantics(fs.readFileSync(SETTLE_FLOW_PATH, 'utf8'));
 });
 
-test('the settle flow guard rejects a changed tap target or inserted scroll', () => {
+test('the settle flow guard rejects a changed tap target, disabled retry, or inserted scroll', () => {
   const flow = fs.readFileSync(SETTLE_FLOW_PATH, 'utf8');
   assert.throws(() => assertSettleFlowSemantics(flow.replace('text: Settings', 'text: Home')));
+  assert.throws(() => assertSettleFlowSemantics(flow.replace(/\n\s*retryTapIfNoChange: true/, '')));
   assert.throws(() => assertSettleFlowSemantics(flow.replace('- tapOn:', '- scroll\n- tapOn:')));
 });
 

packages/maestro/test/conformance/differential/invariants.ts

@@ -12,22 +12,19 @@
 // The evaluator is pure and unit-tested against synthetic traces; the device run
 // that produces a real trace happens only on the scheduled workflow.
 import fs from 'node:fs';
+import type { MaestroRuntimeMetrics } from '../../../src/internal/engine-types.ts';
 
 export type TraceEvent = {
   type: string;
   step?: number;
   command?: string;
   ok?: boolean;
   durationMs?: number;
-  /** Per-step MaestroRuntimeMetrics delta (hierarchyCaptures/screenshotCaptures/tapRetries). */
+  /** Per-step MaestroRuntimeMetrics delta. */
   resultTiming?: Record<string, unknown>;
 };
 
-export type MetricKey =
-  | 'tapRetries'
-  | 'hierarchyCaptures'
-  | 'screenshotCaptures'
-  | 'settleTimeouts';
+export type MetricKey = keyof MaestroRuntimeMetrics;
 
 export type Invariant =
   | {

packages/maestro/test/conformance/differential/scenarios.ts

@@ -96,6 +96,13 @@ export const DIFFERENTIAL_SCENARIOS: DifferentialScenario[] = [
     // Outcome parity cannot see settle ordering: a tap that burns the whole
     // budget still passes. This invariant is the actual bug-class-4 detector.
     engineInvariants: [
+      {
+        kind: 'metricAtLeast',
+        command: 'tapOn',
+        metric: 'settleLatches',
+        min: 1,
+        because: 'the scenario must execute and latch its inline stability loop',
+      },
       {
         kind: 'metricAtMost',
         command: 'tapOn',

src/daemon/adapters/maestro/__tests__/daemon-runtime-port-targets.test.ts

@@ -235,6 +235,7 @@ test('retries an iOS non-hittable coordinate fallback when the hierarchy does no
     hierarchyCaptures: 5,
     screenshotCaptures: 2,
     tapRetries: 1,
+    settleLatches: 2,
     settleTimeouts: 0,
   });
 });
@@ -299,6 +300,7 @@ test('does not retry an iOS tap when only the rendered surface changes', async (
     hierarchyCaptures: 3,
     screenshotCaptures: 2,
     tapRetries: 0,
+    settleLatches: 1,
     settleTimeouts: 0,
   });
 });
@@ -346,10 +348,54 @@ test('uses screenshot evidence without a redundant hierarchy baseline for iOS po
     hierarchyCaptures: 4,
     screenshotCaptures: 2,
     tapRetries: 1,
+    settleLatches: 2,
     settleTimeouts: 0,
   });
 });
 
+test('records an exhausted inline tap settle', async () => {
+  const clock = { value: 0 };
+  let snapshots = 0;
+  const port = createDaemonMaestroRuntimePort({
+    baseReq: makeBaseRequest({ flags: { platform: 'android', replayBackend: 'maestro' } }),
+    invoke: async (request) => {
+      if (request.command !== 'snapshot') return { ok: true, data: {} };
+      snapshots += 1;
+      return {
+        ok: true,
+        data: {
+          nodes: [
+            { index: 0, type: 'Application' },
+            {
+              index: 1,
+              parentIndex: 0,
+              type: snapshots === 1 ? 'Button' : 'Text',
+              ...(snapshots === 1 ? { identifier: 'continue' } : { value: String(snapshots) }),
+              rect: { x: 20, y: 40, width: 120, height: 44 },
+            },
+          ],
+        },
+      };
+    },
+    dependencies: makeDependencies(clock),
+    platform: 'android',
+  });
+
+  await port.execute({
+    command: {
+      kind: 'tapOn',
+      source: { line: 2 },
+      target: { space: 'target', selector: { id: 'continue' } },
+      retryTapIfNoChange: true,
+    },
+    generation: 0,
+    env: {},
+    invalidateObservation() {},
+  });
+
+  expect(port.readMetrics?.()).toMatchObject({ settleTimeouts: 1 });
+});
+
 function solidPng(value: number): Buffer {
   const image = new PNG({ width: 2, height: 2 });
   image.data.fill(value);

src/daemon/adapters/maestro/__tests__/daemon-runtime-port.test.ts

@@ -249,6 +249,7 @@ test('uses an observation as the baseline for a later mutation barrier', async (
     hierarchyCaptures: 2,
     screenshotCaptures: 0,
     tapRetries: 0,
+    settleLatches: 1,
     settleTimeouts: 0,
   });
   expect(clock.value).toBe(MAESTRO_OBSERVATION_POLL_MS);

src/daemon/adapters/maestro/daemon-runtime-port-observation.ts

@@ -43,7 +43,9 @@ export type MaestroSnapshotSource = {
   readonly requireStability: (generation: number) => void;
   readonly consumeStabilityFromVisualWait: (context: MaestroRuntimeReadContext) => void;
   readonly prime: (generation: number, snapshot: SnapshotState) => void;
-  readonly settlePending: (context: MaestroRuntimeReadContext) => Promise<void>;
+  readonly settlePending: (
+    context: MaestroRuntimeReadContext,
+  ) => Promise<StableMaestroSnapshot | undefined>;
 };
 
 export type StableMaestroSnapshot = {

src/daemon/adapters/maestro/daemon-runtime-port-snapshot-source.ts

@@ -101,7 +101,7 @@ export function createDaemonMaestroSnapshotSource(
       primed = { generation, snapshot };
     },
     settlePending: async (context) => {
-      if (stabilityRequiredGeneration === undefined) return;
+      if (stabilityRequiredGeneration === undefined) return undefined;
       if (stabilityRequiredGeneration !== context.generation) {
         throw new AppError(
           'COMMAND_FAILED',
@@ -120,6 +120,7 @@ export function createDaemonMaestroSnapshotSource(
       stabilityRequiredGeneration = undefined;
       stabilityBaseline = undefined;
       primed = { generation: context.generation, snapshot: stable.snapshot };
+      return stable;
     },
   };
 }