Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ appId: com.callstack.agentdevicelab
- launchApp:
clearState: true
- assertVisible: Agent Device Tester
# Discharge launch stabilization before measuring the inline post-tap settle.
- waitForAnimationToEnd
- tapOn:
text: Settings
retryTapIfNoChange: true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,9 @@ function assertSettleFlowSemantics(source: string): void {
parsed.commands.filter((command) => command.kind === 'tap'),
[{ kind: 'tap', longPress: false, repeat: 1, target: { selector: { text: 'Settings' } } }],
);
const tap = program.commands.find((command) => command.kind === 'tapOn');
const tapIndex = program.commands.findIndex((command) => command.kind === 'tapOn');
assert.equal(program.commands[tapIndex - 1]?.kind, 'waitForAnimationToEnd');
const tap = program.commands[tapIndex];
assert.equal(tap?.kind, 'tapOn');
assert.equal(tap?.retryTapIfNoChange, true);
assert.equal(
Expand All @@ -199,7 +201,7 @@ function assertSettleFlowSemantics(source: string): void {
);
}

test('the settle detector reaches its tap without an unrelated setup command', () => {
test('the settle detector isolates its tap from launch stabilization', () => {
assertSettleFlowSemantics(fs.readFileSync(SETTLE_FLOW_PATH, 'utf8'));
});

Expand All @@ -208,6 +210,7 @@ test('the settle flow guard rejects a changed tap target, disabled retry, or ins
assert.throws(() => assertSettleFlowSemantics(flow.replace('text: Settings', 'text: Home')));
assert.throws(() => assertSettleFlowSemantics(flow.replace(/\n\s*retryTapIfNoChange: true/, '')));
assert.throws(() => assertSettleFlowSemantics(flow.replace('- tapOn:', '- scroll\n- tapOn:')));
assert.throws(() => assertSettleFlowSemantics(flow.replace('- waitForAnimationToEnd\n', '')));
});

// --- metricAtLeast: proves a code path actually ran, not just that it passed ---
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { expect, test } from 'vitest';
import { promises as fs } from 'node:fs';
import { PNG } from '@agent-device/capture-kit/png';
import { executeMaestroFlow, inspectMaestroFlow } from '@agent-device/maestro';
import { noMaestroIncludeSources } from '../../../../__tests__/test-utils/replay-script-source.ts';
import type { DaemonInvokeFn, DaemonRequest } from '../../../daemon-request.ts';
import { createDaemonMaestroRuntimePort } from '../daemon-runtime-port.ts';
import { makeBaseRequest, makeDependencies } from './daemon-runtime-port-fixtures.ts';
Expand Down Expand Up @@ -401,3 +403,78 @@ function solidPng(value: number): Buffer {
image.data.fill(value);
return PNG.sync.write(image);
}

test('the differential settle flow excludes a slow launch boundary from tap metrics', async () => {
const flowPath = 'packages/maestro/test/conformance/differential/flows/settle-after-tap.yaml';
const source = await fs.readFile(flowPath, 'utf8');
const clock = { value: 0 };
let captures = 0;
let clicked = false;
const screenshot = PNG.sync.write(new PNG({ width: 1, height: 1 }));
const port = createDaemonMaestroRuntimePort({
baseReq: makeBaseRequest({ flags: { platform: 'ios', replayBackend: 'maestro' } }),
platform: 'ios',
dependencies: makeDependencies(clock),
invoke: async (request) => {
if (request.command === 'click') clicked = true;
if (request.command === 'screenshot') {
await fs.writeFile(request.positionals[0]!, screenshot);
}
if (request.command !== 'snapshot') return { ok: true, data: {} };
captures += 1;
clock.value += 2_100;
return {
ok: true,
data: {
createdAt: captures,
nodes: [
{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } },
{
index: 1,
parentIndex: 0,
type: 'Text',
label: 'Agent Device Tester',
rect: { x: 0, y: 0, width: 300, height: 30 },
},
{
index: 2,
parentIndex: 0,
type: 'Button',
label: 'Settings',
rect: { x: 20, y: 40, width: 120, height: 44 },
},
{
index: 3,
parentIndex: 0,
type: 'Text',
value: clicked ? 'ready' : `launch frame ${captures}`,
},
...(clicked
? [
{
index: 4,
parentIndex: 0,
type: 'Button',
identifier: 'open-inert-surface',
rect: { x: 20, y: 100, width: 120, height: 44 },
},
]
: []),
],
},
};
},
});
let tapMetrics: unknown;
const result = await executeMaestroFlow(inspectMaestroFlow(source, flowPath), port, {
readSource: noMaestroIncludeSources,
observer: {
actionCompleted: (event) => {
if (event.action === 'tapOn') tapMetrics = event.runtimeMetrics;
},
},
});
expect(result, JSON.stringify(result)).toMatchObject({ ok: true });
expect(clicked).toBe(true);
expect(tapMetrics).toMatchObject({ hierarchyCaptures: 2, settleLatches: 1, settleTimeouts: 0 });
});
Loading