diff --git a/apple/snapshot-bridge/README.md b/apple/snapshot-bridge/README.md index 12326d822c..22b3c042c9 100644 --- a/apple/snapshot-bridge/README.md +++ b/apple/snapshot-bridge/README.md @@ -50,13 +50,15 @@ disabled after fallback until that app relaunches. ## Remote content -The reader snapshots one process. A WebKit page — Safari's or a `WKWebView`'s — -lives in a WebContent process and appears in that tree as an `AXRemoteElement` -leaf under the web view. The guest returns the leaf as delivered; the host -refuses a tree in which such a leaf sits under a web view and reaches the -viewport, or reports no frame (`remote-content-boundary`), and routes that app -generation to XCTest, which resolves remote elements (#2484). A zero-area or -off-screen leaf is published: it hosts nothing the capture can miss. +The reader snapshots one process. Two hosts put another process's UI into that +tree as an `AXRemoteElement` leaf: a WebKit page — Safari's or a `WKWebView`'s — +lives in a WebContent process under the web view, and a scene-hosted remote view +controller, such as a share extension presented over the app, lives in the +extension's process under a `_UISceneHostingView`. The guest returns the leaf as +delivered; the host refuses a tree in which such a leaf sits under either host +and reaches the viewport, or reports no frame (`remote-content-boundary`), and +routes that app generation to XCTest, which resolves remote elements (#2484). A +zero-area or off-screen leaf is published: it hosts nothing the capture can miss. ## Bounded depth recovery diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 66e507a6f3..81e31875ef 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -49,9 +49,12 @@ such a leaf sits under a `WebView`-typed ancestor and reaches the viewport (`rem instead of publishing a screen without its page: refs issued from it would target the host views around the page rather than the page. A leaf whose frame is zero-area or off screen hosts nothing the capture can miss and is published; one that reports no frame is refused, because nothing proves -it empty. Remote elements outside a web view are not classified — no capture has shown one — and a -web view truncated away by the node or depth cap stays disclosed as truncation. XCTest resolves -remote elements, so the fallback serves the page (#2484). +it empty. A scene-hosted remote view controller — a share or action extension presented over the +host app — reaches the tree the same way: an `AXRemoteElement` leaf under a `_UISceneHostingView`, +with the extension's controls in its own process. The source refuses that leaf by the same rule. +Remote elements under any other host are not classified — no capture has shown one — and a host +truncated away by the node or depth cap stays disclosed as truncation. XCTest resolves remote +elements, so the fallback serves the page or the extension (#2484). The refusal opens the generation circuit, as any failure that says something about the app itself does, so a hybrid app that showed one web screen takes XCTest for its remaining native screens until diff --git a/packages/platform-apple/src/snapshot-source/adapter.ts b/packages/platform-apple/src/snapshot-source/adapter.ts index 4ffd715553..3ac46f6390 100644 --- a/packages/platform-apple/src/snapshot-source/adapter.ts +++ b/packages/platform-apple/src/snapshot-source/adapter.ts @@ -211,8 +211,9 @@ function createAcquisition( if (typeof generation !== 'string' || !generation) { throw snapshotSourceError('malformed-tree', 'generation-invalid'); } - // A tree that ends at a web view's out-of-process page would present the screen without the - // page, and refs issued from it would target the host views around it rather than the page. + // A tree that ends at out-of-process content — a web view's page, or a scene-hosted extension — + // would present the screen without it, and refs issued from it would target the host views around + // it rather than the content. // The source refuses it as a screen it cannot describe, like a missing automation mode, so the // route serves the XCTest runner, which resolves remote elements (#2484). if (decoded.opaqueRemoteElements > 0) { diff --git a/packages/platform-apple/src/snapshot-source/tree.test.ts b/packages/platform-apple/src/snapshot-source/tree.test.ts index b735e7aaf1..7844bcc9d6 100644 --- a/packages/platform-apple/src/snapshot-source/tree.test.ts +++ b/packages/platform-apple/src/snapshot-source/tree.test.ts @@ -222,7 +222,7 @@ test('the bridge tree counts web-hosted remote leaves that reach the viewport', decode({ [automationType]: 0, [frame]: viewport, [children]: [remoteLeaf(viewport)] }) .opaqueRemoteElements, 0, - 'a remote leaf outside a web view is not classified', + 'a remote leaf outside a remote-content host is not classified', ); assert.equal( decode( @@ -236,6 +236,75 @@ test('the bridge tree counts web-hosted remote leaves that reach the viewport', ); }); +test('the bridge tree counts scene-hosted remote leaves that reach the viewport', () => { + // A share extension presented over Photos on iOS 26, in the shape the bridge captured: the app + // root reports no type, its window reports the viewport, and the extension's controls live in + // their own process below the scene-hosting view, where the reader stops at the remote element. + const viewport = { X: 0, Y: 0, Width: 402, Height: 874 }; + const remoteLeaf = (rect?: Record) => ({ + [application]: 'AXRemoteElement', + [baseType]: 'NSObject', + ...(rect ? { [frame]: rect } : {}), + [children]: [], + }); + const sceneHosted = (content: Record) => ({ + [application]: '_UISceneHostingView', + [frame]: viewport, + [children]: [ + { [application]: '_UIScenePresentationView', [frame]: viewport, [children]: [content] }, + ], + }); + const decode = (host: Record) => + decodeSnapshotBridgeTree( + { + [application]: 'PhotosApplication', + [baseType]: 'UIApplication', + [frame]: viewport, + [children]: [{ [application]: 'UIWindow', [frame]: viewport, [children]: [host] }], + }, + { truncated: false }, + limits, + ); + + const opaque = decode(sceneHosted(remoteLeaf(viewport))); + assert.deepEqual(opaque.viewport, { + kind: 'reported', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }); + assert.equal(opaque.nodes[2]?.role, '_UISceneHostingView'); + assert.equal(opaque.nodes[4]?.role, 'AXRemoteElement'); + assert.equal(opaque.opaqueRemoteElements, 1); + + assert.equal(decode(sceneHosted(remoteLeaf())).opaqueRemoteElements, 1, 'frameless leaf refuses'); + assert.equal( + decode(sceneHosted(remoteLeaf({ X: 0, Y: 0, Width: 0, Height: 0 }))).opaqueRemoteElements, + 0, + 'a dismissed extension leaves a zero-area leaf that hosts nothing', + ); + assert.equal( + decode(sceneHosted(remoteLeaf({ X: 0, Y: 2000, Width: 402, Height: 874 }))) + .opaqueRemoteElements, + 0, + 'off-screen leaf is not on this screen', + ); + assert.equal( + decode( + sceneHosted({ + ...remoteLeaf(viewport), + [children]: [{ [automationType]: 9, [label]: 'Enshrine', [children]: [] }], + }), + ).opaqueRemoteElements, + 0, + 'a crossed boundary is not opaque', + ); + assert.equal( + decode(sceneHosted({ [automationType]: 9, [label]: 'Enshrine', [children]: [] })) + .opaqueRemoteElements, + 0, + 'a scene-hosting view with ordinary children is not refused', + ); +}); + test('the bridge tree reads enabled from the NotEnabled trait', () => { const button = (word?: unknown) => ({ [automationType]: 9, diff --git a/packages/platform-apple/src/snapshot-source/tree.ts b/packages/platform-apple/src/snapshot-source/tree.ts index 0cb4aff91f..e6657eccbf 100644 --- a/packages/platform-apple/src/snapshot-source/tree.ts +++ b/packages/platform-apple/src/snapshot-source/tree.ts @@ -140,19 +140,25 @@ const SELECTED_TRAIT = 1n << 3n; const KEYBOARD_FOCUS_TRAIT = 1n << 21n; /** - * A WebKit page — Safari's, or a `WKWebView`'s — lives in a WebContent process and reaches UIKit's - * tree as an `AXRemoteElement` under the web view, with its children in that other process. The - * guest reader snapshots one process, so it delivers that element as a leaf (#2484). Such a leaf - * is opaque when it sits under a `WebView`-typed ancestor and its frame reaches the viewport: the - * page is on screen and the tree does not describe it. A leaf whose frame is zero-area or off - * screen hosts nothing the capture can miss; one that reports no frame at all is refused, because - * nothing proves it is empty. Remote elements outside a web view are not classified here — no - * capture has shown one — and content truncated away above the web view stays disclosed as - * truncation, not as a boundary. + * Two hosts put another process's UI into this one's tree as an `AXRemoteElement`, with the + * children in that other process: + * + * - a WebKit page — Safari's, or a `WKWebView`'s — lives in a WebContent process under the web + * view (#2484); + * - a scene-hosted remote view controller, such as a share or action extension presented over the + * host app, lives in the extension's process under a `_UISceneHostingView`. + * + * The guest reader snapshots one process, so it delivers that element as a leaf. Such a leaf is + * opaque when it sits under one of those hosts and its frame reaches the viewport: the content is + * on screen and the tree does not describe it. A leaf whose frame is zero-area or off screen hosts + * nothing the capture can miss; one that reports no frame at all is refused, because nothing proves + * it is empty. Remote elements under any other host are not classified here — no capture has shown + * one — and content truncated away above the host stays disclosed as truncation, not as a boundary. */ const REMOTE_ELEMENT_CLASS = 'AXRemoteElement'; const EMPTY_RECT: Rect = { x: 0, y: 0, width: 0, height: 0 }; const WEB_VIEW_TYPE = 'WebView'; +const SCENE_HOSTING_VIEW_CLASS = '_UISceneHostingView'; export function decodeSnapshotBridgeTree( tree: unknown, @@ -164,7 +170,7 @@ export function decodeSnapshotBridgeTree( throw snapshotSourceError('malformed-tree', 'guest-tree-root-invalid'); } const nodes: RawSnapshotNode[] = []; - const webHostedRemoteLeaves: (Rect | undefined)[] = []; + const hostedRemoteLeaves: (Rect | undefined)[] = []; let maxTraversalDepth = 0; for (const root of roots) { visitNode(root, undefined, 0, false); @@ -198,7 +204,7 @@ export function decodeSnapshotBridgeTree( nodes, maxTraversalDepth, viewport, - opaqueRemoteElements: webHostedRemoteLeaves.filter((rect) => isOpaqueRemoteLeaf(rect, viewport)) + opaqueRemoteElements: hostedRemoteLeaves.filter((rect) => isOpaqueRemoteLeaf(rect, viewport)) .length, unresolvedCoordinateSpaceWindows: countUnresolvedCoordinateSpaceWindows(nodes, viewport), }; @@ -207,7 +213,7 @@ export function decodeSnapshotBridgeTree( value: Record, parentIndex: number | undefined, depth: number, - underWebView: boolean, + underRemoteHost: boolean, ): void { if (nodes.length >= limits.maxNodes) { throw snapshotSourceError('malformed-tree', 'node-limit-exceeded', { @@ -227,13 +233,13 @@ export function decodeSnapshotBridgeTree( const node = nodeFacts(value, index, parentIndex, depth); nodes.push(node); maxTraversalDepth = Math.max(maxTraversalDepth, depth); - if (isWebHostedRemoteLeaf(node, children.length, underWebView)) { - webHostedRemoteLeaves.push(node.rect); + if (isHostedRemoteLeaf(node, children.length, underRemoteHost)) { + hostedRemoteLeaves.push(node.rect); } - const hostsWeb = underWebView || node.type === WEB_VIEW_TYPE; + const hostsRemote = underRemoteHost || isRemoteContentHost(node); for (const child of children) { if (!isRecord(child)) throw snapshotSourceError('malformed-tree', 'child-invalid'); - visitNode(child, index, depth + 1, hostsWeb); + visitNode(child, index, depth + 1, hostsRemote); } } } @@ -299,12 +305,16 @@ function elementTypeName( return ELEMENT_TYPE_NAMES[automationType] ?? 'Other'; } -function isWebHostedRemoteLeaf( +function isRemoteContentHost(node: RawSnapshotNode): boolean { + return node.type === WEB_VIEW_TYPE || node.role === SCENE_HOSTING_VIEW_CLASS; +} + +function isHostedRemoteLeaf( node: RawSnapshotNode, childCount: number, - underWebView: boolean, + underRemoteHost: boolean, ): boolean { - return underWebView && node.role === REMOTE_ELEMENT_CLASS && childCount === 0; + return underRemoteHost && node.role === REMOTE_ELEMENT_CLASS && childCount === 0; } function isOpaqueRemoteLeaf(rect: Rect | undefined, viewport: IosViewportEvidence): boolean { diff --git a/packages/platform-apple/src/snapshot-source/types.ts b/packages/platform-apple/src/snapshot-source/types.ts index 661bb3b4b1..16285edfa8 100644 --- a/packages/platform-apple/src/snapshot-source/types.ts +++ b/packages/platform-apple/src/snapshot-source/types.ts @@ -126,8 +126,8 @@ export type SnapshotSourceDecodedTree = Readonly<{ viewport: IosViewportEvidence; maxTraversalDepth: number; /** - * `AXRemoteElement` leaves under a web view whose frame reaches the viewport, or that report no - * frame: pages the reader could not cross into (#2484). + * `AXRemoteElement` leaves under a web view or a scene-hosting view whose frame reaches the + * viewport, or that report no frame: pages and extensions the reader could not cross into (#2484). */ opaqueRemoteElements: number; /**