fix(computer-use): #820 Windows の V1 のレーンの UI Automation に 500 ms のタイムアウトを付ける

apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc

@@ -6788,6 +6788,35 @@ bool CheckWindowsInterlockCapture() {
 
 // ---- Classifier v2, S5-c-2 PR-3: the display name of a file browser item -> the real entry ---------
 
+// ---- The Windows V1 host: what a read that could not be made leaves behind is never "no risk" -----
+
+bool CheckWindowsV1DefaultRisk() {
+  using namespace sprint_coder::computer_use;
+  // The host's WindowsUiaRisk is all false until something is read. A path that ended (a timeout, a
+  // Cancel) returns that default, and the host asks ClassifyNativePreflightFacts with
+  // {metadata_complete, classified, secure, high_impact}.
+  const NativePreflightDecision nothing = ClassifyNativePreflightFacts({false, false, false, false});
+  if (!Check(!NativePreflightAllowsDispatch(nothing) && nothing.kind == NativePreflightKind::kTakeover &&
+                 nothing.reason == NativePreflightReason::kUnclassified,
+             "v1-default-risk-is-unclassified")) return false;
+  // Whatever else is set, facts that are not complete and classified are never an ordinary dispatch.
+  for (int bits = 0; bits < 8; ++bits) {
+    const bool secure = (bits & 1) != 0;
+    const bool high_impact = (bits & 2) != 0;
+    const bool classified = (bits & 4) != 0;
+    const NativePreflightDecision incomplete = ClassifyNativePreflightFacts({false, classified, secure, high_impact});
+    if (!Check(!NativePreflightAllowsDispatch(incomplete) && incomplete.reason == NativePreflightReason::kUnclassified,
+               "v1-incomplete-metadata-is-always-unclassified")) return false;
+    const NativePreflightDecision unclassified = ClassifyNativePreflightFacts({true, false, secure, high_impact});
+    if (!Check(!NativePreflightAllowsDispatch(unclassified) && unclassified.reason == NativePreflightReason::kUnclassified,
+               "v1-unclassified-is-never-ordinary")) return false;
+  }
+  if (!Check(NativePreflightAllowsDispatch(ClassifyNativePreflightFacts({true, true, false, false})),
+             "v1-complete-classified-and-quiet-is-ordinary")) return false;
+  std::cout << "Computer Use native Windows V1 default risk: PASS\n";
+  return true;
+}
+
 bool CheckWindowsShellNameMatch() {
   using namespace sprint_coder::computer_use;
   // A candidate: the file system's long name and the name the SHELL shows for it.
@@ -8321,6 +8350,7 @@ int main() {
   if (!CheckNativeInterlockCapture()) return 1;
   if (!CheckWindowsInterlockCapture()) return 1;
   if (!CheckWindowsShellNameMatch()) return 1;
+  if (!CheckWindowsV1DefaultRisk()) return 1;
   if (!CheckClickRule()) return 1;
   if (!CheckInterlockDispatchCodes()) return 1;
   if (!CheckWindowsInterlockJourneys()) return 1;

apps/desktop/computer-use-native/computer_use_windows_uia_source.h

@@ -164,6 +164,37 @@ class BstrHolder {
 };
 
 
+// UI Automation created WITH its call timeouts, or not at all. CUIAutomation8 (Windows 8 and later) is
+// the only coclass that implements IUIAutomation2, whose ConnectionTimeout / TransactionTimeout bound a
+// call to an app that does not answer. The original CUIAutomation does not, and a call to a hung app
+// then never returns (measured: about 1.5 s for the first call alone, and without an upper bound after
+// it), which holds the thread that made it. False (and *automation null) when the coclass, the
+// interface or either timeout is not available. The CONNECTION timeout is what bounds the first contact
+// with a target (ElementFromHandle / ElementFromPoint / GetFocusedElement: measured 311 ms for 300 ms
+// and 2001 ms for 2000 ms on a window that does not answer, whatever the transaction timeout), the
+// TRANSACTION timeout bounds a call on an element that is already connected. The ONE place the shadow source and the V1 lane
+// (CreateWindowsV1Automation in the host) get their bounded UI Automation from.
+[[nodiscard]] inline bool CreateBoundedAutomation(DWORD connection_timeout_ms, DWORD transaction_timeout_ms,
+                                                  IUIAutomation **automation) {
+  *automation = nullptr;
+  IUIAutomation *created = nullptr;
+  if (FAILED(CoCreateInstance(CLSID_CUIAutomation8, nullptr, CLSCTX_INPROC_SERVER,
+                              IID_PPV_ARGS(&created))) ||
+      created == nullptr)
+    return false;
+  IUIAutomation2 *bounded = nullptr;
+  const bool set = SUCCEEDED(created->QueryInterface(IID_PPV_ARGS(&bounded))) && bounded != nullptr &&
+                   SUCCEEDED(bounded->put_ConnectionTimeout(connection_timeout_ms)) &&
+                   SUCCEEDED(bounded->put_TransactionTimeout(transaction_timeout_ms));
+  if (bounded != nullptr) bounded->Release();
+  if (!set) {
+    created->Release();
+    return false;
+  }
+  *automation = created;
+  return true;
+}
+
 // What the shell knows about an item: the path in the file system, and whether it opens by browsing.
 struct ShellItemAnswer {
   std::wstring path;
@@ -775,22 +806,13 @@ class WindowsUiaShadowSource final : public ShadowUiaSource {
                                                                     DWORD timeout_ms,
                                                                     std::atomic<bool> &lookup_slot) {
     using windows_uia_source_detail::ComRef;
-    // CUIAutomation8 (Windows 8 and later) is the only coclass that implements IUIAutomation2, whose
-    // ConnectionTimeout / TransactionTimeout bound a call to an app that does not answer. The
-    // original CUIAutomation does not, and a call to a hung app then never returns (measured), which
-    // would hold the shadow's thread and its one slot for the life of the helper. So the timeouts are
-    // a precondition: without them there is no source, and the capture is recorded as skipped.
+    // The timeouts are a precondition (see windows_uia_source_detail::CreateBoundedAutomation): a call
+    // to a hung app would otherwise hold the shadow's thread and its one slot for the life of the
+    // helper. Without them there is no source, and the capture is recorded as skipped.
     ComRef<IUIAutomation> automation;
-    if (FAILED(CoCreateInstance(CLSID_CUIAutomation8, nullptr, CLSCTX_INPROC_SERVER,
-                                IID_PPV_ARGS(automation.Receive()))) ||
+    if (!windows_uia_source_detail::CreateBoundedAutomation(timeout_ms, timeout_ms, automation.Receive()) ||
         automation.Get() == nullptr)
       return nullptr;
-    ComRef<IUIAutomation2> bounded;
-    if (FAILED(automation.Get()->QueryInterface(IID_PPV_ARGS(bounded.Receive()))) ||
-        bounded.Get() == nullptr ||
-        FAILED(bounded.Get()->put_ConnectionTimeout(timeout_ms)) ||
-        FAILED(bounded.Get()->put_TransactionTimeout(timeout_ms)))
-      return nullptr;
     ComRef<IUIAutomationTreeWalker> walker;
     if (FAILED(automation.Get()->get_RawViewWalker(walker.Receive())) || walker.Get() == nullptr)
       return nullptr;

apps/desktop/src/main/computer-use-native-host.test.ts

@@ -77,6 +77,28 @@ describe('Computer Use native Main adapter', () => {
         Object.assign(new Error('accepted'), { code: 'STALE_TARGET', accepted: true }),
       ),
     ).toBeNull();
+    // #820: a V1 target that did not answer. Before any input (accepted false) it is a typed rejection
+    // that a new observation can answer; once the action was sent (accepted true) a pattern call that
+    // timed out is unconfirmed and stays an exception for the controller to book as an unknown effect,
+    // exactly like input_not_confirmed (the agent decides whether to repeat it).
+    expect(
+      classifyComputerUseNativeDispatchFailure(
+        Object.assign(new Error('timeout'), { code: 'ACCESSIBILITY_TIMEOUT', accepted: false }),
+      ),
+    ).toEqual({ result: 'rejected', reasonCode: 'native_accessibility_timeout' });
+    expect(
+      classifyComputerUseNativeDispatchFailure(
+        Object.assign(new Error('unconfirmed'), {
+          code: 'INPUT_TIMEOUT_UNCONFIRMED',
+          accepted: true,
+        }),
+      ),
+    ).toBeNull();
+    expect(
+      classifyComputerUseNativeDispatchFailure(
+        Object.assign(new Error('unconfirmed'), { code: 'INPUT_TIMEOUT_UNCONFIRMED' }),
+      ),
+    ).toBeNull();
   });
 
   it('maps the three v2 interlock dispatch codes to the result of their preflight row (#500 S5-c-2 PR-5)', () => {

apps/desktop/src/main/computer-use-windows-shadow-reader.test.ts

@@ -277,6 +277,8 @@ describe('Windows UI Automation source only reads', () => {
       'ShellDriveIsLocalDisk',
       'ShellReparseIsLink',
       'ShellMountTargetPath',
+      'CreateBoundedAutomation',
+      'CreateWindowsV1Automation',
       'ShellDirectoryWithSeparator',
       'ShellIniValueIsSafe',
       'ShellDriveLetterIsLocal',
@@ -423,6 +425,8 @@ describe('Windows UI Automation source only reads', () => {
       'ShellDriveIsLocalDisk',
       'ShellReparseIsLink',
       'ShellMountTargetPath',
+      'CreateBoundedAutomation',
+      'CreateWindowsV1Automation',
       'ShellDirectoryWithSeparator',
       'ShellIniValueIsSafe',
       'ShellDriveLetterIsLocal',
@@ -477,19 +481,28 @@ describe('Windows UI Automation source only reads', () => {
       'static std::unique_ptr<WindowsUiaShadowSource> Create(',
       'WindowsUiaShadowSource(const',
     );
-    // CUIAutomation8 is the only coclass with IUIAutomation2 (ConnectionTimeout / TransactionTimeout);
-    // with the original CUIAutomation a call to a hung app never returns (measured).
-    expect(create).toContain('CLSID_CUIAutomation8');
-    expect(code(create)).not.toMatch(/CLSID_CUIAutomation\b(?!8)/u);
-    expect(create).toContain('ComRef<IUIAutomation2> bounded;');
-    expect(create).toContain('put_ConnectionTimeout(timeout_ms)');
-    expect(create).toContain('put_TransactionTimeout(timeout_ms)');
+    // The coclass with IUIAutomation2 (CUIAutomation8; with the original CUIAutomation a call to a hung
+    // app never returns, measured) is created by the one shared helper, which sets both timeouts and
+    // hands out nothing when it cannot (#820: the V1 lane uses the same helper).
+    const helper = between(
+      source,
+      'inline bool CreateBoundedAutomation(',
+      'What the shell knows about an item',
+    );
+    expect(helper).toContain('CLSID_CUIAutomation8');
+    expect(code(source)).not.toMatch(/CLSID_CUIAutomation\b(?!8)/u);
+    expect(helper).toContain('IUIAutomation2 *bounded = nullptr;');
+    expect(helper).toContain('put_ConnectionTimeout(connection_timeout_ms)');
+    expect(helper).toContain('put_TransactionTimeout(transaction_timeout_ms)');
     expect(code(source)).not.toContain('IUIAutomation6');
-    // Failing to set either is "no source" (the capture is skipped), never a call without a bound.
-    expect(create).toMatch(
-      /FAILED\(bounded\.Get\(\)->put_ConnectionTimeout\(timeout_ms\)\) \|\|\s*FAILED\(bounded\.Get\(\)->put_TransactionTimeout\(timeout_ms\)\)\)\s*return nullptr;/u,
+    expect(helper.replace(/\s+/gu, ' ')).toContain(
+      'if (!set) { created->Release(); return false; }',
+    );
+    // Failing to get a bounded one is "no source" (the capture is skipped), never a call without a bound.
+    expect(create.replace(/\s+/gu, ' ')).toContain(
+      'if (!windows_uia_source_detail::CreateBoundedAutomation(timeout_ms, timeout_ms, automation.Receive()) || automation.Get() == nullptr) return nullptr;',
     );
-    expect(create.indexOf('put_TransactionTimeout')).toBeLessThan(
+    expect(create.indexOf('CreateBoundedAutomation')).toBeLessThan(
       create.indexOf('get_RawViewWalker'),
     );
     // The bound is the caller's (S5-c-2 PR-1): the shadow is asynchronous and waits up to 1500 ms, the