diff --git a/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc b/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc index d7161de9f..d1f8dee3c 100644 --- a/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc +++ b/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc @@ -6788,6 +6788,35 @@ bool CheckWindowsInterlockCapture() { // ---- Classifier v2, S5-c-2 PR-3: the display name of a file browser item -> the real entry --------- +// ---- The Windows V1 host: what a read that could not be made leaves behind is never "no risk" ----- + +bool CheckWindowsV1DefaultRisk() { + using namespace sprint_coder::computer_use; + // The host's WindowsUiaRisk is all false until something is read. A path that ended (a timeout, a + // Cancel) returns that default, and the host asks ClassifyNativePreflightFacts with + // {metadata_complete, classified, secure, high_impact}. + const NativePreflightDecision nothing = ClassifyNativePreflightFacts({false, false, false, false}); + if (!Check(!NativePreflightAllowsDispatch(nothing) && nothing.kind == NativePreflightKind::kTakeover && + nothing.reason == NativePreflightReason::kUnclassified, + "v1-default-risk-is-unclassified")) return false; + // Whatever else is set, facts that are not complete and classified are never an ordinary dispatch. + for (int bits = 0; bits < 8; ++bits) { + const bool secure = (bits & 1) != 0; + const bool high_impact = (bits & 2) != 0; + const bool classified = (bits & 4) != 0; + const NativePreflightDecision incomplete = ClassifyNativePreflightFacts({false, classified, secure, high_impact}); + if (!Check(!NativePreflightAllowsDispatch(incomplete) && incomplete.reason == NativePreflightReason::kUnclassified, + "v1-incomplete-metadata-is-always-unclassified")) return false; + const NativePreflightDecision unclassified = ClassifyNativePreflightFacts({true, false, secure, high_impact}); + if (!Check(!NativePreflightAllowsDispatch(unclassified) && unclassified.reason == NativePreflightReason::kUnclassified, + "v1-unclassified-is-never-ordinary")) return false; + } + if (!Check(NativePreflightAllowsDispatch(ClassifyNativePreflightFacts({true, true, false, false})), + "v1-complete-classified-and-quiet-is-ordinary")) return false; + std::cout << "Computer Use native Windows V1 default risk: PASS\n"; + return true; +} + bool CheckWindowsShellNameMatch() { using namespace sprint_coder::computer_use; // A candidate: the file system's long name and the name the SHELL shows for it. @@ -8321,6 +8350,7 @@ int main() { if (!CheckNativeInterlockCapture()) return 1; if (!CheckWindowsInterlockCapture()) return 1; if (!CheckWindowsShellNameMatch()) return 1; + if (!CheckWindowsV1DefaultRisk()) return 1; if (!CheckClickRule()) return 1; if (!CheckInterlockDispatchCodes()) return 1; if (!CheckWindowsInterlockJourneys()) return 1; diff --git a/apps/desktop/computer-use-native/computer_use_windows_host.cc b/apps/desktop/computer-use-native/computer_use_windows_host.cc index dd2b671a5..dcc40b3c3 100644 --- a/apps/desktop/computer-use-native/computer_use_windows_host.cc +++ b/apps/desktop/computer-use-native/computer_use_windows_host.cc @@ -2440,11 +2440,220 @@ std::uint32_t WindowsBuildNumber() { return version.dwBuildNumber; } +// The bounds on the UI Automation calls of the V1 lane (observe, the point / focus classification, the +// control bindings, the semantic target search, the focused-element signature that the re-validation +// uses). Without them a target that does not answer holds the serial lane for good, and every later +// frame (observe, preflight, dispatch, Cancel) waits behind it (#820). Three rules: +// +// 1. ONE call is bounded. The CONNECTION timeout (500 ms) bounds the first contact with a target and +// the TRANSACTION timeout (300 ms) a call on an element already connected (measured on a window that +// does not answer: 311 ms for 300 and 2001 ms for 2000; the connection timeout is the one that +// governs a first contact). 500 ms leaves a cold app time to load its providers on the first call (a +// call that still times out is reported as accessibility_timeout, a failure that a new observe does +// not repeat once the providers are loaded). A call that waits for the connection and then for the +// transaction is bounded by BOTH: the worst case of one call is counted as 500 + 300 = 800 ms +// (whether the two waits can really follow each other is not established; the sum is the safe +// side), inside the 1 s of the Cancel acknowledgment (COMPUTER_USE_NATIVE_CANCEL_ACK_TIMEOUT_MS). +// The margin is SMALL: 1000 - 800 = about 200 ms for everything else the lane does between the +// epoch moving and the acknowledgment (the pipe, the frame, the answer). Measured on CI: a +// transaction-bound call is cut after 310-320 ms, a first contact after 505-514 ms. The sum of +// these two measurements, taken one at a time, is about 830 ms (the chain itself was not measured; +// 800 ms is the nominal value, not a measured upper bound), which leaves about 170 ms for the pipe, +// the frame and the acknowledgment together; under load it can be less (not verified). +// 2. The FIRST timed-out call (UIA_E_TIMEOUT or an equivalent failure) ends its path for good: no +// UI Automation call after it, fail closed, the "unreadable" of the caller. A target that answers +// the first contact and then stops therefore costs one timeout, not one per remaining call. +// 3. A CANCEL ends the path too, and it is asked before EVERY call (WindowsV1UiaCall, and the entry of +// every read of properties): the epoch is the one that was current when the path began (observe: +// the session's own). So a Cancel that arrives while the lane reads a slow but live app waits for +// at most the call that is under way: one call, 800 ms at the worst, never the whole path. +// There is deliberately NO time budget on a path: a deep tree of a live app (an Electron or Office +// window read up the parents, one call each) can take a long time while every call answers, and +// cutting it short would turn a live app into "unclassified" and change what V1 answers. A Cancel needs +// no budget either (rule 3). The one budget is the whole observe (25 s), and it is there only so that +// the native answers with its own reason (observation_timeout, with the number of nodes it had read +// logged) before the 30 s Main gives an observe runs out; it is wide because a tree of several thousand +// nodes of an ordinary app is read four times (two readings, two binding captures) and must not fail. +// 4. A call that "succeeds" only after (nearly) the whole timeout has not answered either: some +// systems return a default value for a property when the provider times out (measured on a CI runner: +// five property reads of a native provider that does not answer took 1.5 s and all succeeded, with +// empty values). So the time of each call is measured, and a call that took within 30 ms of its +// timeout is a timed-out call (WindowsV1UiaTimed), whatever the HRESULT says. +// A failed (timed out, canceled, over the observe budget) read is exactly the "unreadable" of each +// caller (stale target, rejected action, or an observation that is not produced): the meaning of V1's +// answers is not changed. +constexpr DWORD kWindowsV1UiaConnectionTimeoutMs = 500; +constexpr DWORD kWindowsV1UiaTransactionTimeoutMs = 300; +constexpr std::chrono::milliseconds kWindowsV1UiaObserveBudget{25'000}; + +// The ONE place the V1 lane creates UI Automation. The bounded one when the system has it +// (CreateBoundedAutomation); if IUIAutomation2 cannot be had it falls back to the plain one, as before +// this bound existed (no timeout), and counts it, so that the missing bound is visible and not silent. +std::atomic windows_v1_uia_unbounded_creations{0}; +HRESULT CreateWindowsV1Automation(IUIAutomation **automation) { + *automation = nullptr; + if (sprint_coder::computer_use::windows_uia_source_detail::CreateBoundedAutomation( + kWindowsV1UiaConnectionTimeoutMs, kWindowsV1UiaTransactionTimeoutMs, automation)) + return S_OK; + if (windows_v1_uia_unbounded_creations.fetch_add(1) == 0) + OutputDebugStringA("computer-use: V1 UI Automation without a call timeout (IUIAutomation2 unavailable)\n"); + return CoCreateInstance(CLSID_CUIAutomation, nullptr, CLSCTX_INPROC_SERVER, + IID_PPV_ARGS(automation)); +} + +enum class WindowsV1UiaStop { kNone, kTimeout, kBudget, kCanceled }; + +struct WindowsV1UiaLane { + bool has_deadline = false; + std::chrono::steady_clock::time_point deadline; + std::uint64_t epoch = 0; + std::size_t nodes = 0; // the nodes of the walks read so far (for the log of a spent budget) + WindowsV1UiaStop stop = WindowsV1UiaStop::kNone; +}; +thread_local WindowsV1UiaLane *windows_v1_uia_lane = nullptr; + +// One path of UI Automation calls on the lane. The OUTERMOST scope owns the state; one inside it (the +// classification inside a dispatch's own scope, the signature inside a re-validation) shares it. A path +// always watches the cancellation epoch; only the observe has a time budget. +class WindowsV1UiaPath { +public: + // A path that watches the cancellation epoch as it is now, with no time budget. + WindowsV1UiaPath() + : WindowsV1UiaPath(std::chrono::milliseconds::zero(), + cancellation_epoch.load(std::memory_order_acquire)) {} + WindowsV1UiaPath(std::chrono::milliseconds budget, std::uint64_t epoch) { + if (windows_v1_uia_lane != nullptr) + return; + if (budget > std::chrono::milliseconds::zero()) { + lane_.has_deadline = true; + lane_.deadline = std::chrono::steady_clock::now() + budget; + } + lane_.epoch = epoch; + windows_v1_uia_lane = &lane_; + owner_ = true; + } + ~WindowsV1UiaPath() { + if (owner_) + windows_v1_uia_lane = nullptr; + } + WindowsV1UiaPath(const WindowsV1UiaPath &) = delete; + WindowsV1UiaPath &operator=(const WindowsV1UiaPath &) = delete; + [[nodiscard]] WindowsV1UiaStop Stop() const { return windows_v1_uia_lane != nullptr ? windows_v1_uia_lane->stop : WindowsV1UiaStop::kNone; } + [[nodiscard]] std::size_t Nodes() const { return windows_v1_uia_lane != nullptr ? windows_v1_uia_lane->nodes : 0; } + +private: + WindowsV1UiaLane lane_; + bool owner_ = false; +}; + +// Whether the path may make one more UI Automation call. False for good once a call timed out, the +// action was canceled or (the observe) the budget is spent. Outside any path: false (fail closed). Every +// V1 call is reached from inside a path; a call that is not would watch neither a Cancel nor the first +// timeout, so it is refused instead of made unwatched. +bool WindowsV1UiaMayCall() { + WindowsV1UiaLane *lane = windows_v1_uia_lane; + if (lane == nullptr) + return false; + if (lane->stop == WindowsV1UiaStop::kNone) { + if (cancellation_epoch.load(std::memory_order_acquire) != lane->epoch) + lane->stop = WindowsV1UiaStop::kCanceled; + else if (lane->has_deadline && std::chrono::steady_clock::now() > lane->deadline) + lane->stop = WindowsV1UiaStop::kBudget; + } + return lane->stop == WindowsV1UiaStop::kNone; +} + +constexpr HRESULT kWindowsV1UiaTimeoutResult = static_cast(0x80131505); // UIA_E_TIMEOUT + +// A timeout of UI Automation or of the RPC under it. +bool WindowsV1UiaIsTimeout(HRESULT result) { + constexpr HRESULT kWin32Timeout = static_cast(0x800705B4); // HRESULT_FROM_WIN32(ERROR_TIMEOUT) + constexpr HRESULT kRpcTimeout = static_cast(0x8001011F); // RPC_E_TIMEOUT + return result == kWindowsV1UiaTimeoutResult || result == kWin32Timeout || result == kRpcTimeout; +} + +// The result of a UI Automation call, as success or not. A timed-out call also ends its path. +bool WindowsV1UiaOk(HRESULT result) { + if (SUCCEEDED(result)) + return true; + WindowsV1UiaLane *lane = windows_v1_uia_lane; + if (lane != nullptr && lane->stop == WindowsV1UiaStop::kNone && WindowsV1UiaIsTimeout(result)) + lane->stop = WindowsV1UiaStop::kTimeout; + return false; +} + +// A call that took this long and still "succeeded" has been cut off by its timeout (30 ms under it): +// 270 ms against the 300 ms transaction timeout, 470 ms against the 500 ms connection timeout. +constexpr std::chrono::milliseconds kWindowsV1UiaSlowRead{270}; +constexpr std::chrono::milliseconds kWindowsV1UiaSlowContact{470}; +static_assert(kWindowsV1UiaSlowRead.count() < kWindowsV1UiaTransactionTimeoutMs, "a read is slow before it times out"); +static_assert(kWindowsV1UiaSlowContact.count() < kWindowsV1UiaConnectionTimeoutMs, "a contact is slow before it times out"); + +// One UI Automation call of a path: the cancel / timeout / budget check first (E_ABORT, no call made, +// when the path has ended), then the call, timed. A call that came back only after nearly its whole +// timeout is a timed-out call (UIA_E_TIMEOUT), even if it says S_OK. +template HRESULT WindowsV1UiaTimed(std::chrono::milliseconds slow, Call &&call) { + if (!WindowsV1UiaMayCall()) + return E_ABORT; + const auto start = std::chrono::steady_clock::now(); + HRESULT result = call(); + if (SUCCEEDED(result) && std::chrono::steady_clock::now() - start >= slow) + result = kWindowsV1UiaTimeoutResult; + return result; +} +// A call on a connected element (walker moves, the parent). +template HRESULT WindowsV1UiaCall(Call &&call) { + return WindowsV1UiaTimed(kWindowsV1UiaSlowRead, call); +} +// The first contact with a target (ElementFromHandle / ElementFromPoint / GetFocusedElement). +template HRESULT WindowsV1UiaContact(Call &&call) { + return WindowsV1UiaTimed(kWindowsV1UiaSlowContact, call); +} +// One read of a property: true only for a call that answered in time. +template bool WindowsV1UiaRead(Call &&call) { + return WindowsV1UiaOk(WindowsV1UiaTimed(kWindowsV1UiaSlowRead, call)); +} + +// Why the path ended, now (kNone outside a path). +WindowsV1UiaStop WindowsV1UiaCurrentStop() { + return windows_v1_uia_lane != nullptr ? windows_v1_uia_lane->stop : WindowsV1UiaStop::kNone; +} + +// The reason of an action that was NOT sent because the path had ended (a timeout, a Cancel): nothing +// was done, so it is a rejection before any input. +const char *WindowsV1UiaEndedReason() { + return WindowsV1UiaCurrentStop() == WindowsV1UiaStop::kTimeout ? "accessibility_timeout" + : "semantic_action_failed"; +} + +// One node of a walk was visited. +void WindowsV1UiaCountNode() { + if (windows_v1_uia_lane != nullptr) + ++windows_v1_uia_lane->nodes; +} + +// The reason an observe gives for a read that did not complete: a target that did not answer +// (accessibility_timeout: transient, a new observe can succeed, for example once a cold app has loaded +// its providers), a spent budget (observation_timeout) or a cancel (observation_stale); else the reason +// the step had. +const char *WindowsV1ObserveReason(WindowsV1UiaStop stop, const char *otherwise) { + switch (stop) { + case WindowsV1UiaStop::kTimeout: + return "accessibility_timeout"; + case WindowsV1UiaStop::kBudget: + return "observation_timeout"; + case WindowsV1UiaStop::kCanceled: + return "observation_stale"; + case WindowsV1UiaStop::kNone: + break; + } + return otherwise; +} + bool HasUiAutomation() { IUIAutomation *automation = nullptr; const HRESULT result = - CoCreateInstance(CLSID_CUIAutomation, nullptr, CLSCTX_INPROC_SERVER, - IID_PPV_ARGS(&automation)); + CreateWindowsV1Automation(&automation); if (FAILED(result) || automation == nullptr) return false; automation->Release(); @@ -2986,20 +3195,21 @@ bool AppendUiaTree(IUIAutomationElement *element, IUIAutomationTreeWalker *walker, std::size_t depth, std::size_t *nodes, std::string *json) { if (element == nullptr || walker == nullptr || depth > 16 || - *nodes >= 5'000 || json->size() > 512 * 1024) + *nodes >= 5'000 || json->size() > 512 * 1024 || !WindowsV1UiaMayCall()) return false; ++*nodes; + WindowsV1UiaCountNode(); BSTR name = nullptr; BSTR automation_id = nullptr; BSTR help = nullptr; CONTROLTYPEID control_type = 0; BOOL password = FALSE; const bool attributes_read = - SUCCEEDED(element->get_CurrentName(&name)) && - SUCCEEDED(element->get_CurrentAutomationId(&automation_id)) && - SUCCEEDED(element->get_CurrentHelpText(&help)) && - SUCCEEDED(element->get_CurrentControlType(&control_type)) && - SUCCEEDED(element->get_CurrentIsPassword(&password)); + WindowsV1UiaRead([&] { return element->get_CurrentName(&name); }) && + WindowsV1UiaRead([&] { return element->get_CurrentAutomationId(&automation_id); }) && + WindowsV1UiaRead([&] { return element->get_CurrentHelpText(&help); }) && + WindowsV1UiaRead([&] { return element->get_CurrentControlType(&control_type); }) && + WindowsV1UiaRead([&] { return element->get_CurrentIsPassword(&password); }); if (!attributes_read) { if (name != nullptr) SysFreeString(name); @@ -3053,9 +3263,9 @@ bool AppendUiaTree(IUIAutomationElement *element, *json += ",\"help\":\"" + JsonEscape(help_utf8) + "\""; *json += ",\"children\":["; IUIAutomationElement *child = nullptr; - HRESULT result = walker->GetFirstChildElement(element, &child); + HRESULT result = WindowsV1UiaCall([&] { return walker->GetFirstChildElement(element, &child); }); bool first = true; - bool complete = SUCCEEDED(result); + bool complete = WindowsV1UiaOk(result); while (complete && child != nullptr) { if (!first) *json += ','; @@ -3063,7 +3273,7 @@ bool AppendUiaTree(IUIAutomationElement *element, if (!AppendUiaTree(child, walker, depth + 1, nodes, json)) complete = false; IUIAutomationElement *next = nullptr; - if (complete && FAILED(walker->GetNextSiblingElement(child, &next))) + if (complete && !WindowsV1UiaOk(WindowsV1UiaCall([&] { return walker->GetNextSiblingElement(child, &next); }))) complete = false; child->Release(); child = next; @@ -3104,19 +3314,19 @@ bool IsProcessElevated(std::uint32_t pid) { bool ReadUiaObservation(std::uint32_t pid, HWND window, std::string *tree, std::size_t *nodes) { + const WindowsV1UiaPath path; IUIAutomation *automation = nullptr; - if (FAILED(CoCreateInstance(CLSID_CUIAutomation, nullptr, - CLSCTX_INPROC_SERVER, - IID_PPV_ARGS(&automation))) || + if (FAILED(CreateWindowsV1Automation(&automation)) || automation == nullptr) return false; IUIAutomationElement *root = nullptr; IUIAutomationTreeWalker *walker = nullptr; - const HRESULT root_result = automation->ElementFromHandle(window, &root); + const HRESULT root_result = + WindowsV1UiaContact([&] { return automation->ElementFromHandle(window, &root); }); const HRESULT walker_result = automation->get_RawViewWalker(&walker); std::string value; std::size_t count = 0; - const bool complete = SUCCEEDED(root_result) && SUCCEEDED(walker_result) && + const bool complete = WindowsV1UiaOk(root_result) && SUCCEEDED(walker_result) && root != nullptr && walker != nullptr && AppendUiaTree(root, walker, 0, &count, &value); if (root != nullptr) @@ -3229,7 +3439,7 @@ bool ContainsWindowsRiskText(std::string_view text) { WindowsUiaRisk ClassifyWindowsUiaElementSelf(IUIAutomationElement *element) { WindowsUiaRisk risk; - if (element == nullptr) + if (element == nullptr || !WindowsV1UiaMayCall()) return risk; BSTR name = nullptr; BSTR automation_id = nullptr; @@ -3237,11 +3447,11 @@ WindowsUiaRisk ClassifyWindowsUiaElementSelf(IUIAutomationElement *element) { CONTROLTYPEID control_type = 0; BOOL password = FALSE; const bool attributes_read = - SUCCEEDED(element->get_CurrentName(&name)) && - SUCCEEDED(element->get_CurrentAutomationId(&automation_id)) && - SUCCEEDED(element->get_CurrentHelpText(&help)) && - SUCCEEDED(element->get_CurrentControlType(&control_type)) && - SUCCEEDED(element->get_CurrentIsPassword(&password)); + WindowsV1UiaRead([&] { return element->get_CurrentName(&name); }) && + WindowsV1UiaRead([&] { return element->get_CurrentAutomationId(&automation_id); }) && + WindowsV1UiaRead([&] { return element->get_CurrentHelpText(&help); }) && + WindowsV1UiaRead([&] { return element->get_CurrentControlType(&control_type); }) && + WindowsV1UiaRead([&] { return element->get_CurrentIsPassword(&password); }); if (!attributes_read) { if (name != nullptr) SysFreeString(name); @@ -3297,11 +3507,10 @@ WindowsUiaRisk ClassifyWindowsUiaElement(IUIAutomationElement *element) { WindowsUiaRisk merged; if (element == nullptr) return merged; + const WindowsV1UiaPath path; IUIAutomation *automation = nullptr; IUIAutomationTreeWalker *walker = nullptr; - if (FAILED(CoCreateInstance(CLSID_CUIAutomation, nullptr, - CLSCTX_INPROC_SERVER, - IID_PPV_ARGS(&automation))) || + if (FAILED(CreateWindowsV1Automation(&automation)) || automation == nullptr || FAILED(automation->get_RawViewWalker(&walker)) || walker == nullptr) { if (walker != nullptr) @@ -3319,10 +3528,19 @@ WindowsUiaRisk ClassifyWindowsUiaElement(IUIAutomationElement *element) { merged.secure = merged.secure || risk.secure; merged.high_impact = merged.high_impact || risk.high_impact; metadata_complete = metadata_complete && risk.metadata_complete; + // An element that could not be read ends the climb: the facts are incomplete whatever the parents + // say (ClassifyNativePreflightFacts puts incomplete metadata ahead of secure and high impact, so + // the reason does not change), and asking the parents of a target that stopped answering costs one + // more timeout each. + if (!risk.metadata_complete) + break; IUIAutomationElement *parent = nullptr; - const HRESULT parent_result = walker->GetParentElement(current, &parent); - if (FAILED(parent_result)) { + const HRESULT parent_result = + WindowsV1UiaCall([&] { return walker->GetParentElement(current, &parent); }); + if (!WindowsV1UiaOk(parent_result)) { metadata_complete = false; + if (parent != nullptr) + parent->Release(); parent = nullptr; } current->Release(); @@ -3342,51 +3560,54 @@ WindowsUiaRisk ClassifyWindowsPoint(POINT point, std::uint32_t expected_pid, std::string *control_signature = nullptr) { if (control_signature != nullptr) control_signature->clear(); + const WindowsV1UiaPath path; IUIAutomation *automation = nullptr; - if (FAILED(CoCreateInstance(CLSID_CUIAutomation, nullptr, - CLSCTX_INPROC_SERVER, - IID_PPV_ARGS(&automation))) || + if (FAILED(CreateWindowsV1Automation(&automation)) || automation == nullptr) return {}; IUIAutomationElement *element = nullptr; const POINT query = point; - const HRESULT found = automation->ElementFromPoint(query, &element); + const HRESULT found = + WindowsV1UiaContact([&] { return automation->ElementFromPoint(query, &element); }); automation->Release(); - if (FAILED(found) || element == nullptr) { + if (!WindowsV1UiaOk(found) || element == nullptr) { if (element != nullptr) element->Release(); return {}; } int owner_pid = 0; - if (FAILED(element->get_CurrentProcessId(&owner_pid)) || owner_pid <= 0 || + if (!WindowsV1UiaRead([&] { return element->get_CurrentProcessId(&owner_pid); }) || owner_pid <= 0 || static_cast(owner_pid) != expected_pid) { element->Release(); return {}; } const WindowsUiaRisk risk = ClassifyWindowsUiaElement(element); - if (control_signature != nullptr) + // The control signature is only for a target that is going to be acted on: when the facts are + // incomplete the caller rejects the action and never reads it, so it is not asked for (one more call + // to an element that has just failed to answer). + if (control_signature != nullptr && risk.metadata_complete) ComputeWindowsControlSignature(element, expected_pid, nullptr, control_signature); element->Release(); return risk; } WindowsUiaRisk ClassifyWindowsFocusedElement(std::uint32_t expected_pid) { + const WindowsV1UiaPath path; IUIAutomation *automation = nullptr; - if (FAILED(CoCreateInstance(CLSID_CUIAutomation, nullptr, - CLSCTX_INPROC_SERVER, - IID_PPV_ARGS(&automation))) || + if (FAILED(CreateWindowsV1Automation(&automation)) || automation == nullptr) return {}; IUIAutomationElement *element = nullptr; - const HRESULT found = automation->GetFocusedElement(&element); + const HRESULT found = + WindowsV1UiaContact([&] { return automation->GetFocusedElement(&element); }); automation->Release(); - if (FAILED(found) || element == nullptr) { + if (!WindowsV1UiaOk(found) || element == nullptr) { if (element != nullptr) element->Release(); return {}; } int owner_pid = 0; - if (FAILED(element->get_CurrentProcessId(&owner_pid)) || owner_pid <= 0 || + if (!WindowsV1UiaRead([&] { return element->get_CurrentProcessId(&owner_pid); }) || owner_pid <= 0 || static_cast(owner_pid) != expected_pid) { element->Release(); return {}; @@ -3400,7 +3621,7 @@ bool ComputeWindowsControlSignature(IUIAutomationElement *element, std::uint32_t expected_pid, const RECT *containing_bounds, std::string *signature) { - if (element == nullptr || signature == nullptr) + if (element == nullptr || signature == nullptr || !WindowsV1UiaMayCall()) return false; int owner_pid = 0; CONTROLTYPEID control_type = 0; @@ -3408,18 +3629,18 @@ bool ComputeWindowsControlSignature(IUIAutomationElement *element, BSTR automation_id = nullptr; SAFEARRAY *runtime_id = nullptr; const bool facts = - SUCCEEDED(element->get_CurrentProcessId(&owner_pid)) && owner_pid > 0 && + WindowsV1UiaRead([&] { return element->get_CurrentProcessId(&owner_pid); }) && owner_pid > 0 && static_cast(owner_pid) == expected_pid && - SUCCEEDED(element->get_CurrentControlType(&control_type)) && - SUCCEEDED(element->get_CurrentBoundingRectangle(&bounds)) && + WindowsV1UiaRead([&] { return element->get_CurrentControlType(&control_type); }) && + WindowsV1UiaRead([&] { return element->get_CurrentBoundingRectangle(&bounds); }) && bounds.right > bounds.left && bounds.bottom > bounds.top && (containing_bounds == nullptr || (bounds.left >= containing_bounds->left && bounds.top >= containing_bounds->top && bounds.right <= containing_bounds->right && bounds.bottom <= containing_bounds->bottom)) && - SUCCEEDED(element->get_CurrentAutomationId(&automation_id)) && - SUCCEEDED(element->GetRuntimeId(&runtime_id)) && runtime_id != nullptr && + WindowsV1UiaRead([&] { return element->get_CurrentAutomationId(&automation_id); }) && + WindowsV1UiaRead([&] { return element->GetRuntimeId(&runtime_id); }) && runtime_id != nullptr && SafeArrayGetDim(runtime_id) == 1; if (!facts) { if (automation_id != nullptr) @@ -3478,8 +3699,8 @@ bool ReadWindowsElementTargetId(IUIAutomationElement *element, BSTR automation_id = nullptr; BSTR name = nullptr; const bool read = - SUCCEEDED(element->get_CurrentAutomationId(&automation_id)) && - SUCCEEDED(element->get_CurrentName(&name)); + WindowsV1UiaRead([&] { return element->get_CurrentAutomationId(&automation_id); }) && + WindowsV1UiaRead([&] { return element->get_CurrentName(&name); }); if (!read) { if (automation_id != nullptr) SysFreeString(automation_id); @@ -3521,9 +3742,10 @@ bool CollectWindowsControlBindings(IUIAutomationElement *element, WindowsControlBindings *bindings, std::set *duplicate_targets) { if (element == nullptr || walker == nullptr || nodes == nullptr || bindings == nullptr || - duplicate_targets == nullptr || depth > 16 || *nodes >= 5'000) + duplicate_targets == nullptr || depth > 16 || *nodes >= 5'000 || !WindowsV1UiaMayCall()) return false; ++*nodes; + WindowsV1UiaCountNode(); std::string signature; if (!ComputeWindowsControlSignature(element, expected_pid, nullptr, &signature)) @@ -3545,9 +3767,13 @@ bool CollectWindowsControlBindings(IUIAutomationElement *element, } IUIAutomationElement *child = nullptr; - const HRESULT first_result = walker->GetFirstChildElement(element, &child); - if (FAILED(first_result)) + const HRESULT first_result = + WindowsV1UiaCall([&] { return walker->GetFirstChildElement(element, &child); }); + if (!WindowsV1UiaOk(first_result)) { + if (child != nullptr) + child->Release(); return false; + } while (child != nullptr) { if (!CollectWindowsControlBindings(child, walker, expected_pid, depth + 1, nodes, bindings, duplicate_targets)) { @@ -3555,9 +3781,10 @@ bool CollectWindowsControlBindings(IUIAutomationElement *element, return false; } IUIAutomationElement *next = nullptr; - const HRESULT next_result = walker->GetNextSiblingElement(child, &next); + const HRESULT next_result = + WindowsV1UiaCall([&] { return walker->GetNextSiblingElement(child, &next); }); child->Release(); - if (FAILED(next_result)) { + if (!WindowsV1UiaOk(next_result)) { if (next != nullptr) next->Release(); return false; @@ -3571,15 +3798,16 @@ bool CaptureWindowsControlBindings(std::uint32_t pid, HWND window, WindowsControlBindings *bindings) { if (bindings == nullptr) return false; + const WindowsV1UiaPath path; IUIAutomation *automation = nullptr; - if (FAILED(CoCreateInstance(CLSID_CUIAutomation, nullptr, CLSCTX_INPROC_SERVER, - IID_PPV_ARGS(&automation))) || + if (FAILED(CreateWindowsV1Automation(&automation)) || automation == nullptr) return false; IUIAutomationElement *root = nullptr; IUIAutomationTreeWalker *walker = nullptr; const bool initialized = - SUCCEEDED(automation->ElementFromHandle(window, &root)) && root != nullptr && + WindowsV1UiaOk(WindowsV1UiaContact([&] { return automation->ElementFromHandle(window, &root); })) && + root != nullptr && SUCCEEDED(automation->get_RawViewWalker(&walker)) && walker != nullptr; std::size_t nodes = 0; std::set duplicate_targets; @@ -3599,16 +3827,16 @@ bool WindowsFocusedElementSignature(std::uint32_t expected_pid, std::string *signature) { if (signature == nullptr) return false; + const WindowsV1UiaPath path; IUIAutomation *automation = nullptr; - if (FAILED(CoCreateInstance(CLSID_CUIAutomation, nullptr, - CLSCTX_INPROC_SERVER, - IID_PPV_ARGS(&automation))) || + if (FAILED(CreateWindowsV1Automation(&automation)) || automation == nullptr) return false; IUIAutomationElement *element = nullptr; - const HRESULT found = automation->GetFocusedElement(&element); + const HRESULT found = + WindowsV1UiaContact([&] { return automation->GetFocusedElement(&element); }); automation->Release(); - if (FAILED(found) || element == nullptr) { + if (!WindowsV1UiaOk(found) || element == nullptr) { if (element != nullptr) element->Release(); return false; @@ -3644,14 +3872,15 @@ bool FindUiaTargetRecursive(IUIAutomationElement *element, std::size_t *visited, IUIAutomationElement **match, bool *duplicate) { if (element == nullptr || walker == nullptr || depth > 16 || - *visited >= 5'000) + *visited >= 5'000 || !WindowsV1UiaMayCall()) return false; ++*visited; + WindowsV1UiaCountNode(); BSTR automation_id = nullptr; BSTR name = nullptr; const bool attributes_read = - SUCCEEDED(element->get_CurrentAutomationId(&automation_id)) && - SUCCEEDED(element->get_CurrentName(&name)); + WindowsV1UiaRead([&] { return element->get_CurrentAutomationId(&automation_id); }) && + WindowsV1UiaRead([&] { return element->get_CurrentName(&name); }); if (!attributes_read) { if (automation_id != nullptr) SysFreeString(automation_id); @@ -3682,8 +3911,11 @@ bool FindUiaTargetRecursive(IUIAutomationElement *element, } } IUIAutomationElement *child = nullptr; - if (FAILED(walker->GetFirstChildElement(element, &child))) + if (!WindowsV1UiaOk(WindowsV1UiaCall([&] { return walker->GetFirstChildElement(element, &child); }))) { + if (child != nullptr) + child->Release(); return false; + } while (child != nullptr && !*duplicate) { if (!FindUiaTargetRecursive(child, walker, target_id, depth + 1, visited, match, duplicate)) { @@ -3691,7 +3923,9 @@ bool FindUiaTargetRecursive(IUIAutomationElement *element, return false; } IUIAutomationElement *next = nullptr; - if (FAILED(walker->GetNextSiblingElement(child, &next))) { + if (!WindowsV1UiaOk(WindowsV1UiaCall([&] { return walker->GetNextSiblingElement(child, &next); }))) { + if (next != nullptr) + next->Release(); child->Release(); return false; } @@ -3712,17 +3946,16 @@ bool FindWindowsSemanticTarget(const WindowsSession &session, return false; } IUIAutomation *automation = nullptr; - if (FAILED(CoCreateInstance(CLSID_CUIAutomation, nullptr, - CLSCTX_INPROC_SERVER, - IID_PPV_ARGS(&automation))) || + if (FAILED(CreateWindowsV1Automation(&automation)) || automation == nullptr) { *reason = "accessibility_unavailable"; return false; } IUIAutomationElement *root = nullptr; IUIAutomationTreeWalker *walker = nullptr; + const WindowsV1UiaPath path; const bool initialized = - SUCCEEDED(automation->ElementFromHandle(session.active_window, &root)) && + WindowsV1UiaOk(WindowsV1UiaContact([&] { return automation->ElementFromHandle(session.active_window, &root); })) && root != nullptr && SUCCEEDED(automation->get_RawViewWalker(&walker)) && walker != nullptr; std::size_t visited = 0; @@ -3739,7 +3972,10 @@ bool FindWindowsSemanticTarget(const WindowsSession &session, if (!searched || duplicate || match == nullptr) { if (match != nullptr) match->Release(); - *reason = duplicate ? "target_ambiguous" : "target_required"; + // A call that timed out is not "no such target": the target did not answer (transient). + *reason = duplicate ? "target_ambiguous" + : path.Stop() == WindowsV1UiaStop::kTimeout ? "accessibility_timeout" + : "target_required"; return false; } *output = match; @@ -3763,6 +3999,22 @@ bool Utf16FromUtf8(std::string_view value, std::wstring *output) { return true; } +// A pattern of the target, read through the timed read. A read that did not complete (it timed out, or +// "succeeded" too slowly to be believed) hands nothing back: a pattern it did produce is released here, +// so the caller never holds, or acts on, the answer of a target that stopped answering. +template +bool ReadWindowsV1Pattern(IUIAutomationElement *target, PATTERNID id, Pattern **pattern) { + *pattern = nullptr; + const bool read = WindowsV1UiaRead([&] { return target->GetCurrentPatternAs(id, IID_PPV_ARGS(pattern)); }); + if (!read || *pattern == nullptr) { + if (*pattern != nullptr) + (*pattern)->Release(); + *pattern = nullptr; + return false; + } + return true; +} + bool DispatchWindowsSemanticAction(const WindowsSession &session, const RECT &expected, std::uint64_t expected_epoch, @@ -3800,12 +4052,16 @@ bool DispatchWindowsSemanticAction(const WindowsSession &session, HRESULT action_result = E_FAIL; if (kind == "invoke") { IUIAutomationInvokePattern *pattern = nullptr; - if (SUCCEEDED(target->GetCurrentPatternAs(UIA_InvokePatternId, - IID_PPV_ARGS(&pattern))) && - pattern != nullptr) { - *accepted = true; - session.input_api_attempts->fetch_add(1, std::memory_order_acq_rel); - action_result = pattern->Invoke(); + if (ReadWindowsV1Pattern(target, UIA_InvokePatternId, &pattern)) { + // The action is SENT only while the path is alive: after a timeout or a Cancel nothing is done + // (a rejection before any input), never a call that cannot be confirmed. + if (!WindowsV1UiaMayCall()) { + *reason = WindowsV1UiaEndedReason(); + } else { + *accepted = true; + session.input_api_attempts->fetch_add(1, std::memory_order_acq_rel); + action_result = pattern->Invoke(); + } pattern->Release(); } else { *reason = "semantic_pattern_unavailable"; @@ -3817,17 +4073,21 @@ bool DispatchWindowsSemanticAction(const WindowsSession &session, if (!ReadJsonString(metadata, "text", &text) || !Utf16FromUtf8(text, &wide_text)) { *reason = "invalid_text"; - } else if (SUCCEEDED(target->GetCurrentPatternAs(UIA_ValuePatternId, - IID_PPV_ARGS(&pattern))) && - pattern != nullptr) { + } else if (ReadWindowsV1Pattern(target, UIA_ValuePatternId, &pattern)) { BSTR value = SysAllocStringLen(wide_text.data(), static_cast(wide_text.size())); if (value == nullptr && !wide_text.empty()) { *reason = "semantic_action_failed"; } else { - *accepted = true; - session.input_api_attempts->fetch_add(1, std::memory_order_acq_rel); - action_result = pattern->SetValue(value); + // The action is SENT only while the path is alive: after a timeout or a Cancel nothing is done + // (a rejection before any input), never a call that cannot be confirmed. + if (!WindowsV1UiaMayCall()) { + *reason = WindowsV1UiaEndedReason(); + } else { + *accepted = true; + session.input_api_attempts->fetch_add(1, std::memory_order_acq_rel); + action_result = pattern->SetValue(value); + } } if (value != nullptr) SysFreeString(value); @@ -3839,22 +4099,29 @@ bool DispatchWindowsSemanticAction(const WindowsSession &session, std::string selected_value; IUIAutomationSelectionItemPattern *pattern = nullptr; BSTR name = nullptr; - target->get_CurrentName(&name); + const bool name_read = WindowsV1UiaRead([&] { return target->get_CurrentName(&name); }); const std::string target_name = - name == nullptr + name == nullptr || !name_read ? std::string{} : Utf8FromWide(std::wstring_view(name, SysStringLen(name))); if (name != nullptr) SysFreeString(name); - if (!ReadJsonString(metadata, "value", &selected_value) || + if (!name_read) { + // The target did not answer: nothing is compared with a name that was not read, and nothing is sent. + *reason = WindowsV1UiaEndedReason(); + } else if (!ReadJsonString(metadata, "value", &selected_value) || (selected_value != target_id && selected_value != target_name)) { *reason = "semantic_value_mismatch"; - } else if (SUCCEEDED(target->GetCurrentPatternAs(UIA_SelectionItemPatternId, - IID_PPV_ARGS(&pattern))) && - pattern != nullptr) { - *accepted = true; - session.input_api_attempts->fetch_add(1, std::memory_order_acq_rel); - action_result = pattern->Select(); + } else if (ReadWindowsV1Pattern(target, UIA_SelectionItemPatternId, &pattern)) { + // The action is SENT only while the path is alive: after a timeout or a Cancel nothing is done + // (a rejection before any input), never a call that cannot be confirmed. + if (!WindowsV1UiaMayCall()) { + *reason = WindowsV1UiaEndedReason(); + } else { + *accepted = true; + session.input_api_attempts->fetch_add(1, std::memory_order_acq_rel); + action_result = pattern->Select(); + } pattern->Release(); } else { *reason = "semantic_pattern_unavailable"; @@ -3866,21 +4133,26 @@ bool DispatchWindowsSemanticAction(const WindowsSession &session, metadata.find("\"value\":false") != std::string::npos; IUIAutomationTogglePattern *pattern = nullptr; if ((!wants_true && !wants_false) || - FAILED(target->GetCurrentPatternAs(UIA_TogglePatternId, - IID_PPV_ARGS(&pattern))) || + !ReadWindowsV1Pattern(target, UIA_TogglePatternId, &pattern) || pattern == nullptr) { *reason = "semantic_pattern_unavailable"; } else { ToggleState current = ToggleState_Indeterminate; - if (FAILED(pattern->get_CurrentToggleState(¤t)) || + if (!WindowsV1UiaRead([&] { return pattern->get_CurrentToggleState(¤t); }) || current == ToggleState_Indeterminate) { *reason = "semantic_state_unavailable"; } else if ((current == ToggleState_On) == wants_true) { action_result = S_OK; } else { - *accepted = true; - session.input_api_attempts->fetch_add(1, std::memory_order_acq_rel); - action_result = pattern->Toggle(); + // The action is SENT only while the path is alive: after a timeout or a Cancel nothing is done + // (a rejection before any input), never a call that cannot be confirmed. + if (!WindowsV1UiaMayCall()) { + *reason = WindowsV1UiaEndedReason(); + } else { + *accepted = true; + session.input_api_attempts->fetch_add(1, std::memory_order_acq_rel); + action_result = pattern->Toggle(); + } } pattern->Release(); } @@ -3891,29 +4163,42 @@ bool DispatchWindowsSemanticAction(const WindowsSession &session, metadata.find("\"expanded\":false") != std::string::npos; IUIAutomationExpandCollapsePattern *pattern = nullptr; if ((!wants_expand && !wants_collapse) || - FAILED(target->GetCurrentPatternAs(UIA_ExpandCollapsePatternId, - IID_PPV_ARGS(&pattern))) || + !ReadWindowsV1Pattern(target, UIA_ExpandCollapsePatternId, &pattern) || pattern == nullptr) { *reason = "semantic_pattern_unavailable"; } else { ExpandCollapseState current = ExpandCollapseState_LeafNode; - if (FAILED(pattern->get_CurrentExpandCollapseState(¤t)) || + if (!WindowsV1UiaRead([&] { return pattern->get_CurrentExpandCollapseState(¤t); }) || current == ExpandCollapseState_LeafNode) { *reason = "semantic_state_unavailable"; } else if ((current == ExpandCollapseState_Expanded) == wants_expand) { action_result = S_OK; } else { - *accepted = true; - session.input_api_attempts->fetch_add(1, std::memory_order_acq_rel); - action_result = wants_expand ? pattern->Expand() : pattern->Collapse(); + // The action is SENT only while the path is alive: after a timeout or a Cancel nothing is done + // (a rejection before any input), never a call that cannot be confirmed. + if (!WindowsV1UiaMayCall()) { + *reason = WindowsV1UiaEndedReason(); + } else { + *accepted = true; + session.input_api_attempts->fetch_add(1, std::memory_order_acq_rel); + action_result = wants_expand ? pattern->Expand() : pattern->Collapse(); + } } pattern->Release(); } } target->Release(); + if (FAILED(action_result) && !*accepted && + WindowsV1UiaCurrentStop() == WindowsV1UiaStop::kTimeout) + *reason = "accessibility_timeout"; if (FAILED(action_result)) { if (reason->empty()) - *reason = *accepted ? "input_not_confirmed" : "semantic_action_failed"; + // A pattern call (Invoke, SetValue, Select, Toggle, Expand) that timed out may still have been + // applied by the target: unconfirmed, and told apart from the other failures so that the caller + // does not repeat the action blindly (a retry is the agent's decision, after a new observation). + *reason = !*accepted ? "semantic_action_failed" + : WindowsV1UiaIsTimeout(action_result) ? "input_timeout_unconfirmed" + : "input_not_confirmed"; return false; } return true; @@ -4657,9 +4942,12 @@ bool DispatchWindowsAction(const WindowsSession &session, return false; } if (!dispatched) { - if (*accepted) - *reason = "input_not_confirmed"; - else if (cancellation_epoch.load(std::memory_order_acquire) != + if (*accepted) { + // A pattern call that timed out keeps its own reason (input_timeout_unconfirmed); every other + // input that was sent and not confirmed is input_not_confirmed. + if (*reason != "input_timeout_unconfirmed") + *reason = "input_not_confirmed"; + } else if (cancellation_epoch.load(std::memory_order_acquire) != expected_epoch) *reason = "canceled"; else if (reason->empty()) @@ -5255,6 +5543,18 @@ bool ObserveWindowsWindow(WindowsSession &session, std::string *response, std::string *reason) { RECT bounds{}; WindowsDialogSetSnapshot dialogs; + // The UI Automation reads of the whole observe share one budget and the cancellation epoch (see the + // bounds above). The reason of a read that did not complete says why (WindowsV1ObserveReason). + const WindowsV1UiaPath uia_path(kWindowsV1UiaObserveBudget, session.cancel_epoch); + const auto unreadable = [&uia_path](const char *otherwise) -> const char * { + if (uia_path.Stop() == WindowsV1UiaStop::kBudget) { + // Only a count (never a label or a text): how far the walks got before the budget was spent. + const std::string note = "computer-use: observe UI Automation budget spent after " + + std::to_string(uia_path.Nodes()) + " nodes\n"; + OutputDebugStringA(note.c_str()); + } + return WindowsV1ObserveReason(uia_path.Stop(), otherwise); + }; if (!ValidateWindowsSession(session, &bounds, reason, &dialogs) || session.cancel_epoch != cancellation_epoch.load(std::memory_order_acquire)) @@ -5262,13 +5562,13 @@ bool ObserveWindowsWindow(WindowsSession &session, std::string *response, std::string tree; std::size_t nodes = 0; if (!ReadUiaObservation(session.pid, dialogs.active_window, &tree, &nodes)) { - *reason = "accessibility_unavailable"; + *reason = unreadable("accessibility_unavailable"); return false; } WindowsControlBindings bindings; if (!CaptureWindowsControlBindings(session.pid, dialogs.active_window, &bindings)) { - *reason = "accessibility_binding_unavailable"; + *reason = unreadable("accessibility_binding_unavailable"); return false; } std::string stable_tree; @@ -5276,7 +5576,7 @@ bool ObserveWindowsWindow(WindowsSession &session, std::string *response, if (!ReadUiaObservation(session.pid, dialogs.active_window, &stable_tree, &stable_nodes) || stable_tree != tree || stable_nodes != nodes) { - *reason = "observation_stale"; + *reason = unreadable("observation_stale"); return false; } WindowsControlBindings stable_bindings; @@ -5284,14 +5584,18 @@ bool ObserveWindowsWindow(WindowsSession &session, std::string *response, &stable_bindings) || stable_bindings.semantic != bindings.semantic || stable_bindings.visual != bindings.visual) { - *reason = "observation_stale"; + *reason = unreadable("observation_stale"); return false; } const WindowsUiaRisk focused_risk = ClassifyWindowsFocusedElement(session.pid); + // The focused element's signature is read whatever its classification says, as before #820: a live + // focus that is only unclassified (deeper than the climb, say) keeps its signature, so its key / type + // is refused later as target_unclassified, not as a focus that moved. After a timeout or a Cancel in + // this observe the read is refused by the path without a call (no second wait on the element). std::string focused_element_signature; - const bool has_focused_element_signature = WindowsFocusedElementSignature( - session.pid, bounds, &focused_element_signature); + const bool has_focused_element_signature = + WindowsFocusedElementSignature(session.pid, bounds, &focused_element_signature); RECT screen_bounds{}; if (!ScreenBoundsPhysicalForWindow(dialogs.active_window, &screen_bounds)) { *reason = "screen_geometry_unavailable"; @@ -5513,6 +5817,9 @@ bool HandlePipeRequest(HANDLE pipe, const BackendProbe &probe, break; } case MessageType::kPreflight: { + // One path for the whole frame: the first timed-out UI Automation call of any of its steps ends all + // the later ones, and a Cancel ends them too (the inner paths share this state). + const WindowsV1UiaPath preflight_uia_path; if (!HandleWindowsPreflight(pipe, request, metadata)) return false; break; } @@ -5537,6 +5844,10 @@ bool HandlePipeRequest(HANDLE pipe, const BackendProbe &probe, std::string result; std::string reason; bool accepted = false; + // One path for the whole dispatch (the interlock, the classification, the signatures, the + // re-validation): the first timed-out UI Automation call ends all the later ones, and a Cancel ends + // them too (the inner paths share this state). + const WindowsV1UiaPath dispatch_uia_path; const bool succeeded = DispatchWindowsAction(*session, metadata, &result, &reason, &accepted); if (!RespondAndCache( diff --git a/apps/desktop/computer-use-native/computer_use_windows_uia_source.h b/apps/desktop/computer-use-native/computer_use_windows_uia_source.h index 188e533cb..1fd1faee8 100644 --- a/apps/desktop/computer-use-native/computer_use_windows_uia_source.h +++ b/apps/desktop/computer-use-native/computer_use_windows_uia_source.h @@ -164,6 +164,37 @@ class BstrHolder { }; +// UI Automation created WITH its call timeouts, or not at all. CUIAutomation8 (Windows 8 and later) is +// the only coclass that implements IUIAutomation2, whose ConnectionTimeout / TransactionTimeout bound a +// call to an app that does not answer. The original CUIAutomation does not, and a call to a hung app +// then never returns (measured: about 1.5 s for the first call alone, and without an upper bound after +// it), which holds the thread that made it. False (and *automation null) when the coclass, the +// interface or either timeout is not available. The CONNECTION timeout is what bounds the first contact +// with a target (ElementFromHandle / ElementFromPoint / GetFocusedElement: measured 311 ms for 300 ms +// and 2001 ms for 2000 ms on a window that does not answer, whatever the transaction timeout), the +// TRANSACTION timeout bounds a call on an element that is already connected. The ONE place the shadow source and the V1 lane +// (CreateWindowsV1Automation in the host) get their bounded UI Automation from. +[[nodiscard]] inline bool CreateBoundedAutomation(DWORD connection_timeout_ms, DWORD transaction_timeout_ms, + IUIAutomation **automation) { + *automation = nullptr; + IUIAutomation *created = nullptr; + if (FAILED(CoCreateInstance(CLSID_CUIAutomation8, nullptr, CLSCTX_INPROC_SERVER, + IID_PPV_ARGS(&created))) || + created == nullptr) + return false; + IUIAutomation2 *bounded = nullptr; + const bool set = SUCCEEDED(created->QueryInterface(IID_PPV_ARGS(&bounded))) && bounded != nullptr && + SUCCEEDED(bounded->put_ConnectionTimeout(connection_timeout_ms)) && + SUCCEEDED(bounded->put_TransactionTimeout(transaction_timeout_ms)); + if (bounded != nullptr) bounded->Release(); + if (!set) { + created->Release(); + return false; + } + *automation = created; + return true; +} + // What the shell knows about an item: the path in the file system, and whether it opens by browsing. struct ShellItemAnswer { std::wstring path; @@ -775,22 +806,13 @@ class WindowsUiaShadowSource final : public ShadowUiaSource { DWORD timeout_ms, std::atomic &lookup_slot) { using windows_uia_source_detail::ComRef; - // CUIAutomation8 (Windows 8 and later) is the only coclass that implements IUIAutomation2, whose - // ConnectionTimeout / TransactionTimeout bound a call to an app that does not answer. The - // original CUIAutomation does not, and a call to a hung app then never returns (measured), which - // would hold the shadow's thread and its one slot for the life of the helper. So the timeouts are - // a precondition: without them there is no source, and the capture is recorded as skipped. + // The timeouts are a precondition (see windows_uia_source_detail::CreateBoundedAutomation): a call + // to a hung app would otherwise hold the shadow's thread and its one slot for the life of the + // helper. Without them there is no source, and the capture is recorded as skipped. ComRef automation; - if (FAILED(CoCreateInstance(CLSID_CUIAutomation8, nullptr, CLSCTX_INPROC_SERVER, - IID_PPV_ARGS(automation.Receive()))) || + if (!windows_uia_source_detail::CreateBoundedAutomation(timeout_ms, timeout_ms, automation.Receive()) || automation.Get() == nullptr) return nullptr; - ComRef bounded; - if (FAILED(automation.Get()->QueryInterface(IID_PPV_ARGS(bounded.Receive()))) || - bounded.Get() == nullptr || - FAILED(bounded.Get()->put_ConnectionTimeout(timeout_ms)) || - FAILED(bounded.Get()->put_TransactionTimeout(timeout_ms))) - return nullptr; ComRef walker; if (FAILED(automation.Get()->get_RawViewWalker(walker.Receive())) || walker.Get() == nullptr) return nullptr; diff --git a/apps/desktop/src/main/computer-use-native-host.test.ts b/apps/desktop/src/main/computer-use-native-host.test.ts index 39b4bbb8e..f00744339 100644 --- a/apps/desktop/src/main/computer-use-native-host.test.ts +++ b/apps/desktop/src/main/computer-use-native-host.test.ts @@ -77,6 +77,28 @@ describe('Computer Use native Main adapter', () => { Object.assign(new Error('accepted'), { code: 'STALE_TARGET', accepted: true }), ), ).toBeNull(); + // #820: a V1 target that did not answer. Before any input (accepted false) it is a typed rejection + // that a new observation can answer; once the action was sent (accepted true) a pattern call that + // timed out is unconfirmed and stays an exception for the controller to book as an unknown effect, + // exactly like input_not_confirmed (the agent decides whether to repeat it). + expect( + classifyComputerUseNativeDispatchFailure( + Object.assign(new Error('timeout'), { code: 'ACCESSIBILITY_TIMEOUT', accepted: false }), + ), + ).toEqual({ result: 'rejected', reasonCode: 'native_accessibility_timeout' }); + expect( + classifyComputerUseNativeDispatchFailure( + Object.assign(new Error('unconfirmed'), { + code: 'INPUT_TIMEOUT_UNCONFIRMED', + accepted: true, + }), + ), + ).toBeNull(); + expect( + classifyComputerUseNativeDispatchFailure( + Object.assign(new Error('unconfirmed'), { code: 'INPUT_TIMEOUT_UNCONFIRMED' }), + ), + ).toBeNull(); }); it('maps the three v2 interlock dispatch codes to the result of their preflight row (#500 S5-c-2 PR-5)', () => { diff --git a/apps/desktop/src/main/computer-use-windows-shadow-reader.test.ts b/apps/desktop/src/main/computer-use-windows-shadow-reader.test.ts index ae0c36c17..6c53800f2 100644 --- a/apps/desktop/src/main/computer-use-windows-shadow-reader.test.ts +++ b/apps/desktop/src/main/computer-use-windows-shadow-reader.test.ts @@ -277,6 +277,8 @@ describe('Windows UI Automation source only reads', () => { 'ShellDriveIsLocalDisk', 'ShellReparseIsLink', 'ShellMountTargetPath', + 'CreateBoundedAutomation', + 'CreateWindowsV1Automation', 'ShellDirectoryWithSeparator', 'ShellIniValueIsSafe', 'ShellDriveLetterIsLocal', @@ -423,6 +425,8 @@ describe('Windows UI Automation source only reads', () => { 'ShellDriveIsLocalDisk', 'ShellReparseIsLink', 'ShellMountTargetPath', + 'CreateBoundedAutomation', + 'CreateWindowsV1Automation', 'ShellDirectoryWithSeparator', 'ShellIniValueIsSafe', 'ShellDriveLetterIsLocal', @@ -477,19 +481,28 @@ describe('Windows UI Automation source only reads', () => { 'static std::unique_ptr Create(', 'WindowsUiaShadowSource(const', ); - // CUIAutomation8 is the only coclass with IUIAutomation2 (ConnectionTimeout / TransactionTimeout); - // with the original CUIAutomation a call to a hung app never returns (measured). - expect(create).toContain('CLSID_CUIAutomation8'); - expect(code(create)).not.toMatch(/CLSID_CUIAutomation\b(?!8)/u); - expect(create).toContain('ComRef bounded;'); - expect(create).toContain('put_ConnectionTimeout(timeout_ms)'); - expect(create).toContain('put_TransactionTimeout(timeout_ms)'); + // The coclass with IUIAutomation2 (CUIAutomation8; with the original CUIAutomation a call to a hung + // app never returns, measured) is created by the one shared helper, which sets both timeouts and + // hands out nothing when it cannot (#820: the V1 lane uses the same helper). + const helper = between( + source, + 'inline bool CreateBoundedAutomation(', + 'What the shell knows about an item', + ); + expect(helper).toContain('CLSID_CUIAutomation8'); + expect(code(source)).not.toMatch(/CLSID_CUIAutomation\b(?!8)/u); + expect(helper).toContain('IUIAutomation2 *bounded = nullptr;'); + expect(helper).toContain('put_ConnectionTimeout(connection_timeout_ms)'); + expect(helper).toContain('put_TransactionTimeout(transaction_timeout_ms)'); expect(code(source)).not.toContain('IUIAutomation6'); - // Failing to set either is "no source" (the capture is skipped), never a call without a bound. - expect(create).toMatch( - /FAILED\(bounded\.Get\(\)->put_ConnectionTimeout\(timeout_ms\)\) \|\|\s*FAILED\(bounded\.Get\(\)->put_TransactionTimeout\(timeout_ms\)\)\)\s*return nullptr;/u, + expect(helper.replace(/\s+/gu, ' ')).toContain( + 'if (!set) { created->Release(); return false; }', + ); + // Failing to get a bounded one is "no source" (the capture is skipped), never a call without a bound. + expect(create.replace(/\s+/gu, ' ')).toContain( + 'if (!windows_uia_source_detail::CreateBoundedAutomation(timeout_ms, timeout_ms, automation.Receive()) || automation.Get() == nullptr) return nullptr;', ); - expect(create.indexOf('put_TransactionTimeout')).toBeLessThan( + expect(create.indexOf('CreateBoundedAutomation')).toBeLessThan( create.indexOf('get_RawViewWalker'), ); // The bound is the caller's (S5-c-2 PR-1): the shadow is asynchronous and waits up to 1500 ms, the diff --git a/apps/desktop/src/main/computer-use-windows-v1-uia-timeout-live-seam.test.ts b/apps/desktop/src/main/computer-use-windows-v1-uia-timeout-live-seam.test.ts new file mode 100644 index 000000000..353db0fb6 --- /dev/null +++ b/apps/desktop/src/main/computer-use-windows-v1-uia-timeout-live-seam.test.ts @@ -0,0 +1,1193 @@ +import { spawn, spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +// Contracts and a LIVE seam for the bounds on the UI Automation calls of the V1 lane (#820). Without +// them, a target that does not answer holds the serial lane for good: every later frame waits behind +// it. The V1 lane creates its UI Automation in ONE place (CreateWindowsV1Automation), which sets the +// ConnectionTimeout / TransactionTimeout of IUIAutomation2, and every path of calls has its own bound +// (WindowsV1UiaPath: the first timed-out call, or a moved cancellation epoch, ends the path; only the +// observe as a whole has a time budget, no short path has one). +// +// What the live seam measures: every case below stops at ONE call, cut off either by the +// ConnectionTimeout (a first contact, measured 505-514 ms in CI) or by the TransactionTimeout (a call on +// a connected element, 310-320 ms). The bound is therefore one ConnectionTimeout and a 200 ms margin. +// The 800 ms (Connection + Transaction in ONE call) that the ADR counts against the 1 s Cancel +// acknowledgment is a safe upper estimate that has not been observed; if a call ever did run both waits, +// it would exceed this bound and the seam would fail, which is the alarm wanted. No case here sets out to +// cause that chain, so the 800 ms itself is not verified (a known limit, as the ADR says). +// +// The live seam compiles the real host (computer_use_windows_host.cc, with its entry point renamed) into a +// test program. The program starts ITSELF as a second process that owns the windows (so every UI +// Automation call crosses a process boundary, as it does against a real app): a window that stops +// answering altogether, an ordinary one, and one whose native UI Automation provider answers a few +// property requests and then stops. The V1 paths are called against them (the point and focus +// classification, the observation, the control bindings, the semantic target search, the focused-element +// signature that the re-validation uses). Each must come back within one timeout plus a small margin and +// fail closed. Windows only: elsewhere, or where no MSVC can be found, it is skipped (but never in CI, +// where a runner that cannot compile must not turn this into a silent pass). The build output goes to a +// temporary directory that is removed after the run. + +const nativeDirectory = join(__dirname, '../../computer-use-native'); +const host = readFileSync(join(nativeDirectory, 'computer_use_windows_host.cc'), 'utf8'); +const uiaSource = readFileSync(join(nativeDirectory, 'computer_use_windows_uia_source.h'), 'utf8'); +const mainHost = readFileSync(join(__dirname, 'computer-use-native-host.ts'), 'utf8'); +const mainWindows = readFileSync(join(__dirname, 'computer-use-native-windows.ts'), 'utf8'); + +/** Runs a program without blocking the event loop (the test worker must keep answering its parent). */ +function run( + file: string, + args: string[], + options: { + cwd?: string; + windowsVerbatimArguments?: boolean; + env?: NodeJS.ProcessEnv; + timeout: number; + }, +): Promise<{ status: number | null; stdout: string; stderr: string }> { + return new Promise((resolve) => { + const child = spawn(file, args, { + ...(options.cwd === undefined ? {} : { cwd: options.cwd }), + ...(options.windowsVerbatimArguments === undefined + ? {} + : { windowsVerbatimArguments: options.windowsVerbatimArguments }), + ...(options.env === undefined ? {} : { env: options.env }), + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk: Buffer) => (stdout += chunk.toString('utf8'))); + child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString('utf8'))); + const timer = setTimeout(() => child.kill(), options.timeout); + child.on('close', (status) => { + clearTimeout(timer); + resolve({ status, stdout, stderr }); + }); + child.on('error', (error) => { + clearTimeout(timer); + resolve({ status: -1, stdout, stderr: stderr + String(error) }); + }); + }); +} + +/** The code of a file without its line comments. */ +const code = (text: string): string => text.replace(/\/\/.*$/gmu, ''); + +/** A number constant of the host, e.g. "constexpr DWORD kName = 500;" -> 500 (digit separators removed). */ +function hostNumber(name: string): number { + const match = new RegExp(`${name}(?:\\{|\\s*=\\s*)([0-9']+)`, 'u').exec(code(host)); + expect(match, name).not.toBeNull(); + return Number(match![1]!.replaceAll("'", '')); +} + +const sixPaths = [ + 'ClassifyWindowsPoint', + 'ReadUiaObservation', + 'CaptureWindowsControlBindings', + 'FindWindowsSemanticTarget', + 'ClassifyWindowsFocusedElement', + 'WindowsFocusedElementSignature', +] as const; + +describe('V1 lane UI Automation is created in one place, with its call timeouts (#820)', () => { + it('creates UI Automation only in CreateWindowsV1Automation (and CreateBoundedAutomation)', () => { + // The host: no CLSID_CUIAutomation8 at all, and exactly one coclass creation: the fallback inside + // the one helper. + expect(code(host)).not.toContain('CLSID_CUIAutomation8'); + expect([...code(host).matchAll(/CoCreateInstance\(\s*CLSID_CUIAutomation/gu)]).toHaveLength(1); + expect([...code(host).matchAll(/CLSID_CUIAutomation(?!8)/gu)]).toHaveLength(1); + const helper = host.slice( + host.indexOf('HRESULT CreateWindowsV1Automation('), + host.indexOf('enum class WindowsV1UiaStop'), + ); + expect(helper).toContain('CLSID_CUIAutomation, nullptr, CLSCTX_INPROC_SERVER'); + // Every other use goes through the helper. + const callers = [...code(host).matchAll(/CreateWindowsV1Automation\(&automation\)/gu)]; + expect(callers.length).toBeGreaterThanOrEqual(8); + // The source header: one place creates the bounded coclass (CUIAutomation8), shared by the shadow + // source and the V1 lane. + expect([...code(uiaSource).matchAll(/CLSID_CUIAutomation8/gu)]).toHaveLength(1); + expect(code(uiaSource)).not.toMatch(/CLSID_CUIAutomation(?!8)/u); + }); + + it('bounds a call with both timeouts, and falls back to the old behaviour visibly', () => { + const bounded = uiaSource.slice( + uiaSource.indexOf('inline bool CreateBoundedAutomation('), + uiaSource.indexOf('// What the shell knows about an item'), + ); + const compact = bounded.replace(/\s+/gu, ' '); + expect(compact).toContain( + 'CoCreateInstance(CLSID_CUIAutomation8, nullptr, CLSCTX_INPROC_SERVER', + ); + expect(compact).toContain('bounded->put_ConnectionTimeout(connection_timeout_ms)'); + expect(compact).toContain('bounded->put_TransactionTimeout(transaction_timeout_ms)'); + // Without the interface or either timeout, no automation is handed out. + expect(compact).toContain('if (!set) { created->Release(); return false; }'); + const helper = host.slice( + host.indexOf('// The bounds on the UI Automation calls of the V1 lane'), + host.indexOf('enum class WindowsV1UiaStop'), + ); + const helperCompact = helper.replace(/\s+/gu, ' '); + expect(helperCompact).toContain( + 'CreateBoundedAutomation( kWindowsV1UiaConnectionTimeoutMs, kWindowsV1UiaTransactionTimeoutMs, automation)', + ); + // The fallback is the plain coclass (as before), counted and logged once, never silent. + expect(helperCompact).toContain('windows_v1_uia_unbounded_creations.fetch_add(1) == 0'); + expect(helperCompact).toContain('OutputDebugStringA('); + // The shadow source uses the same creation, with its own (longer) bound. + expect(uiaSource.replace(/\s+/gu, ' ')).toContain( + 'windows_uia_source_detail::CreateBoundedAutomation(timeout_ms, timeout_ms, automation.Receive())', + ); + }); + + it('keeps the worst cases inside what the lane is given, from the constants themselves', () => { + const connection = hostNumber('kWindowsV1UiaConnectionTimeoutMs'); + const transaction = hostNumber('kWindowsV1UiaTransactionTimeoutMs'); + const observeBudget = hostNumber('kWindowsV1UiaObserveBudget'); + const cancelAck = Number( + /COMPUTER_USE_NATIVE_CANCEL_ACK_TIMEOUT_MS = ([0-9_]+)/u + .exec(mainHost)![1]! + .replaceAll('_', ''), + ); + const observeTimeout = Number( + /operation === 'list_windows' \|\| operation === 'observe'\) return ([0-9_]+)/u + .exec(mainWindows)![1]! + .replaceAll('_', ''), + ); + expect(cancelAck).toBe(1000); + expect(observeTimeout).toBe(30_000); + // One call: the first contact waits for the connection and then for the transaction, so its worst + // case is counted as the SUM of the two (the safe side), and a Cancel waits for at most that one + // call (the epoch is asked before every call): it is inside the acknowledgment. + expect(connection + transaction).toBeLessThan(cancelAck); + // The first contact is allowed to be slower than a call on a connected element. + expect(connection).toBeGreaterThanOrEqual(transaction); + // There is NO time budget on a path: a deep tree of a live app must not become "unclassified". + expect(code(host)).not.toContain('kWindowsV1UiaShortPathBudget'); + expect(code(host)).not.toMatch(/ShortPath/u); + // The observe has a budget of its own, inside what Main gives an observe (which also captures). + expect(observeBudget).toBeLessThan(observeTimeout); + // It is wide (an ordinary app's tree of several thousand nodes is read four times) and there only to + // answer with the native's own reason before Main's 30 s: not the cause of a Cancel's delay. + expect(observeBudget).toBe(25_000); + // A call that "succeeded" after nearly its whole timeout has timed out all the same. + const slowRead = hostNumber('kWindowsV1UiaSlowRead'); + const slowContact = hostNumber('kWindowsV1UiaSlowContact'); + expect(slowRead).toBeLessThan(transaction); + expect(slowContact).toBeLessThan(connection); + expect(slowRead).toBeGreaterThan(transaction - 100); + expect(slowContact).toBeGreaterThan(connection - 100); + }); + + it('ends a path at its first timed-out call, and stops climbing at an element that cannot be read', () => { + const c = code(host).replace(/\s+/gu, ' '); + // The state: a timeout, a spent budget and a cancel end the path for good. + expect(c).toContain('enum class WindowsV1UiaStop { kNone, kTimeout, kBudget, kCanceled };'); + expect(c).toContain('lane->stop = WindowsV1UiaStop::kTimeout;'); + expect(c).toContain('lane->stop = WindowsV1UiaStop::kBudget;'); + expect(c).toContain('lane->stop = WindowsV1UiaStop::kCanceled;'); + expect(c).toContain( + 'return result == kWindowsV1UiaTimeoutResult || result == kWin32Timeout || result == kRpcTimeout;', + ); + // Every walk checks it at each node, and every chain of reads goes through the checked result. + for (const walk of [ + 'bool AppendUiaTree(', + 'bool CollectWindowsControlBindings(', + 'bool FindUiaTargetRecursive(', + ]) { + const at = host.indexOf(walk); + expect(at, walk).toBeGreaterThan(0); + expect(host.slice(at, at + 700), walk).toContain('!WindowsV1UiaMayCall()'); + } + expect(c).toContain( + 'WindowsUiaRisk ClassifyWindowsUiaElementSelf(IUIAutomationElement *element) { WindowsUiaRisk risk; if (element == nullptr || !WindowsV1UiaMayCall()) return risk;', + ); + expect(c).toContain( + 'if (element == nullptr || signature == nullptr || !WindowsV1UiaMayCall()) return false;', + ); + // The climb stops at an element that could not be read (before the parent is asked for), and the + // parent is not asked for after the path has ended. + const climb = c.slice( + c.indexOf('WindowsUiaRisk ClassifyWindowsUiaElement(IUIAutomationElement *element)'), + ); + expect(climb.indexOf('if (!risk.metadata_complete) break;')).toBeGreaterThan(0); + expect(climb.indexOf('if (!risk.metadata_complete) break;')).toBeLessThan( + climb.indexOf('walker->GetParentElement(current, &parent)'), + ); + expect(climb).toContain( + 'WindowsV1UiaCall([&] { return walker->GetParentElement(current, &parent); })', + ); + // The point classification reads the signature only for facts that are complete. + expect(c).toContain( + 'if (control_signature != nullptr && risk.metadata_complete) ComputeWindowsControlSignature(element, expected_pid, nullptr, control_signature);', + ); + // The observe reads the focused element's signature whatever the classification says (a live but + // unclassified focus keeps it, so its action is refused as target_unclassified); after a timeout the + // shared path refuses the read without a call. + expect(c).toContain( + 'const bool has_focused_element_signature = WindowsFocusedElementSignature(session.pid, bounds, &focused_element_signature);', + ); + expect(c).not.toContain( + 'focused_risk.classified && focused_risk.metadata_complete && WindowsFocusedElementSignature(', + ); + // The scopes: every path has its own (which watches the cancellation epoch as it was when the path + // began), and only the observe has a budget, with the session's epoch. + expect([...c.matchAll(/const WindowsV1UiaPath path;/gu)]).toHaveLength(7); + expect(c).toContain( + 'const WindowsV1UiaPath uia_path(kWindowsV1UiaObserveBudget, session.cancel_epoch);', + ); + expect(c).toContain( + 'WindowsV1UiaPath() : WindowsV1UiaPath(std::chrono::milliseconds::zero(), cancellation_epoch.load(std::memory_order_acquire)) {}', + ); + // A Cancel ends the path before a spent budget is looked at, and the check is the first thing + // every call goes through. + expect(c).toContain( + 'if (cancellation_epoch.load(std::memory_order_acquire) != lane->epoch) lane->stop = WindowsV1UiaStop::kCanceled; else if (lane->has_deadline', + ); + // Outside any path a call is refused (fail closed), not made unwatched; only the owner (the + // outermost path) clears the lane, so an inner path that ends leaves the outer one's state. + expect(c).toContain( + 'WindowsV1UiaLane *lane = windows_v1_uia_lane; if (lane == nullptr) return false;', + ); + expect(c).toContain('if (windows_v1_uia_lane != nullptr) return;'); + expect(c).toContain('~WindowsV1UiaPath() { if (owner_) windows_v1_uia_lane = nullptr; }'); + // An inner path stays a non-owner: owner_ starts false and is set in one place only (the end of + // the constructor, after the early return of an inner path). + expect(c.split('bool owner_ = false;')).toHaveLength(2); + expect(c.split('owner_ = true;')).toHaveLength(2); + expect(c.indexOf('owner_ = true;')).toBeGreaterThan( + c.indexOf('if (windows_v1_uia_lane != nullptr) return;'), + ); + // Every input of a semantic action is sent only while the path is alive: when it is not, nothing + // is sent, the attempt is not counted and the action is not accepted (so it cannot become + // input_timeout_unconfirmed, which needs an accepted action). + const semanticStart = c.indexOf('bool DispatchWindowsSemanticAction('); + expect(semanticStart).toBeGreaterThanOrEqual(0); + const semanticRest = c.slice(semanticStart); + const semantic = semanticRest.slice(0, semanticRest.indexOf(' return true; }')); + const gate = + 'if (!WindowsV1UiaMayCall()) { *reason = WindowsV1UiaEndedReason(); } else { *accepted = true; session.input_api_attempts->fetch_add(1, std::memory_order_acq_rel);'; + // Counted against each other: an input added outside the gate makes the two counts differ. + const gated = semantic.split(gate).length - 1; + expect(gated).toBeGreaterThanOrEqual(5); + expect(semantic.split('input_api_attempts->fetch_add(').length - 1).toBe(gated); + expect(semantic.split('*accepted = true;').length - 1).toBe(gated); + expect(semantic).not.toContain('*accepted = false;'); + // ...and each gate's else block holds the pattern call itself, which is assigned to action_result + // (its own HRESULT decides input_timeout_unconfirmed). No pattern call stands anywhere else. + const escapedGate = gate.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&'); + const effect = + 'action_result = [^;]*?pattern->(?:Invoke|SetValue|Select|Toggle|Expand|Collapse)\\('; + expect([...semantic.matchAll(new RegExp(`${escapedGate}[^}]*?${effect}`, 'gu'))]).toHaveLength( + gated, + ); + expect([...semantic.matchAll(new RegExp(effect, 'gu'))]).toHaveLength(gated); + // Outside those assignments only reads (get_*/Get*) and housekeeping are called through a pointer, + // whatever the variable or the method is called: any other call there would be an unguarded input. + const otherCalls = [ + ...semantic.replace(/action_result = [^;]*;/gu, '').matchAll(/\b\w+->(?!get_|Get)(\w+)\(/gu), + ].map((match) => match[1]); + expect( + otherCalls.filter((name) => !['Release', 'AddRef', 'fetch_add', 'empty'].includes(name!)), + ).toEqual([]); + expect(semantic).toContain( + '*reason = !*accepted ? "semantic_action_failed" : WindowsV1UiaIsTimeout(action_result) ? "input_timeout_unconfirmed"', + ); + // The interlock: a Cancel is looked at before anything is created, a worker still stuck means an + // immediate kBudget (no wait for it), and only then the one bounded wait. + const interlock = c.slice(c.indexOf('InterlockEvaluation EvaluateWindowsInterlockOnLane(')); + const order = [ + 'if (!epoch_unchanged())', + 'std::make_shared()', + 'if (running.exchange(true)) { unavailable.no_verdict = InterlockNoVerdict::kBudget; return unavailable; }', + 'pending->ready.wait_for(lock, kInterlockLaneHoldLimit,', + ].map((text) => interlock.indexOf(text)); + expect(order.every((at) => at >= 0)).toBe(true); + expect([...order].sort((a, b) => a - b)).toEqual(order); + expect(c).toContain( + 'template HRESULT WindowsV1UiaCall(Call &&call) { return WindowsV1UiaTimed(kWindowsV1UiaSlowRead, call); }', + ); + // EVERY call that reaches the target goes through WindowsV1UiaCall: the first contact, the walks + // (first child, next sibling) and the climb to the parent. The reads of properties are asked at the + // entry of each function that makes them (checked above). + const calls = [ + ...c.matchAll( + /->(ElementFromPoint|ElementFromHandle|GetFocusedElement|GetFirstChildElement|GetNextSiblingElement|GetParentElement)\(/gu, + ), + ]; + const guarded = [ + ...c.matchAll( + /WindowsV1Uia(?:Call|Contact)\(\[&\] \{ return \w+->(ElementFromPoint|ElementFromHandle|GetFocusedElement|GetFirstChildElement|GetNextSiblingElement|GetParentElement)\(/gu, + ), + ]; + expect(calls.length).toBeGreaterThanOrEqual(12); + expect(guarded).toHaveLength(calls.length); + // The first contacts use the contact wrapper (the longer slow threshold), the moves the call one. + for (const name of ['ElementFromPoint', 'ElementFromHandle', 'GetFocusedElement']) + expect(c, name).not.toMatch( + new RegExp(`WindowsV1UiaCall\\(\\[&\\] \\{ return \\w+->${name}\\(`, 'u'), + ); + // EVERY call of a UI Automation interface in the host, whatever the variable is called (element, + // target, pattern, walker, automation, ...), goes through a wrapper of the V1 lane: the reads of + // properties and patterns through WindowsV1UiaRead, the moves and the first contacts through + // WindowsV1UiaCall / WindowsV1UiaContact. None is made bare, so none can take a cut-off "success" for an + // answer, or go on after a Cancel. + const uiaNames = + 'get_Current\\w+|get_Cached\\w+|GetCurrentPattern\\w*|GetCachedPattern\\w*|GetCurrentPropertyValue\\w*|GetCachedPropertyValue\\w*|BuildUpdatedCache|GetRuntimeId|ElementFrom\\w+|FindFirst\\w*|FindAll\\w*|GetFocusedElement|GetFirstChildElement|GetNextSiblingElement|GetPreviousSiblingElement|GetLastChildElement|GetParentElement'; + const bareUiaCalls = (text: string): number => + [...text.matchAll(new RegExp(`->(?:${uiaNames})\\(`, 'gu'))].length - + [ + ...text.matchAll( + new RegExp( + `WindowsV1Uia(?:Read|Call|Contact)\\(\\[&\\] \\{ return \\w+->(?:${uiaNames})\\(`, + 'gu', + ), + ), + ].length; + // The check itself catches a bare call (a wrapped one and a bare one of the same method). + expect( + bareUiaCalls( + 'WindowsV1UiaRead([&] { return element->get_CurrentName(&n); }); element->get_CurrentName(&n);', + ), + ).toBe(1); + expect([...c.matchAll(new RegExp(`->(?:${uiaNames})\\(`, 'gu'))].length).toBeGreaterThanOrEqual( + 35, + ); + expect(bareUiaCalls(c)).toBe(0); + // The calls that DO something (Invoke, SetValue, Select, Toggle, Expand, Collapse) are made only + // while the path is alive: after a timeout or a Cancel nothing is sent (a rejection before any input). + expect([...c.matchAll(/->(Invoke|SetValue|Select|Toggle|Expand|Collapse)\(/gu)]).toHaveLength( + 6, + ); + const semanticBody = c.slice( + c.indexOf('bool DispatchWindowsSemanticAction('), + c.indexOf('std::string ResponseCacheKey('), + ); + expect([...semanticBody.matchAll(/\*accepted = true;/gu)]).toHaveLength(5); + expect([ + ...c.matchAll( + /if \(!WindowsV1UiaMayCall\(\)\) \{ \*reason = WindowsV1UiaEndedReason\(\); \} else \{ \*accepted = true; session\.input_api_attempts->fetch_add\(1, std::memory_order_acq_rel\); action_result = [^;]*->(Invoke|SetValue|Select|Toggle|Expand|Collapse)\(/gu, + ), + ]).toHaveLength(5); + // What a read that could not be made leaves behind is NOT "no risk": the risk is all false, which + // ClassifyNativePreflightFacts (checked in the harness) puts first of all as unclassified, and + // RejectWindowsRisk rejects as target_unclassified. + expect(c).toContain( + 'struct WindowsUiaRisk { bool classified = false; bool secure = false; bool high_impact = false; bool metadata_complete = false; };', + ); + expect(c).toContain('if (element == nullptr || !WindowsV1UiaMayCall()) return risk;'); + // The semantic action reads its patterns through ONE timed helper that releases what a read that did + // not complete produced, and the name of a select is not compared when it was not read. + expect(c).toContain( + 'bool ReadWindowsV1Pattern(IUIAutomationElement *target, PATTERNID id, Pattern **pattern)', + ); + expect([ + ...c.matchAll(/ReadWindowsV1Pattern\(target, UIA_\w+PatternId, &pattern\)/gu), + ]).toHaveLength(5); + expect(c).not.toMatch(/target->GetCurrentPatternAs\(UIA_/u); + expect(c).toContain( + 'const bool name_read = WindowsV1UiaRead([&] { return target->get_CurrentName(&name); });', + ); + expect(c).toContain('if (!name_read) {'); + // A child that a cut-off first-child call produced is released (no output pointer is leaked). + expect(c).toMatch( + /if \(!WindowsV1UiaOk\(first_result\)\) \{\s*if \(child != nullptr\)\s*child->Release\(\);\s*return false;/u, + ); + expect(readFileSync(join(nativeDirectory, 'computer_use_protocol_fuzz.cc'), 'utf8')).toContain( + 'v1-default-risk-is-unclassified', + ); + // The time of a call is measured, and a slow "success" is a timeout. + expect(c).toContain( + 'if (SUCCEEDED(result) && std::chrono::steady_clock::now() - start >= slow) result = kWindowsV1UiaTimeoutResult;', + ); + // The frames of a preflight and of a dispatch each run under one outer path, so the later steps of + // a frame share the state of the first timed-out call and of a Cancel. + expect(c).toContain('const WindowsV1UiaPath preflight_uia_path; if (!HandleWindowsPreflight('); + expect(c).toContain( + 'const WindowsV1UiaPath dispatch_uia_path; const bool succeeded = DispatchWindowsAction(', + ); + // The observe counts nodes, logs only a count when its budget is spent, and gives the reasons of + // WindowsV1ObserveReason. + expect(c).toContain('WindowsV1UiaCountNode();'); + expect(c).toContain('budget spent after'); + expect(c).toContain('return WindowsV1ObserveReason(uia_path.Stop(), otherwise);'); + }); + + it('does not hand the job, or anything else of the test, to the window process', () => { + // The job is made without security attributes (so it is not inheritable) and marked not + // inheritable all the same; the only handle the child inherits is the write end of its pipe, by an + // explicit list, and the read end is not inheritable. + expect(program).toContain('CreateJobObjectW(nullptr, nullptr)'); + expect(program).toContain('SetHandleInformation(job, HANDLE_FLAG_INHERIT, 0);'); + expect(program).toContain('SetHandleInformation(read_end, HANDLE_FLAG_INHERIT, 0);'); + expect(program).toContain('HANDLE inherited[1] = {write_end};'); + expect(program).toContain('PROC_THREAD_ATTRIBUTE_HANDLE_LIST, inherited'); + expect(program).not.toMatch(/inherited\[[^\]]*\] = \{[^}]*job/u); + expect(program).not.toMatch(/&inherit[^;]*Job|CreateJobObjectW\(&/u); + expect(program).toContain('EXTENDED_STARTUPINFO_PRESENT | CREATE_SUSPENDED'); + expect(program).toContain('JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE'); + }); + + it('names why a read did not complete, and keeps a timed-out action apart from an unconfirmed one', () => { + const c = code(host).replace(/\s+/gu, ' '); + // The observe: a target that did not answer is transient (accessibility_timeout), a spent budget is + // observation_timeout, a cancel is observation_stale. + expect(c).toContain('return "accessibility_timeout";'); + expect(c).toContain('return "observation_timeout";'); + expect(c).toContain('*reason = unreadable("accessibility_unavailable");'); + expect(c).toContain('*reason = unreadable("accessibility_binding_unavailable");'); + // The semantic target search tells a timeout from "no such target". + expect(c).toContain( + 'path.Stop() == WindowsV1UiaStop::kTimeout ? "accessibility_timeout" : "target_required"', + ); + // A pattern call that timed out may have been applied: unconfirmed, told apart, and the common tail + // of a dispatch keeps that reason instead of writing input_not_confirmed over it. + expect(c).toContain( + 'WindowsV1UiaIsTimeout(action_result) ? "input_timeout_unconfirmed" : "input_not_confirmed"', + ); + expect(c).toContain( + 'if (*accepted) { if (*reason != "input_timeout_unconfirmed") *reason = "input_not_confirmed"; } else if (cancellation_epoch', + ); + // A failed read is the "unreadable" of each caller (fail closed). + expect(c).toContain( + 'const auto decision = ClassifyNativePreflightFacts( {risk.metadata_complete, risk.classified, risk.secure, risk.high_impact});', + ); + for (const name of ['ClassifyWindowsPoint(', 'ClassifyWindowsFocusedElement(']) { + const at = host.indexOf(`WindowsUiaRisk ${name}`); + expect(at, name).toBeGreaterThan(0); + expect(host.slice(at, at + 1700), name).toContain('return {};'); + } + expect(host).toContain('*reason = "accessibility_unavailable";'); + }); +}); + +const vswhere = 'C:\\Program Files (x86)\\Microsoft Visual Studio\\Installer\\vswhere.exe'; +const libraries = [ + 'ole32.lib', + 'oleaut32.lib', + 'uiautomationcore.lib', + 'user32.lib', + 'dwmapi.lib', + 'd3d11.lib', + 'dxgi.lib', + 'windowscodecs.lib', + 'bcrypt.lib', + 'advapi32.lib', + 'userenv.lib', + 'crypt32.lib', + 'wintrust.lib', + 'shell32.lib', + 'runtimeobject.lib', + 'windowsapp.lib', + 'uuid.lib', +]; + +function compileArguments(path: string, output: string, provenance: string): string[] { + return [ + '/nologo', + '/std:c++20', + '/EHsc', + '/permissive-', + '/utf-8', + '/DUNICODE', + '/D_UNICODE', + '/DWIN32_LEAN_AND_MEAN', + '/DNOMINMAX', + '/D_WIN32_WINNT=0x0A00', + `/FI${provenance}`, + `/I${nativeDirectory}`, + path, + `/Fe:${output}`, + `/Fo:${output}.obj`, + '/link', + '/DEBUG:NONE', + ...libraries, + ]; +} + +type Build = Readonly<{ name: string; file: string; args: string[]; verbatim: boolean }>; + +function findBuild(path: string, output: string, provenance: string): Build | null { + if (process.platform !== 'win32') return null; + if (spawnSync('cl.exe', ['/?'], { stdio: 'ignore' }).status === 0) + return { + name: 'cl.exe', + file: 'cl.exe', + args: compileArguments(path, output, provenance), + verbatim: false, + }; + if (!existsSync(vswhere)) return null; + const found = spawnSync( + vswhere, + [ + '-latest', + '-products', + '*', + '-requires', + 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', + '-property', + 'installationPath', + ], + { encoding: 'utf8' }, + ); + const root = found.stdout.trim(); + const vcvars = join(root, 'VC', 'Auxiliary', 'Build', 'vcvars64.bat'); + if (found.status !== 0 || root === '' || !existsSync(vcvars)) return null; + return { + name: 'cl.exe (vcvars64)', + file: 'cmd.exe', + args: [ + '/d', + '/s', + '/c', + `"call "${vcvars}" >nul && cl.exe ${compileArguments(path, output, provenance) + .map((argument) => `"${argument}"`) + .join(' ')}"`, + ], + verbatim: true, + }; +} + +const program = String.raw` +// The real host, its entry point renamed so this program has its own. +#define wmain computer_use_host_wmain +#include "computer_use_windows_host.cc" +#undef wmain + +#include +#include +#include + +namespace sc820 { + +int failures = 0; +void SeamCheck(bool ok, const std::string &name) { + if (!ok) { + ++failures; + std::cout << "FAIL " << name << std::endl; + } +} + +// ---------------------------------------------------------------- the process that owns the windows + +std::atomic g_provider_stall_after{-1}; // property requests answered before the provider stops +std::atomic g_provider_calls{0}; + +// A minimal native provider: answers a few property requests, then stops answering (sleeps). +class Provider final : public IRawElementProviderSimple { +public: + explicit Provider(HWND window) : window_(window) {} + ULONG STDMETHODCALLTYPE AddRef() override { return ++references_; } + ULONG STDMETHODCALLTYPE Release() override { + const ULONG left = --references_; + if (left == 0) delete this; + return left; + } + HRESULT STDMETHODCALLTYPE QueryInterface(REFIID id, void **out) override { + if (id == __uuidof(IUnknown) || id == __uuidof(IRawElementProviderSimple)) { + *out = static_cast(this); + AddRef(); + return S_OK; + } + *out = nullptr; + return E_NOINTERFACE; + } + HRESULT STDMETHODCALLTYPE get_ProviderOptions(ProviderOptions *options) override { + *options = ProviderOptions_ServerSideProvider; + return S_OK; + } + HRESULT STDMETHODCALLTYPE GetPatternProvider(PATTERNID, IUnknown **pattern) override { + *pattern = nullptr; + return S_OK; + } + HRESULT STDMETHODCALLTYPE GetPropertyValue(PROPERTYID property, VARIANT *value) override { + VariantInit(value); + const int stall_after = g_provider_stall_after.load(); + if (stall_after >= 0 && g_provider_calls.fetch_add(1) >= stall_after) Sleep(3000); + if (property == UIA_NamePropertyId) { + value->vt = VT_BSTR; + value->bstrVal = SysAllocString(L"sc820-provider"); + } else if (property == UIA_AutomationIdPropertyId) { + value->vt = VT_BSTR; + value->bstrVal = SysAllocString(L"provider"); + } else if (property == UIA_ControlTypePropertyId) { + value->vt = VT_I4; + value->lVal = UIA_PaneControlTypeId; + } else if (property == UIA_IsPasswordPropertyId) { + value->vt = VT_BOOL; + value->boolVal = VARIANT_FALSE; + } + return S_OK; + } + HRESULT STDMETHODCALLTYPE get_HostRawElementProvider(IRawElementProviderSimple **host_provider) override { + return UiaHostProviderFromHwnd(window_, host_provider); + } + +private: + HWND window_; + std::atomic references_{1}; +}; + +LRESULT CALLBACK HangProc(HWND window, UINT message, WPARAM w, LPARAM l) { + if (message == WM_APP + 1) { + Sleep(3000); // a window that does not answer: its thread is busy and pumps nothing + return 0; + } + if (message == WM_APP + 3) { // put the keyboard focus on its edit control + HWND edit = FindWindowExW(window, nullptr, L"EDIT", nullptr); + if (edit != nullptr) SetFocus(edit); + return 0; + } + if (message == WM_DESTROY) { + PostQuitMessage(0); + return 0; + } + return DefWindowProcW(window, message, w, l); +} + +LRESULT CALLBACK ProviderProc(HWND window, UINT message, WPARAM w, LPARAM l) { + if (message == WM_APP + 2) { // arm the provider: it answers w requests, then stops + g_provider_calls.store(0); + g_provider_stall_after.store(static_cast(w)); + return 0; + } + if (message == WM_GETOBJECT && static_cast(l) == static_cast(UiaRootObjectId)) { + Provider *provider = new Provider(window); + const LRESULT answer = UiaReturnRawElementProvider(window, w, l, provider); + provider->Release(); + return answer; + } + if (message == WM_DESTROY) { + PostQuitMessage(0); + return 0; + } + return DefWindowProcW(window, message, w, l); +} + +// A window of a deep tree that answers, slowly: every request of UI Automation to it takes 15 ms. +LRESULT CALLBACK DeepProc(HWND window, UINT message, WPARAM w, LPARAM l) { + if (message == WM_GETOBJECT || message == WM_GETTEXT || message == WM_GETTEXTLENGTH) Sleep(15); + if (message == WM_DESTROY && GetParent(window) == nullptr) { + PostQuitMessage(0); + return 0; + } + return DefWindowProcW(window, message, w, l); +} + +struct Win { + HWND handle = nullptr; + std::thread thread; + HWND deepest = nullptr; +}; + +// A top-level window with a chain of nested child windows (five in all with the top one: with the +// desktop that is six elements up to the root, inside what the V1 climb reads). +void MakeDeepWindow(Win &win, int x, int y) { + std::atomic ready{false}; + win.thread = std::thread([&win, &ready, x, y]() { + WNDCLASSW cls{}; + cls.lpfnWndProc = DeepProc; + cls.hInstance = GetModuleHandleW(nullptr); + cls.lpszClassName = L"Sc820Deep"; + RegisterClassW(&cls); + win.handle = CreateWindowExW(WS_EX_TOPMOST, L"Sc820Deep", L"sc820-deep", WS_OVERLAPPEDWINDOW | WS_VISIBLE, x, y, 300, 200, + nullptr, nullptr, cls.hInstance, nullptr); + HWND parent = win.handle; + for (int level = 0; level < 4; ++level) { + parent = CreateWindowExW(0, L"Sc820Deep", L"", WS_CHILD | WS_VISIBLE, 10, 10, 200 - level * 20, 100 - level * 10, parent, + nullptr, cls.hInstance, nullptr); + } + win.deepest = parent; + ShowWindow(win.handle, SW_SHOW); + ready.store(true); + MSG message; + while (GetMessageW(&message, nullptr, 0, 0) > 0) { + TranslateMessage(&message); + DispatchMessageW(&message); + } + }); + for (int i = 0; i < 500 && !ready.load(); ++i) Sleep(10); +} + +void MakeWindow(Win &win, WNDPROC proc, const wchar_t *class_name, const wchar_t *title, int x, int y, bool children) { + std::atomic ready{false}; + win.thread = std::thread([&win, &ready, proc, class_name, title, x, y, children]() { + WNDCLASSW cls{}; + cls.lpfnWndProc = proc; + cls.hInstance = GetModuleHandleW(nullptr); + cls.lpszClassName = class_name; + RegisterClassW(&cls); + win.handle = CreateWindowExW(WS_EX_TOPMOST, class_name, title, WS_OVERLAPPEDWINDOW | WS_VISIBLE, x, y, 300, 200, + nullptr, nullptr, cls.hInstance, nullptr); + if (children) { + CreateWindowExW(0, L"BUTTON", L"ok", WS_CHILD | WS_VISIBLE, 20, 20, 100, 30, win.handle, nullptr, cls.hInstance, nullptr); + CreateWindowExW(0, L"EDIT", L"text", WS_CHILD | WS_VISIBLE | WS_BORDER, 20, 70, 200, 30, win.handle, nullptr, cls.hInstance, nullptr); + } + ShowWindow(win.handle, SW_SHOW); + ready.store(true); + MSG message; + while (GetMessageW(&message, nullptr, 0, 0) > 0) { + TranslateMessage(&message); + DispatchMessageW(&message); + } + }); + for (int i = 0; i < 500 && !ready.load(); ++i) Sleep(10); +} + +int RunWindowProcess() { + CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED); + Win hang, ordinary, provider, deep; + MakeDeepWindow(deep, 340, 240); + MakeWindow(hang, HangProc, L"Sc820Hang", L"sc820-hang", 20, 20, true); + MakeWindow(ordinary, DefWindowProcW, L"Sc820Ordinary", L"sc820-ordinary", 20, 240, true); + MakeWindow(provider, ProviderProc, L"Sc820Provider", L"sc820-provider", 340, 20, false); + std::cout << "READY " << GetCurrentProcessId() << " " << reinterpret_cast(hang.handle) << " " + << reinterpret_cast(ordinary.handle) << " " << reinterpret_cast(provider.handle) << " " + << reinterpret_cast(deep.deepest) << std::endl; + for (;;) Sleep(1000); // the test ends this process +} + +// ---------------------------------------------------------------- the driver + +long long Millis(std::chrono::steady_clock::time_point since) { + return std::chrono::duration_cast(std::chrono::steady_clock::now() - since).count(); +} + +// Runs the call while the window is stuck, and waits until it wakes before the next case. +template long long WhileStuck(HWND window, UINT message, WPARAM w, Call &&call) { + PostMessageW(window, message, w, 0); + Sleep(150); + const auto start = std::chrono::steady_clock::now(); + call(); + const long long elapsed = Millis(start); + Sleep(3300); + return elapsed; +} + +bool BringToForeground(HWND window) { + if (GetForegroundWindow() == window) return true; + const DWORD me = GetCurrentThreadId(); + HWND current = GetForegroundWindow(); + const DWORD other = current != nullptr ? GetWindowThreadProcessId(current, nullptr) : 0; + const DWORD target = GetWindowThreadProcessId(window, nullptr); + if (other != 0 && other != me) AttachThreadInput(me, other, TRUE); + if (target != 0 && target != me) AttachThreadInput(me, target, TRUE); + ShowWindow(window, SW_SHOW); + BringWindowToTop(window); + SetForegroundWindow(window); + if (other != 0 && other != me) AttachThreadInput(me, other, FALSE); + if (target != 0 && target != me) AttachThreadInput(me, target, FALSE); + Sleep(300); + if (GetForegroundWindow() == window) return true; + // The last resort: the Alt key lets a process move the foreground. + keybd_event(VK_MENU, 0, 0, 0); + keybd_event(VK_MENU, 0, KEYEVENTF_KEYUP, 0); + SetForegroundWindow(window); + Sleep(300); + return GetForegroundWindow() == window; +} + +int RunDriver(bool ci) { + CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED); + wchar_t self[MAX_PATH]{}; + GetModuleFileNameW(nullptr, self, MAX_PATH); + SECURITY_ATTRIBUTES inherit{sizeof(inherit), nullptr, TRUE}; + HANDLE read_end = nullptr, write_end = nullptr; + CreatePipe(&read_end, &write_end, &inherit, 0); + SetHandleInformation(read_end, HANDLE_FLAG_INHERIT, 0); + // The window process lives in a job that is closed with this process: if this process is killed (the + // test's timeout), the job closes and the windows go with it. And it inherits the pipe's write end and + // nothing else, so it cannot hold on to the handles of the process that started this one. + HANDLE job = CreateJobObjectW(nullptr, nullptr); + SetHandleInformation(job, HANDLE_FLAG_INHERIT, 0); + JOBOBJECT_EXTENDED_LIMIT_INFORMATION limits{}; + limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + SetInformationJobObject(job, JobObjectExtendedLimitInformation, &limits, sizeof(limits)); + SIZE_T attribute_bytes = 0; + InitializeProcThreadAttributeList(nullptr, 1, 0, &attribute_bytes); + std::vector attribute_storage(attribute_bytes); + LPPROC_THREAD_ATTRIBUTE_LIST attributes = reinterpret_cast(attribute_storage.data()); + InitializeProcThreadAttributeList(attributes, 1, 0, &attribute_bytes); + HANDLE inherited[1] = {write_end}; + UpdateProcThreadAttribute(attributes, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, inherited, sizeof(inherited), nullptr, nullptr); + STARTUPINFOEXW startup{}; + startup.StartupInfo.cb = sizeof(startup); + startup.StartupInfo.dwFlags = STARTF_USESTDHANDLES; + startup.StartupInfo.hStdOutput = write_end; + startup.StartupInfo.hStdError = write_end; + startup.StartupInfo.hStdInput = nullptr; + startup.lpAttributeList = attributes; + PROCESS_INFORMATION child{}; + std::wstring command = std::wstring(L"\"") + self + L"\" window-process"; + if (!CreateProcessW(nullptr, command.data(), nullptr, nullptr, TRUE, EXTENDED_STARTUPINFO_PRESENT | CREATE_SUSPENDED, nullptr, + nullptr, &startup.StartupInfo, &child)) { + std::cout << "FAIL could_not_start_the_window_process" << std::endl; + return 2; + } + SeamCheck(AssignProcessToJobObject(job, child.hProcess) != 0, "the_window_process_is_in_the_job"); + ResumeThread(child.hThread); + DeleteProcThreadAttributeList(attributes); + CloseHandle(write_end); + std::string line; + for (char c; line.find('\n') == std::string::npos;) { + DWORD got = 0; + if (!ReadFile(read_end, &c, 1, &got, nullptr) || got == 0) break; + line.push_back(c); + } + unsigned long pid = 0; + unsigned long long hang_value = 0, ordinary_value = 0, provider_value = 0, deepest_value = 0; + if (sscanf_s(line.c_str(), "READY %lu %llu %llu %llu %llu", &pid, &hang_value, &ordinary_value, &provider_value, &deepest_value) != 5) { + std::cout << "FAIL the_window_process_did_not_report " << line << std::endl; + TerminateProcess(child.hProcess, 1); + return 2; + } + const HWND hang = reinterpret_cast(static_cast(hang_value)); + const HWND ordinary = reinterpret_cast(static_cast(ordinary_value)); + const HWND provider = reinterpret_cast(static_cast(provider_value)); + const HWND deepest = reinterpret_cast(static_cast(deepest_value)); + Sleep(600); + std::cout << "child pid=" << pid << std::endl; + + const long long kOneTimeout = static_cast(kWindowsV1UiaConnectionTimeoutMs); + // One timeout (the longer, the connection) and a small margin. Every case stops at one call; the + // counted 800 ms worst case (both waits in one call) is not expected here and would fail this bound. + const long long kBound = kOneTimeout + 200; + const long long kHeldUp = 250; // a call that came back sooner was never held up + + // The very first call of this new process, to a window of another process, reads it: a cold start + // is not turned into "unreadable" by the timeout. + { + std::string tree; + std::size_t nodes = 0; + const auto start = std::chrono::steady_clock::now(); + const bool first = ReadUiaObservation(static_cast(pid), ordinary, &tree, &nodes); + std::cout << "first_call_ok=" << first << " ms=" << Millis(start) << " nodes=" << nodes << std::endl; + SeamCheck(first, "the_first_call_of_a_new_process_reads_an_ordinary_window"); + } + SeamCheck(windows_v1_uia_unbounded_creations.load() == 0, "no_unbounded_creation"); + + // A measurement means nothing if the point is over some other window. + const auto over = [](POINT point, HWND window) { + for (int attempt = 0; attempt < 10; ++attempt) { + HWND at = WindowFromPoint(point); + if (at != nullptr && GetAncestor(at, GA_ROOT) == window) return true; + Sleep(200); + } + return false; + }; + struct Case { + std::string name; + long long elapsed; + bool failed_closed; + bool required_held_up; + }; + std::vector cases; + const auto client_point = [](HWND window, LONG x, LONG y) { + POINT point{x, y}; + ClientToScreen(window, &point); + return point; + }; + + // ---- a window that does not answer at all (another process) + { + WindowsUiaRisk risk; + std::string signature; + const POINT on_button = client_point(hang, 60, 35); // on the BUTTON child + SeamCheck(over(on_button, hang), "the_button_point_is_over_the_hung_window"); + const long long ms = WhileStuck(hang, WM_APP + 1, 0, [&] { + risk = ClassifyWindowsPoint(on_button, static_cast(pid), &signature); + }); + std::string reason; + cases.push_back({"ClassifyWindowsPoint", ms, RejectWindowsRisk(risk, &reason), true}); + } + { + std::string out; + std::size_t count = 0; + bool ok = true; + const long long ms = WhileStuck(hang, WM_APP + 1, 0, [&] { ok = ReadUiaObservation(static_cast(pid), hang, &out, &count); }); + cases.push_back({"ReadUiaObservation", ms, !ok, true}); + } + { + WindowsControlBindings bindings; + bool ok = true; + const long long ms = WhileStuck(hang, WM_APP + 1, 0, [&] { ok = CaptureWindowsControlBindings(static_cast(pid), hang, &bindings); }); + cases.push_back({"CaptureWindowsControlBindings", ms, !ok, true}); + } + { + WindowsSession session; + session.active_window = hang; + session.pid = static_cast(pid); + IUIAutomationElement *target = nullptr; + std::string reason; + bool ok = true; + const long long ms = WhileStuck(hang, WM_APP + 1, 0, [&] { ok = FindWindowsSemanticTarget(session, "x", &target, &reason); }); + if (target != nullptr) target->Release(); + SeamCheck(reason == "accessibility_timeout", "a_timed_out_target_search_says_accessibility_timeout"); + cases.push_back({"FindWindowsSemanticTarget", ms, !ok, true}); + } + // ---- the focused element: the hung window has to be the foreground one, with the focus in its edit + bool focus_classification = false; + bool focus_signature = false; + for (int attempt = 0; attempt < 3 && !focus_classification; ++attempt) { + if (!BringToForeground(hang)) continue; + PostMessageW(hang, WM_APP + 3, 0, 0); + Sleep(300); + if (GetForegroundWindow() != hang) continue; + WindowsUiaRisk risk; + const long long ms = WhileStuck(hang, WM_APP + 1, 0, [&] { risk = ClassifyWindowsFocusedElement(static_cast(pid)); }); + std::string reason; + if (ms >= kHeldUp) { + focus_classification = true; + cases.push_back({"ClassifyWindowsFocusedElement", ms, RejectWindowsRisk(risk, &reason), true}); + } + } + for (int attempt = 0; attempt < 3 && !focus_signature; ++attempt) { + if (!BringToForeground(hang)) continue; + PostMessageW(hang, WM_APP + 3, 0, 0); + Sleep(300); + if (GetForegroundWindow() != hang) continue; + std::string signature; + RECT client{}; + GetClientRect(hang, &client); + bool ok = true; + const long long ms = WhileStuck(hang, WM_APP + 1, 0, [&] { ok = WindowsFocusedElementSignature(static_cast(pid), client, &signature); }); + if (ms >= kHeldUp) { + focus_signature = true; + cases.push_back({"WindowsFocusedElementSignature", ms, !ok, true}); + } + } + const bool focus_case = focus_classification && focus_signature; + std::cout << "focus_case=" << focus_case + << (focus_case ? "" : " focus_skipped_reason=the_hung_window_could_not_be_made_the_foreground_one_with_the_focus") << std::endl; + + // ---- the boundaries of a path: the state is shared by the paths inside it, a Cancel ends it before the + // next call, and the reason an observe gives follows from why it ended + { + SeamCheck(std::string(WindowsV1ObserveReason(WindowsV1UiaStop::kTimeout, "x")) == "accessibility_timeout", "reason_of_a_timeout"); + SeamCheck(std::string(WindowsV1ObserveReason(WindowsV1UiaStop::kBudget, "x")) == "observation_timeout", "reason_of_a_spent_budget"); + SeamCheck(std::string(WindowsV1ObserveReason(WindowsV1UiaStop::kCanceled, "x")) == "observation_stale", "reason_of_a_cancel"); + SeamCheck(std::string(WindowsV1ObserveReason(WindowsV1UiaStop::kNone, "x")) == "x", "reason_otherwise_is_the_steps_own"); + } + { // outside any path a call is refused (fail closed): no call is made, and an action is not sent + bool called = false; + const HRESULT refused = WindowsV1UiaCall([&] { called = true; return S_OK; }); + SeamCheck(refused == E_ABORT && !called && !WindowsV1UiaOk(refused), "a_call_outside_any_path_is_refused_without_being_made"); + SeamCheck(!WindowsV1UiaMayCall() && std::string(WindowsV1UiaEndedReason()) == "semantic_action_failed", "an_action_outside_any_path_is_not_sent"); + SeamCheck(!WindowsV1UiaIsTimeout(E_ABORT), "a_refused_call_is_not_a_timeout"); + SeamCheck(!WindowsV1UiaIsTimeout(E_FAIL), "the_initial_result_of_an_action_not_sent_is_not_a_timeout"); + } + { // a timeout inside an inner path ends the outer one: the later reads of the frame make no call + const WindowsV1UiaPath frame; + WindowsUiaRisk risk; + std::string signature; + const POINT on_button = client_point(hang, 60, 35); + WhileStuck(hang, WM_APP + 1, 0, [&] { risk = ClassifyWindowsPoint(on_button, static_cast(pid), &signature); }); + SeamCheck(frame.Stop() == WindowsV1UiaStop::kTimeout, "the_frame_sees_the_timeout_of_its_inner_path"); + std::string tree; + std::size_t nodes = 0; + const auto start = std::chrono::steady_clock::now(); + const bool read = ReadUiaObservation(static_cast(pid), ordinary, &tree, &nodes); + const long long ms = Millis(start); + std::cout << "after_timeout_in_frame ms=" << ms << " read=" << read << std::endl; + SeamCheck(!read && ms < 50, "a_frame_makes_no_call_after_its_first_timeout"); + SeamCheck(std::string(WindowsV1ObserveReason(frame.Stop(), "x")) == "accessibility_timeout", "the_frames_reason_is_accessibility_timeout"); + } + { // a Cancel ends the path before the next call, even of an app that answers + const WindowsV1UiaPath frame; + cancellation_epoch.fetch_add(1); + std::string tree; + std::size_t nodes = 0; + const auto start = std::chrono::steady_clock::now(); + const bool read = ReadUiaObservation(static_cast(pid), ordinary, &tree, &nodes); + const long long ms = Millis(start); + std::cout << "after_cancel_in_frame ms=" << ms << " read=" << read << std::endl; + SeamCheck(!read && ms < 50, "a_cancel_ends_the_path_before_the_next_call"); + SeamCheck(frame.Stop() == WindowsV1UiaStop::kCanceled, "the_frame_is_canceled"); + SeamCheck(std::string(WindowsV1ObserveReason(frame.Stop(), "x")) == "observation_stale", "the_canceled_observes_reason"); + // A classification whose calls are refused (E_ABORT, not a timeout) falls to the rejecting side. + RECT ordinary_rect{}; + GetWindowRect(ordinary, &ordinary_rect); + const POINT inside_ordinary{(ordinary_rect.left + ordinary_rect.right) / 2, (ordinary_rect.top + ordinary_rect.bottom) / 2}; + std::string signature; + std::string reason; + const WindowsUiaRisk refused = ClassifyWindowsPoint(inside_ordinary, static_cast(pid), &signature); + SeamCheck(!refused.metadata_complete, "a_refused_classification_is_incomplete"); + SeamCheck(RejectWindowsRisk(refused, &reason) && reason == "target_unclassified", "a_refused_classification_is_rejected_as_unclassified"); + } + { // the semantic dispatch of a target that does not answer: a transient rejection before any input + WindowsSession session; + session.active_window = hang; + session.pid = static_cast(pid); + bool accepted = false; + std::string reason; + bool ok = true; + const WindowsV1UiaPath frame; // as the dispatch frame of HandlePipeRequest gives it + const long long ms = WhileStuck(hang, WM_APP + 1, 0, [&] { + ok = DispatchWindowsSemanticAction(session, RECT{}, cancellation_epoch.load(), "invoke", "{\"targetId\":\"x\"}", &accepted, &reason); + }); + std::cout << "semantic_dispatch ms=" << ms << " ok=" << ok << " accepted=" << accepted << " reason=" << reason << std::endl; + SeamCheck(!ok && !accepted && reason == "accessibility_timeout", "a_semantic_dispatch_to_a_target_that_does_not_answer_is_a_transient_rejection"); + SeamCheck(ms <= kBound, "the_semantic_dispatch_returns_within_one_timeout_and_a_margin"); + } + + // ---- a deep tree of a LIVE app that answers every request, slowly: reading it up the parents makes + // dozens of calls and takes longer than any budget a short path might have had, and it must still be + // classified (not rejected as unclassified): the bounds only cut off what does not answer. + { + RECT deepest_rect{}; + GetWindowRect(deepest, &deepest_rect); + const POINT inside{(deepest_rect.left + deepest_rect.right) / 2, (deepest_rect.top + deepest_rect.bottom) / 2}; + SeamCheck(over(inside, GetAncestor(deepest, GA_ROOT)), "the_point_is_over_the_deep_window"); + std::string signature; + const auto start = std::chrono::steady_clock::now(); + const WindowsUiaRisk risk = ClassifyWindowsPoint(inside, static_cast(pid), &signature); + const long long ms = Millis(start); + std::string reason; + const bool rejected = RejectWindowsRisk(risk, &reason); + std::cout << "deep_tree ms=" << ms << " rejected=" << rejected << " reason=" << reason << " complete=" << risk.metadata_complete << std::endl; + SeamCheck(!rejected, "a_deep_tree_that_answers_is_not_rejected"); + SeamCheck(risk.metadata_complete && risk.classified, "a_deep_tree_that_answers_is_classified"); + } + + // ---- a native provider that answers a few requests and then stops: the call that is held up is not + // the first one, so the calls after it are the ones that must not be made + bool provider_held_up = false; + for (int answers : {0, 1, 2, 3, 5}) { + RECT window_rect{}; + GetWindowRect(provider, &window_rect); + const POINT inside{(window_rect.left + window_rect.right) / 2, (window_rect.top + window_rect.bottom) / 2}; + SeamCheck(over(inside, provider), "the_point_is_over_the_provider_window"); + WindowsUiaRisk risk; + std::string signature; + const long long ms = WhileStuck(provider, WM_APP + 2, static_cast(answers), [&] { + risk = ClassifyWindowsPoint(inside, static_cast(pid), &signature); + }); + std::string reason; + const bool held = ms >= kHeldUp; + provider_held_up = provider_held_up || held; + cases.push_back({"ClassifyWindowsPoint@provider-answers-" + std::to_string(answers), ms, held ? RejectWindowsRisk(risk, &reason) : true, false}); + g_provider_stall_after.store(-1); + } + { + std::string baseline_tree; + std::size_t baseline_nodes = 0; + g_provider_stall_after.store(-1); + const bool baseline = ReadUiaObservation(static_cast(pid), provider, &baseline_tree, &baseline_nodes); + std::cout << "provider_baseline_observation=" << baseline << " nodes=" << baseline_nodes << std::endl; + } + for (int answers : {0, 2, 6}) { + std::string out; + std::size_t count = 0; + bool ok = true; + const long long ms = WhileStuck(provider, WM_APP + 2, static_cast(answers), [&] { + ok = ReadUiaObservation(static_cast(pid), provider, &out, &count); + }); + cases.push_back({"ReadUiaObservation@provider-answers-" + std::to_string(answers), ms, ms < kHeldUp ? true : !ok, false}); + } + SeamCheck(provider_held_up, "the_provider_stopped_answering_in_at_least_one_case"); + + for (const Case &c : cases) { + std::cout << "ran " << c.name << " ms=" << c.elapsed << " failed_closed=" << c.failed_closed << std::endl; + SeamCheck(c.elapsed <= kBound, c.name + "_returns_within_one_timeout_and_a_margin"); + if (c.required_held_up) SeamCheck(c.elapsed >= kHeldUp, c.name + "_was_really_held_up"); + SeamCheck(c.failed_closed, c.name + "_fails_closed"); + } + + TerminateProcess(child.hProcess, 0); + CloseHandle(child.hProcess); + CloseHandle(child.hThread); + CloseHandle(job); + std::cout << (failures == 0 ? "OK" : "FAILED") << std::endl; + return failures == 0 ? 0 : 1; +} + +} // namespace sc820 + +int main(int argc, char **argv) { + const std::string mode = argc > 1 ? argv[1] : ""; + if (mode == "window-process") return sc820::RunWindowProcess(); + return sc820::RunDriver(mode == "ci"); +} +`; + +const onWindows = process.platform === 'win32'; + +type Context = { scratch: string; source: string; binary: string; compiler: Build | null }; +let context: Context | null = null; + +// Nothing is created, searched for or compiled off Windows. On Windows without a compiler the test is +// skipped, except in CI, where a runner that cannot compile must fail loudly, not pass silently. The +// scratch directory is made when the tests start and removed when they end, whatever happened. +beforeAll(() => { + if (!onWindows) return; + const scratch = mkdtempSync(join(tmpdir(), 'computer-use-v1-uia-timeout-live-seam-')); + const source = join(scratch, 'seam.cc'); + const binary = join(scratch, 'seam.exe'); + const provenance = join(scratch, 'provenance.h'); + writeFileSync( + provenance, + '#pragma once\n#define SPRINT_CODER_SOURCE_COMMIT "0123456789abcdef0123456789abcdef01234567"\n', + ); + context = { scratch, source, binary, compiler: findBuild(source, binary, provenance) }; +}); + +// The directory is removed with retries (a program that has only just ended may still hold its files for +// a moment) and a warning, not a silent leak, if it cannot be. +afterAll(async () => { + if (context === null) return; + const scratch = context.scratch; + context = null; + let lastError: unknown = null; + for (let attempt = 0; attempt < 6; attempt += 1) { + try { + rmSync(scratch, { recursive: true, force: true }); + return; + } catch (error) { + lastError = error; + await new Promise((resolve) => setTimeout(resolve, 500)); + } + } + console.warn(`The scratch directory ${scratch} could not be removed: ${String(lastError)}`); +}); + +describe('V1 lane UI Automation against windows that do not answer, live, across processes (#820)', () => { + it('comes back within one timeout and fails closed on every V1 path', async (testContext) => { + if (!onWindows || (context?.compiler === null && !process.env['CI'])) { + testContext.skip(); + return; + } + expect(context, 'a Windows build context').not.toBeNull(); + const build = context!; + expect(build.compiler, 'cl.exe (MSVC) is required on a Windows CI runner').not.toBeNull(); + writeFileSync(build.source, program); + const compiler = build.compiler!; + const built = await run(compiler.file, compiler.args, { + cwd: build.scratch, + windowsVerbatimArguments: compiler.verbatim, + timeout: 280_000, + env: { ...process.env, VSLANG: '1033' }, + }); + expect(built.status, `${compiler.name}\n${built.stdout}\n${built.stderr}`).toBe(0); + const result = await run(build.binary, process.env['CI'] ? ['ci'] : [], { timeout: 200_000 }); + console.log(result.stdout); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain('OK'); + // The six V1 paths all ran (the focus ones only when the window could be made the foreground one: + // always required in CI, where the program itself fails without them). + const ran = new Set( + [...result.stdout.matchAll(/^ran (\w+)(?:@[\w-]+)? /gmu)].map((match) => match[1]), + ); + // The six paths all ran, unless the hung window could not be made the foreground one (a machine + // where the foreground cannot be taken): then the two focus paths are skipped and the reason is on + // record, and the four that do not depend on the focus still ran. + const focusSkipped = /^focus_case=0 focus_skipped_reason=\w+/mu.test(result.stdout); + const required = focusSkipped ? sixPaths.slice(0, 4) : sixPaths; + for (const path of required) expect(ran.has(path), `${path} ran`).toBe(true); + if (focusSkipped) + console.warn( + 'The focus paths were skipped: ' + /focus_skipped_reason=\w+/u.exec(result.stdout)![0], + ); + expect(result.stdout).toMatch(/first_call_ok=1/u); + }, 480_000); +}); diff --git a/tasks/designs/adr-computer-use-agent-driven-v2.md b/tasks/designs/adr-computer-use-agent-driven-v2.md index 1ba33a042..c7b3c5439 100644 --- a/tasks/designs/adr-computer-use-agent-driven-v2.md +++ b/tasks/designs/adr-computer-use-agent-driven-v2.md @@ -666,6 +666,8 @@ computer_start がここで返る → { sessionId, state, stopReason, mode, roun **S5-c-2 PR-3(Explorer の実ファイル名。2026-10-17。動作不変)**: 上の S5-c (2) の「Explorer の項目を表示名で探す」を、ビューの項目の走査から、フォルダのパスと列挙に替えた。**照合は規則ではなく、シェルが実際に出す表示名で行う**(拡張子を隠す設定、未登録の拡張子、`.lnk` / `.website`、ドットを含むフォルダ名、ローカライズ名はシェルの答えに任せ、固定の拡張子リストと語幹の規則は持たない)。`ShellItemPathByName` は view を特定した後、次の順で進める。(1) **入口で外す場所(どれも列挙の前。各 I/O の前に取消しと期限を確かめる)**: ID リストが空(デスクトップ名前空間のルート。ビューは Public\Desktop も統合して見せる)、`SFGAO_ISSLOW`、ファイルシステムのディレクトリでない場所、`FOLDERID_Desktop` / `FOLDERID_PublicDesktop` のパス(`KF_FLAG_DONT_VERIFY` で尋ねる。実機では未確認の保険)。ネットワークは 3 段階で外す: ①ドライブ文字の種類(`GetDriveTypeW` を先頭 3 文字に。I/O なし。応答しない SMB 共有に割り当てたドライブには触れない)、②パスの各要素をドライブのルートから見て、symbolic link は拒否し、mount point(ジャンクション)は `FSCTL_GET_REPARSE_POINT` で行き先を読み、ドライブ文字のローカルディスクを指し、かつ行き先のパスの各要素をドライブのルートから見てリンク(symlink、別のジャンクションを含む)が 1 つも無いときだけ許す(行き先の途中が UNC への symlink でありうるため。再帰はせず深さ 1)(`\??\Volume{...}` のようなボリューム指定は不明のまま拒否)、③その後に `GetVolumePathNameW` のボリュームを同じ判定にかける(マウントしたボリュームのルートそのものはパスより長くなるので判定なし)。リンクの判定は `IsReparseTagNameSurrogate`(タグの bit 29)で、クラウドファイルのタグには立たないので許される。経路の途中のジャンクションにも同じ緩和を当てた(ジャンクションの行き先はドライブ文字のパスかボリュームで UNC は指せず、行き先のパスの各要素も上のとおり I/O して確かめる)。desktop.ini に `[LocalizedFileNames]` があるフォルダは判定なし: シェルは読み取り専用か system のフォルダでだけ desktop.ini を使う(実機で確認)ので、そのときだけ読む。ファイルは `CreateFileW(FILE_FLAG_OPEN_REPARSE_POINT)` で 1 回だけ開き、同じハンドルで reparse の有無(種類を問わず「あり」)、大きさの上限(64 KB)、読み取りまでを行い、UTF-16(LE / BE)か ANSI を自前のパーサで判定する(`GetPrivateProfileSectionW` は使わない。パスで開き直してリンクをたどるため)。セクションが空かは、次のセクションまでに空行とコメント以外の行があるかで決め、判定できないものは「あり」とする。(2) フォルダを `FindFirstFileExW(フォルダ\*)` で 1 回列挙する(メタデータだけ。時間は `kShellLookupBudgetNs`)。シェル表示名を取る候補は、ファイル名が表示名で始まるものと、読み取り専用または system のディレクトリ(desktop.ini の LocalizedResourceName で別名になるのはこのフォルダだけ)の合計で、64 件を超えたら切り捨てずに全体を判定なしにする。隠しファイルは外さない。新しい列挙は隠し項目も候補に含めるので、Explorer に表示されない項目にも I/O が発生しうる。ただし、その I/O は上の確認を通ったローカルのものだけ。候補がリンク(symlink、行き先がローカルでない mount point)だと、見つけた時点で全体を判定なしにし、残りの I/O は省く。(3) 候補ごとに、解決先(`SIGDN_FILESYSPATH` の末尾)が候補名と一致するときだけ、`SHCreateItemFromParsingName` と `GetDisplayName(SIGDN_NORMALDISPLAY)` で表示名を取る(末尾がドットや空白の名前は Win32 が別のエントリに正規化するので採らない)。読み取り専用 / system の候補ディレクトリは、シェルがその desktop.ini を読んで名前を決める(リンクをたどる、リソース参照のモジュールを読み込む、CLSID を bind する)ので、先に同じ方式(ハンドルで 1 回)で読み、リンク、読めない、判定できない、`[.ShellClassInfo]` の `CLSID` / `CLSID2` / `UICLSID` がある、`LocalizedResourceName` / `IconResource` / `IconFile` / `InfoTip`(と Logo など)の値が許可リストに無い、のどれかなら、シェルには尋ねず unknown にする。許可するのは、外側の引用符(単一・二重)と空白を外した値が (a) `\ / : % @ ,` を含まないプレーンテキスト、または (b) `%SystemRoot%` / `%windir%` / `%ProgramFiles%` / `%ProgramFiles(x86)%` / `%CommonProgramFiles%` で始まる(先頭に `@`、末尾に `,-N` か `,N` があってもよい)パスで、`/`、連続した `\`、`..`、ほかの `%` を含まないもの、の 2 つだけ。`X:\` で始まる絶対パスは、途中が UNC への symbolic link でありうるので許可しない。ただし環境変数の展開先のフォルダにリンクがある可能性は残る(**既知の限界**。確認していない)。portable の `MatchShellDisplayName` は、表示名がちょうど 1 件に一致し、候補にリンクも読めないものも無いときだけ、その実パスを返す。拡張子の表示設定は読まない。`is_folder` は `FILE_ATTRIBUTE_DIRECTORY` から取る。(4) lookup のスレッドは同期のファイルシステム I/O を発行する。取消しと期限は、新しい I/O を始める直前ごと(入口のシェル呼び出し、場所の判定、リンク、desktop.ini の open / 情報 / 読み取り、最初の列挙、各候補)に確かめる。待ちが切れた側は lookup に放棄を知らせ、スレッドの複製ハンドルへ `CancelSynchronousIo` を送ってブロックした呼び出しを打ち切る。ドライバが戻らない場合は取り消せず、そのとき lookup の枠が埋まり続けて以降の lookup は判定なしになる(V1 に任せる)のは**既知の限界**。確認してから列挙や desktop.ini の open までの間に、経路の途中の要素がリンクへ差し替えられる競合(TOCTOU)も残る(`FILE_FLAG_OPEN_REPARSE_POINT` が守るのは末尾の desktop.ini だけ)。これは攻撃者が同じユーザーでローカルの操作をできる場合に限られる残余のリスクで、同じユーザーのローカルの操作に限られ、候補は FILESYSPATH の末尾で自分自身と確かめられ、dispatch 直前に再評価される(PR-5)ので許容する。lookup のスレッドは例外で process を落とさず、COM の解放、枠の解放、待っている側への通知は必ず通る。UIA 側では、ListItem が shell の項目 view(`SHELLDLL_DefView` または `UIItemsView`)の中にあるときだけ尋ね、プレビューペインや検索候補の ListItem は実パス無し(kIncomplete)とする。(5) **常に判定なしになる場所**(fail closed の代償)。開発機で、実フォルダを読み取りだけで確かめた結果: Documents、Downloads、C:\ProgramData、C:\Program Files は判定できる(Documents の Hidden + System のジャンクション 3 個、ProgramData の 5 個は行き先がローカルなので通る)。判定なしになるのは、ユーザーのホーム(互換用のジャンクションのうち Recent は desktop.ini を開けず unknown になる。ほかの 21 件は通る)、C:\Windows(Downloaded Program Files、Fonts、Offline Web Pages の 3 つが `CLSID` を持つ名前空間拡張のフォルダで、方針どおり unknown)、C:\Users(All Users が symbolic link)。C:\\ のルートは判定できる(読み取りのみで実測。Windows、Program Files が取れた。ルートでは directory の区切りを二重にしていたため全候補が unknown になる不具合があり、`ShellDirectoryWithSeparator` で直した)。OneDrive の desktop.ini がクラウドファイル(reparse point)なら、種類を問わず unknown になる(この機で OneDrive は通った)。アクセス拒否で desktop.ini が開けない候補を「シェルも読めないので安全」とみなす緩和は、fail closed を保つため行っていない。(6) 実測値の読み方: 以前の「ホームで 22 件すべて表示名が取れた」は、リンクの候補を区別する前の数字で、現在の規則では上記のとおり。時間は自作の一時フォルダで、項目数 4 / 120 / 500 / 5,000 のどれでも実パスが取れ、1 回 10〜14 ms 程度(リンクの確認と候補の desktop.ini の確認を含む。最大 21 ms)で、項目数への依存は見えなかった。(7) (1) の旧記述との食い違いの整理: 以前は「symbolic link / mount point を一律に外す」と書き、その結果ホームなどで常に判定なしになることを (7) に書いていたが、現在は mount point の行き先がローカルのドライブ文字パスなら通す(symbolic link と行き先が不明のものは外す)。lookup の待ち(150 ms)、PR-1 で分けた枠の引数、実パスが無いファイル項目は kIncomplete にする規則(S5-c-1 (2))は変えていない。実機で確かめたもの: `x` と `x.website`(判定なし)、LICENSE と LICENSE.md、フォルダ `data` と `data.xxx`、`[LocalizedFileNames]` のある desktop.ini(UTF-16 と ANSI、読み取り専用のフォルダで判定なし、通常のフォルダでは無視される)、コメントだけの節(取れる)、desktop.ini で別名になる読み取り専用フォルダ、末尾がドットの名前、ジャンクション(行き先がローカルなら通る)、UNC のリソース参照や CLSID を持つ候補ディレクトリ(判定なし)。desktop.ini の symbolic link は権限がなく作れなかったので、harness と source 契約のみで固定している。拡張子を隠す設定と Desktop / Public Desktop は実機で未確認。 +**V1 のレーンの UI Automation にタイムアウトを付けた(#820。S5-c-2 の計画で範囲外としていた R8)**: V1 自身がレーン(直列の処理スレッド)の上で呼ぶ UI Automation(observe の木の読み取り、点とフォーカスの分類、control bindings、semantic target の検索、再検証が使うフォーカス要素の署名)にはタイムアウトが無く、操作の対象が応答しなくなると呼び出しが戻らず、その後の observe、preflight、dispatch、Cancel がすべて進まなくなった。V1 の UIA は `CreateWindowsV1Automation` の 1 か所で作り(8 か所の `CoCreateInstance(CLSID_CUIAutomation, ...)` をまとめた)、shadow の `WindowsUiaShadowSource::Create` と共通の `CreateBoundedAutomation`(`CLSID_CUIAutomation8` + `IUIAutomation2`)で ConnectionTimeout / TransactionTimeout を設定する。`IUIAutomation2` が取れないときは従来どおりのタイムアウトの無い `CLSID_CUIAutomation` に戻り(コード上の直接生成はこのフォールバック 1 か所だけ)、回数を数えて初回だけ `OutputDebugString` に記録する。**3 つの規則**。(1) 呼び出し 1 回を縛る: ConnectionTimeout は 500 ms、TransactionTimeout は 300 ms。応答しない窓で実測すると、最初の接触(`ElementFromHandle` / `ElementFromPoint` / `GetFocusedElement`)を縛るのは ConnectionTimeout のほうで(300 に設定して 311 ms、2000 で 2001 ms。TransactionTimeout を変えても変わらない)、接続済みの要素への呼び出しを縛るのが TransactionTimeout である。接続は cold な app が provider を読み込む最初の呼び出しでもあるので 500 ms にして生きている app を「読めない」にしにくくし、接続後の呼び出しは 300 ms で足りる。1 回の呼び出しが接続の待ちのあとに transaction の待ちも続けて起こしうるかは確かめていないので、**1 回の最悪値は安全側に 500 + 300 = 800 ms と数える**。(2) 最初に時間切れ(`UIA_E_TIMEOUT`、`ERROR_TIMEOUT`、`RPC_E_TIMEOUT`)になった呼び出しで、その経路(`WindowsV1UiaPath`)を終わらせ、その後は UI Automation を一切呼ばない。最初の接触に答えてから止まる対象は、残りの呼び出しの数ではなく、時間切れ 1 回分で済む。(3) Cancel が届いた(`cancellation_epoch` が経路の開始時から動いた)ら、同じく経路を終わらせる。確認は**すべての呼び出しの前**(`WindowsV1UiaCall` と、各関数の入口の property 読み取り)で行うので、Cancel の遅れは、そのとき行われている呼び出し 1 回分以下(最悪 800 ms)になり、Main の Cancel の確認の 1 s(`COMPUTER_USE_NATIVE_CANCEL_ACK_TIMEOUT_MS`)に収まる(`native_stop_unconfirmed` にならない)。**ただし余裕は小さい**: 1000 − 800 = 約 200 ms が、epoch が動いてから確認を返すまでの他の処理(pipe、フレーム、応答)の分のすべてで、これは「余裕がある」とは言えない値である。CI(Windows)の実測では、transaction で縛られる呼び出しは 310〜320 ms で、最初の接触は 505〜514 ms で切れたので、1 つずつ測った 2 つの値の合計は約 830 ms で(連鎖そのものは測っていない。800 ms は公称値で、測った上限ではない)、pipe・フレーム・確認の応答に使える余裕は合わせて約 170 ms になる。負荷の高い CI や実機ではさらに小さくなりうるが、これは確かめていない。**interlock の縮退も既知の限界である**(PR-5 の設計のまま、#820 では変えない): interlock が時間切れ(kBudget)や、動いたままの worker による single-flight で判定を出せないときは『判定なし』で、interlock としては拒否しない(fail open)。その場合は V1 自身の preflight と分類が決める(未分類なら `target_unclassified` で拒否する)。見捨てた worker が UI Automation の呼び出しから戻らない間は、その後の interlock もすべて判定なしになり、その間 interlock は実質的に働かない。この値を変えるとき(timeout を伸ばすとき)は、この 1 s との関係を先に見直すこと。live seam が CI で確かめる上限は「Connection の 500 ms + 余裕 200 ms」で、これは各ケースが 1 回の呼び出し(接続か transaction のどちらか一方)で止まることを測っている。数えた 800 ms(1 回で両方の待ちが続く場合)は観測されていない安全側の見積りで、もし起きれば live seam の上限を超えて失敗する(それが警報になる)。ただしこの警報は、live seam の窓(応答しない窓、途中で止まる provider)でその連鎖が実際に起きたときにだけ鳴る。連鎖を意図して起こす fixture は無いので、800 ms の連鎖そのものは CI で検証されていない(**既知の限界**)。**経路に時間の予算は置かない**。一度、短い経路に 400 ms の予算を置いたが、深い木(Electron や Office で、要素を親へたどって 1 回ずつ呼ぶ)では、応答している app でも 400 ms を超えて `metadata_complete=false` になり、`target_unclassified` で拒否されてしまう。V1 の答えを変える退行なので外した。時間の予算は observe 全体(25 s)だけに残す。Cancel のためではなく(それは epoch の確認の役目)、Main が observe に与える 30 s が尽きる前に native が自分の理由(`observation_timeout`)で答えるためで、数千ノードの普通の app の木を 4 回(2 回の読み取りと 2 回の bindings)たどっても失敗しない広さにした。予算切れのときは、読んだノード数だけを `OutputDebugString` に出す(ラベルや文字列は出さない)。**時間切れを HRESULT だけで判断しない**。CI の Windows で、まったく答えない native provider の observe が 1.5 s かかって成功扱いになった: 5 つの property 読み取りがそれぞれ約 300 ms で時間切れになりながら、S_OK と空の値を返していた(ローカルでは UIA_E_TIMEOUT が返る)。そこで、各呼び出しの時間を測り、timeout の 30 ms 手前以上かかった呼び出しは、HRESULT が成功でも時間切れとして扱う(transaction の 300 ms に対して 270 ms、connection の 500 ms に対して 470 ms)。property の読み取りは、semantic action の中(名前、pattern の取得、toggle / expand の状態)も含めて、すべて `WindowsV1UiaRead` を通す。読み取りが完了しなかったとき(時間切れ、遅い成功)は、返ってきた pattern や BSTR は解放して捨て、副作用のある呼び出し(Invoke / SetValue / Select / Toggle / Expand / Collapse)は、直前にもう一度 `WindowsV1UiaMayCall`(時間切れと epoch)を確かめて、止まっていれば送らない(accepted は偽のまま、理由は `accessibility_timeout`)。select は名前が読めなかったら、読めなかった名前と比較せずに止まる。**フレーム全体**: preflight と dispatch の 1 フレームは外側の `WindowsV1UiaPath`(予算なし、開始時の epoch)で包み、フレームの中の短い経路(interlock の後の署名、分類、再検証)はその状態を共有する。フレームの中で最初に時間切れが起きたら残りの UIA は呼ばず、Cancel が届いたら次の呼び出しの前で止まる(key / type の 2 回目の署名や `ClassifyWindowsFocusedElement` の前も同じ)。Cancel の遅れは、そのとき行われている 1 回の呼び出し(V1 の UIA は最悪 800 ms、interlock の worker は 500 ms で打ち切り)以下で、1 s 以内になる。dispatch の共通の後処理は、`input_timeout_unconfirmed` を `input_not_confirmed` で上書きしない。**途中で止まる対象**。ClassifyWindowsUiaElement は、Self が読めなかった(`metadata_complete` が偽)ところで親をたどるのをやめる。`ClassifyNativePreflightFacts` は metadata が不完全なら secure や high impact より先に「未分類」を返すので、理由コード(`target_unclassified`)は変わらない。ClassifyWindowsPoint は、facts が不完全なら control signature を計算しない(呼び出し元は拒否して署名を読まない)。observe は、フォーカス要素の署名を分類の結果に関わらず読む(#820 より前と同じ)。生きているが未分類なだけのフォーカス(たとえば、たどる上限より深い要素)は署名を保つので、その key / type は「フォーカスが動いた」ではなく `target_unclassified` で拒否される。同じ observe の中で時間切れや Cancel が起きたあとは、経路がその読み取りを呼ばずに拒否するので、応答しない要素をもう一度待つことはない。**fail closed と理由**。時間切れの呼び出しは失敗の HRESULT を返し、各呼び出し元は既に「読めない」として扱っている: 点とフォーカスの分類は未分類のリスクを返して `RejectWindowsRisk` が `target_unclassified` で拒否し、observe の読み取り、control bindings、フォーカス要素の署名は false になって観測の失敗または `stale_target`、semantic target の検索は失敗する。V1 の判定の意味は変えていない。区別のために理由を足した: 時間切れは `accessibility_timeout`(一時的。cold な app が provider を読み込んだあとの再 observe では起きない。UIA が無い `accessibility_unavailable` と区別する)、observe 全体の予算切れは `observation_timeout`、cancel は `observation_stale`。native のエラーの code は Main の写しにそのまま渡り(observe は例外として、dispatch の失敗は accepted が偽なら `native_accessibility_timeout` の rejected として)、理由の一覧は Main には無い。**副作用のある pattern**(Invoke / SetValue / Select / Toggle / Expand)の呼び出しも同じ TransactionTimeout で切れるが、切れたときに操作が適用済みの可能性がある。timeout は変えず、その失敗が時間切れなら理由を `input_timeout_unconfirmed` にして `input_not_confirmed` と区別する。時間切れは待ちを打ち切るだけで、provider 側の処理はあとから適用されうるので、`input_timeout_unconfirmed` は「効果不明」の意味で固定する。時間切れと判定するのは時間切れ系の HRESULT(`UIA_E_TIMEOUT`、`ERROR_TIMEOUT`、`RPC_E_TIMEOUT`)だけで、それ以外の失敗は従来どおりの理由になる。どちらも accepted が真なので Main は例外として扱い、controller が効果不明として記録する(`input_not_confirmed` と同じ)。二重実行を避けるため、再試行は自動では行わず、新しい observe を見た agent の判断に任せる。副作用のある呼び出しそのものは時間を測るラッパーに入れない(遅れて成功したものを時間切れに書き換えると、適用された操作を未適用と扱うことになる)。直前の `WindowsV1UiaMayCall` から呼び出しまでの間に対象が止まる余地は構造上残る(**既知の限界**)が、その呼び出しも TransactionTimeout で切れて効果不明として返る。テスト: source 契約で、生成箇所、定数どうしの関係(Connection + Transaction が Cancel の確認の 1 s 未満、observe の予算が 30 s 未満、短い経路の予算が無いこと)、すべての呼び出しが `WindowsV1UiaCall` を通ること、各経路の止め方を固定する。live seam は、本物の host.cc をコンパイルし、自分自身を Job Object(親が終われば閉じる)に入れた子プロセスとして起動して、別プロセスの窓(応答しない窓、普通の窓、数回の問い合わせに答えてから止まる native provider の窓、応答は遅いが答える深い木)に対して V1 の 6 経路と provider の途中停止を呼び、1 回の timeout + 小さな余裕の範囲に戻って閉じる方向に失敗すること、深い木が拒否されないこと、新しいプロセスの最初の 1 回で普通の窓が読めることを確かめる。フォーカス系の 2 経路は、窓を前面にできたときだけ走り、できなければ理由を出力して skip する(flaky を避けるため、CI でも失敗にしない)。経路の境界(入れ子の経路が時間切れの状態を共有すること、Cancel が次の呼び出しの前で止めること、理由の写し方、semantic dispatch の時間切れが accepted の偽の `accessibility_timeout` になること)も確かめる。 + **S5-c-2 PR-5(拒否の有効化。2026-10-17)**: Windows の preflight と dispatch から同期の入口を呼び、S4-1a の受領書で拒否する。S5-c の (1)〜(6) は次のとおり解決済みである。 (1) UIA のタイムアウトは同期の入口で 250 ms、レーンの保持の上限は 500 ms(最悪の見積りは PR-1 の節のとおり。preflight と dispatch の 2 回で 1 操作あたり最大約 1 s の遅れ)。(2) Explorer の項目は PR-3 のとおりフォルダのパスと列挙で実パスを取る(lookup は 150 ms の待ち)。実パスが取れたファイル項目(`run.bat` の実行トリガ、`invoice.pdf` の承認が閉じた件)は、この PR で拒否される。取れない・曖昧・仮想フォルダ・枠の使用中は判定なし(kIncomplete)のまま。(3) lookup の枠は shadow と同期評価で分けた。(4) MTA は評価ごとのワーカーで実現し、主スレッドは picker と WinRT の捕捉のために STA に残す。(5) 座標写像は `WindowsActionPoint` の 1 つ。(6) 開始前の取消しは UIA を作らずに kCanceled。 **配線**: `WindowsInterlockRefusal`(同期の入口の区間にある小さな関数。seam から差し替えられる)が、評価と、判定なしの理由ごとの件数(`OutputDebugStringA`。enum 名と件数だけでラベルは出さない)を受け持つ。ruleset は `kWindowsInterlockRuleset`(承認経路なし。win32 の v2 build は `build-computer-use-eligibility.mjs` で拒否されるので、C4-2c-2 (7) の門は Windows では自明に閉じている。Windows に route v2 は無い)。`HandleWindowsPreflight` は V1 の binding の検証が成功した後、ticket を出す前に評価し、拒否なら ticket を出さずに `blocked` / `takeover` の受領書(`DecisionName` / `ReasonCode` / `DeniedResult`)を返す。送る前に `NativePreflightRefusalAllowed` を通し、表に無い組は `denied` / `native_preflight_invalid`(fail closed)。判定なしなら従来どおり V1 の ordinary ticket。shadow は従来どおり呼ぶ(拒否のとき `issued=false`)。`DispatchWindowsAction` は binding の検証の直後、最初の入力 API の試行(`input_api_attempts`、`SetCursorPos`、`SendInput`、`DispatchWindowsSemanticAction`)より前にもう一度評価し、拒否なら `DispatchCode()`(`shell_surface_blocked` / `execution_trigger_blocked` / `approval_not_enabled`)と `accepted:false` で入力 0 回で返す。その後の V1 の guard と再検証は同じ順で残す。Main の `classifyComputerUseNativeDispatchFailure` は、この 3 つのコードを `NATIVE_PREFLIGHT_REFUSALS` の同じ行の result に写す(実行トリガは `paused`、それ以外は `rejected`)。V1 のコードは `rejected` のまま。この写し方は macOS の `dispatchApproved` と共有する。