From 5929a99e846e9c46e71edd404e3ef6ef315944a0 Mon Sep 17 00:00:00 2001 From: yusei Date: Sat, 17 Oct 2026 11:14:04 +0900 Subject: [PATCH 1/5] =?UTF-8?q?feat(computer-use):=20#500=20S5-c-2=20PR-5?= =?UTF-8?q?=20Windows=20=E3=81=AE=20preflight=20=E3=81=A8=20dispatch=20?= =?UTF-8?q?=E3=81=A7=20interlock=20=E3=81=AE=E6=8B=92=E5=90=A6=E3=82=92?= =?UTF-8?q?=E6=9C=89=E5=8A=B9=E5=8C=96=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- .../computer_use_native_interlock.h | 15 ++ .../computer_use_protocol_fuzz.cc | 166 +++++++++++++ .../computer_use_windows_host.cc | 88 ++++++- .../src/main/computer-use-native-host.test.ts | 57 +++++ .../src/main/computer-use-native-host.ts | 19 ++ .../computer-use-windows-click-rule.test.ts | 3 +- ...se-windows-dispatch-interlock-seam.test.ts | 218 ++++++++++++++++++ ...uter-use-windows-interlock-capture.test.ts | 120 +++++++++- ...r-use-windows-interlock-vocabulary.test.ts | 83 +++++++ ...computer-use-windows-shadow-wiring.test.ts | 9 +- .../computer-use-windows-ticket-seam.test.ts | 99 ++++++++ .../adr-computer-use-agent-driven-v2.md | 8 + 12 files changed, 867 insertions(+), 18 deletions(-) create mode 100644 apps/desktop/src/main/computer-use-windows-dispatch-interlock-seam.test.ts create mode 100644 apps/desktop/src/main/computer-use-windows-interlock-vocabulary.test.ts diff --git a/apps/desktop/computer-use-native/computer_use_native_interlock.h b/apps/desktop/computer-use-native/computer_use_native_interlock.h index be0e562c4..b0be06919 100644 --- a/apps/desktop/computer-use-native/computer_use_native_interlock.h +++ b/apps/desktop/computer-use-native/computer_use_native_interlock.h @@ -60,6 +60,21 @@ struct InterlockVerdict { } return {}; } + // The error code a dispatch gives when it refuses (the preflight code without its "native_" + // prefix: Main adds the prefix back). One reason, one code; empty when there is no verdict. + [[nodiscard]] std::string_view DispatchCode() const { + switch (reason) { + case InterlockReason::kShellSurface: + return "shell_surface_blocked"; + case InterlockReason::kExecutionTrigger: + return "execution_trigger_blocked"; + case InterlockReason::kApprovalClosed: + return "approval_not_enabled"; + case InterlockReason::kNone: + break; + } + return {}; + } // The receipt's decision for this refusal (the first column of the receipt table). [[nodiscard]] std::string_view DecisionName() const { switch (outcome) { diff --git a/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc b/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc index 915c11e31..114f661a6 100644 --- a/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc +++ b/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc @@ -7627,6 +7627,170 @@ bool CheckClickRule() { return true; } +// ---- #500 S5-c-2 PR-5: the dispatch's refusal codes --------------------------------------------- + +bool CheckInterlockDispatchCodes() { + using namespace sprint_coder::computer_use; + // One reason, one dispatch code: the preflight code without its "native_" prefix (Main adds it back + // and takes the result from the same row of the receipt table). No verdict, no code. + struct Row { InterlockOutcome outcome; InterlockReason reason; const char *dispatch; const char *preflight; }; + const Row rows[] = { + {InterlockOutcome::kBlocked, InterlockReason::kShellSurface, "shell_surface_blocked", "native_shell_surface_blocked"}, + {InterlockOutcome::kTakeover, InterlockReason::kExecutionTrigger, "execution_trigger_blocked", "native_execution_trigger_blocked"}, + {InterlockOutcome::kBlocked, InterlockReason::kApprovalClosed, "approval_not_enabled", "native_approval_not_enabled"}, + }; + for (const Row &row : rows) { + InterlockVerdict verdict; + verdict.outcome = row.outcome; + verdict.reason = row.reason; + if (!Check(verdict.DispatchCode() == row.dispatch, "dispatch-code-of-the-reason")) return false; + if (!Check(std::string("native_") + std::string(verdict.DispatchCode()) == verdict.ReasonCode() && + verdict.ReasonCode() == row.preflight, + "dispatch-code-is-the-preflight-code-without-the-prefix")) return false; + // The pair the preflight sends is a row of the closed table (the dispatch has the same reason). + if (!Check(NativePreflightRefusalAllowed(verdict.DecisionName(), verdict.ReasonCode(), verdict.DeniedResult()), + "interlock-pair-is-a-table-row")) return false; + } + // One to one: the three codes are pairwise different, and a verdict with nothing has none. + if (!Check(std::string_view("shell_surface_blocked") != "execution_trigger_blocked" && + std::string_view("execution_trigger_blocked") != "approval_not_enabled" && + std::string_view("shell_surface_blocked") != "approval_not_enabled", + "dispatch-codes-differ")) return false; + if (!Check(InterlockVerdict{}.DispatchCode().empty(), "no-verdict-has-no-dispatch-code")) return false; + if (!Check(InterlockVerdict{}.ReasonCode().empty() && InterlockVerdict{}.DecisionName().empty(), + "no-verdict-has-no-receipt-fields")) return false; + std::cout << "Computer Use native interlock dispatch codes: PASS\n"; + return true; +} + +// ---- #500 S5-c-2 PR-5: what V1's users can reach is not newly refused -------------------------------- +// +// V1 reaches the acceptance fixture and Notepad only. Their UI Automation trees (every control of the +// fixture; the old and the new Notepad) are fed to the interlock with every action kind and a title in +// English and Japanese. A refusal is allowed only where V1's own word list refuses too, or for one of +// the words the owner accepted (the vocabulary test pins the list). + +bool CheckWindowsInterlockJourneys() { + using namespace sprint_coder::computer_use; + const EffectRuleset closed; // the Windows rule set: no approval path (download_open / file_manager_open false) + const auto node = [](const char *type, const char *klass, const std::string &name, bool password = false) { + ShadowUiaNode item; + item.control_type = type; + item.class_name = klass; + item.name = name; + item.is_password = password; + return item; + }; + const auto capture = [&](std::vector chain, const std::string &window_class, + const std::string &process_path) { + ShadowUiaCapture result; + result.target_found = true; + result.chain = std::move(chain); + result.chain_reached_root = true; + result.neighbors_complete = true; + result.window_class_read = true; + result.window_class = window_class; + result.process_path_read = true; + result.process_path = process_path; + return result; + }; + const char *const kinds[] = {"click", "invoke", "key", "type", "set_text", "scroll", "select", "toggle", "expand_collapse"}; + const auto refused = [&](const ShadowUiaCapture &tree, const char *kind, const char *language) { + ShadowRequestInfo request = UiaRequest(kind); + request.policy_language = language; + request.key = "Enter"; + request.click_tracked = true; + // The worst case for a click: it may open (a double click). + request.click_opens = std::string_view(kind) == "click"; + return EvaluateWindowsInterlockFromCapture(tree, request, false, closed).Refuses(); + }; + + // The acceptance fixture: every control, in the window of the fixture. + const std::string fixture_title = "Sprint Coder Computer Use acceptance fixture"; + const std::string fixture_path = "C:\\Fixture\\win32_acceptance_fixture.exe"; + struct Control { const char *type; const char *klass; const char *name; bool password; }; + const Control controls[] = { + {"Text", "Static", "Deterministic non-production fixture. Normal controls are safe; boundary controls must stop Computer Use.", false}, + {"Text", "Static", "Normal text", false}, + {"Edit", "Edit", "Normal text", false}, + {"Button", "Button", "Run deterministic normal action", false}, + {"CheckBox", "Button", "Enable deterministic option", false}, + {"Text", "Static", "Normal selection", false}, + {"ComboBox", "ComboBox", "Normal selection", false}, + {"Text", "Static", "Password field - MUST BLOCK", false}, + {"Edit", "Edit", "Password field", true}, + {"Text", "Static", "Safety boundary controls below are inert test affordances. They perform no payment and retain no selected path.", false}, + {"Button", "Button", "Authorize test payment - MUST BLOCK", false}, + {"Button", "Button", "Open file picker - USER TAKEOVER", false}, + {"Button", "Button", "Reset deterministic state", false}, + {"Button", "Button", "Open same-owner safe dialog", false}, + {"Text", "Static", "NORMAL_ACTION_COMPLETED: deterministic button invoked", false}, + }; + for (const Control &control : controls) + for (const char *kind : kinds) + for (const char *language : {"en", "ja"}) + if (!Check(!refused(capture({node(control.type, control.klass, control.name, control.password), + node("Window", "SprintCoderComputerUseFixtureV1", fixture_title)}, + "SprintCoderComputerUseFixtureV1", fixture_path), + kind, language), + (std::string("journey-fixture-not-refused-") + control.name).c_str())) + return false; + // The safe dialog the fixture opens, and the file picker (the picker is a takeover of V1's own, + // outside the interlock). + for (const char *kind : kinds) + if (!Check(!refused(capture({node("Button", "Button", "Close safe dialog"), + node("Window", "SprintCoderComputerUseSafeDialogV1", "Fixture same-owner safe dialog")}, + "SprintCoderComputerUseSafeDialogV1", fixture_path), + kind, "en"), + "journey-safe-dialog-not-refused")) + return false; + + // Notepad, old (a Window, an Edit, a menu bar) and new (a Document), English and Japanese titles. + const std::string notepad_path = "C:\\Windows\\System32\\notepad.exe"; + struct Tree { const char *name; std::vector chain; }; + const std::vector notepads = { + {"old-edit", {node("Edit", "Edit", "Text Editor"), node("Window", "Notepad", "Untitled - Notepad")}}, + {"old-edit-ja", {node("Edit", "Edit", u8"テキスト エディター"_u8), node("Window", "Notepad", u8"無題 - メモ帳"_u8)}}, + {"old-menu", {node("MenuItem", "", "File"), node("MenuBar", "", "Application"), node("Window", "Notepad", "Untitled - Notepad")}}, + {"old-menu-ja", {node("MenuItem", "", u8"ファイル(F)"_u8), node("MenuBar", "", u8"アプリケーション"_u8), node("Window", "Notepad", u8"無題 - メモ帳"_u8)}}, + {"old-find", {node("Button", "Button", "Find Next"), node("Window", "#32770", "Find")}}, + {"old-replace", {node("Button", "Button", "Replace All"), node("Window", "#32770", "Replace")}}, + {"old-font", {node("Button", "Button", "OK"), node("Window", "#32770", "Font")}}, + {"new-document", {node("Document", "RichEditD2DPT", "Text editor"), node("Window", "Notepad", "Untitled - Notepad")}}, + {"new-document-ja", {node("Document", "RichEditD2DPT", u8"テキスト エディター"_u8), node("Window", "Notepad", u8"無題 - メモ帳"_u8)}}, + {"new-saved-file", {node("Document", "RichEditD2DPT", "Text editor"), node("Window", "Notepad", "notes.txt - Notepad")}}, + {"new-saved-file-ja", {node("Document", "RichEditD2DPT", u8"テキスト エディター"_u8), node("Window", "Notepad", u8"メモ.txt - メモ帳"_u8)}}, + }; + for (const Tree &tree : notepads) + for (const char *kind : kinds) + for (const char *language : {"en", "ja"}) + if (!Check(!refused(capture(tree.chain, tree.chain.back().class_name, notepad_path), kind, language), + (std::string("journey-notepad-not-refused-") + tree.name).c_str())) + return false; + + // Where the refusal is allowed. A title with a word V1 refuses too: refused (no change for V1). + const auto titled = [&](const std::string &title) { + return capture({node("Edit", "Edit", "Text Editor"), node("Window", "Notepad", title)}, "Notepad", notepad_path); + }; + for (const char *title : {"terminal notes.txt - Notepad", "console log - Notepad", "shell.txt - Notepad", "powershell tips - Notepad"}) + if (!Check(refused(titled(title), "click", "en") && refused(titled(title), "type", "en"), + (std::string("journey-v1-word-in-the-title-is-refused-") + title).c_str())) + return false; + // The words the owner accepted (not refused by V1 today): refused now, and only these. + for (const std::string &title : std::vector{"xterm settings - Notepad", "run script.txt - Notepad", u8"ターミナル手順.txt - メモ帳"_u8, + u8"コンソール - メモ帳"_u8, u8"コマンドプロンプト入門 - メモ帳"_u8, u8"スクリプトを実行 - メモ帳"_u8}) + if (!Check(refused(titled(title), "click", "ja"), (std::string("journey-accepted-new-word-is-refused-") + title).c_str())) + return false; + // The same words in full width are folded to ASCII before they are matched (NFKC): refused as in + // ASCII. This is the spelling difference the owner accepted (V1 matches the characters as they are). + for (const std::string &title : std::vector{u8"ｘｔｅｒｍ settings - Notepad"_u8, u8"ｔｅｒｍｉｎａｌ notes - Notepad"_u8}) + if (!Check(refused(titled(title), "click", "en"), (std::string("journey-full-width-word-is-refused-") + title).c_str())) + return false; + + std::cout << "Computer Use native Windows interlock journeys: PASS\n"; + return true; +} + int main() { if (!CheckNativePreflightClassifier()) return 1; std::cout << "Computer Use native classifier core: PASS (16 facts, 100 authority tuples)\n"; @@ -7651,6 +7815,8 @@ int main() { if (!CheckNativeInterlockCapture()) return 1; if (!CheckWindowsInterlockCapture()) return 1; if (!CheckClickRule()) return 1; + if (!CheckInterlockDispatchCodes()) return 1; + if (!CheckWindowsInterlockJourneys()) return 1; if (!CheckNativeApprovedRisk()) return 1; if (!CheckNativeApprovalOffer()) return 1; if (!CheckNativePreflightV2()) return 1; diff --git a/apps/desktop/computer-use-native/computer_use_windows_host.cc b/apps/desktop/computer-use-native/computer_use_windows_host.cc index 798ec1167..73c61bf75 100644 --- a/apps/desktop/computer-use-native/computer_use_windows_host.cc +++ b/apps/desktop/computer-use-native/computer_use_windows_host.cc @@ -80,6 +80,7 @@ #include "computer_use_windows_uia_source.h" #include "computer_use_windows_shadow_scheduler.h" #include "computer_use_windows_interlock_capture.h" +#include "computer_use_native_refusal_table.h" namespace { @@ -4368,6 +4369,12 @@ bool WindowsActionPoint(const RECT &bounds, const std::string &metadata, } // END windows action point +// The v2 interlocks' answer for an action whose V1 binding is valid (defined with the synchronous +// entry): a refusal, or no verdict. +sprint_coder::computer_use::InterlockEvaluation +WindowsInterlockRefusal(const WindowsSession &session, const std::string &metadata, + const std::string &kind) noexcept; + bool DispatchWindowsAction(const WindowsSession &session, const std::string &metadata, std::string *result, std::string *reason, bool *accepted) { @@ -4377,6 +4384,18 @@ bool DispatchWindowsAction(const WindowsSession &session, RECT bounds{}; if (!ValidateWindowsDispatchBinding(session, metadata, &kind, &bounds, reason)) return false; + // The v2 interlocks again, right before any input (#500 S5-c-2 PR-5): the screen can have changed + // since the preflight (the ticket lives seconds), and the click facts (the last input time) are read + // again by the evaluation. A refusal is made before the first input API attempt, SetCursorPos or + // SendInput. No verdict leaves the V1 guards below as they are. + { + const sprint_coder::computer_use::InterlockEvaluation evaluation = + WindowsInterlockRefusal(session, metadata, kind); + if (evaluation.Refuses()) { + *reason = std::string(evaluation.verdict.DispatchCode()); + return false; + } + } bool dispatched = false; if (kind == "click") { double x = 0; @@ -4726,13 +4745,19 @@ std::string WindowsPreflightReceipt(const WindowsSession &session, std::uint64_t observation_revision, std::string_view decision, std::string_view denied_reason, - const std::string &ticket) { + const std::string &ticket, + std::string_view refusal_result = "rejected") { const bool denied = decision == "denied"; const bool ordinary = decision == "ordinary"; + // A v2 refusal (blocked / takeover) carries its reason and the result of its own table row. + const bool refused = decision == "blocked" || decision == "takeover"; + const bool with_reason = denied || refused; return "{\"decision\":\"" + std::string(decision) + "\",\"reasonCode\":" + - (denied ? "\"" + std::string(denied_reason) + "\"" : std::string("null")) + - ",\"deniedResult\":" + (denied ? "\"rejected\"" : "null") + + (with_reason ? "\"" + std::string(denied_reason) + "\"" : std::string("null")) + + ",\"deniedResult\":" + + (denied ? "\"rejected\"" + : (refused ? "\"" + std::string(refusal_result) + "\"" : std::string("null"))) + ",\"ticket\":" + (ordinary ? "\"" + ticket + "\"" : "null") + ",\"classifierVersion\":" + std::to_string(sprint_coder::computer_use::kNativePreflightClassifierVersion) + @@ -4915,7 +4940,7 @@ std::atomic &InterlockWorkerRunning() { return *running; } -[[maybe_unused]] sprint_coder::computer_use::InterlockEvaluation EvaluateWindowsInterlockOnLane( +sprint_coder::computer_use::InterlockEvaluation EvaluateWindowsInterlockOnLane( const WindowsSession &session, const std::string &metadata, const std::string &kind, const sprint_coder::computer_use::EffectRuleset &ruleset) noexcept { using namespace sprint_coder::computer_use; @@ -5032,6 +5057,39 @@ std::atomic &InterlockWorkerRunning() { return unavailable; } } +// Windows has no approval path (the route v2 build is refused for win32 by +// build-computer-use-eligibility.mjs), so a file opened from a file manager or a download list is a +// closed approval. +constexpr sprint_coder::computer_use::EffectRuleset kWindowsInterlockRuleset{ + /*download_open=*/false, /*file_manager_open=*/false}; + +// How often the interlocks had NO verdict, by reason, and how often the lane's time ran out. Only +// enum names and counts are written, never a label or a title. +void CountWindowsInterlockNoVerdict(sprint_coder::computer_use::InterlockNoVerdict reason) noexcept { + using sprint_coder::computer_use::InterlockNoVerdict; + static std::atomic counts[4]; + const std::size_t index = static_cast(reason); + if (index >= 4) return; + const std::uint64_t count = counts[index].fetch_add(1, std::memory_order_relaxed) + 1; + const char *name = reason == InterlockNoVerdict::kCanceled + ? "canceled" + : (reason == InterlockNoVerdict::kBudget ? "budget" : "incomplete"); + char line[96]; + const int written = std::snprintf(line, sizeof(line), "computer-use interlock: no verdict %s count=%llu\n", + name, static_cast(count)); + if (written > 0) OutputDebugStringA(line); +} + +sprint_coder::computer_use::InterlockEvaluation +WindowsInterlockRefusal(const WindowsSession &session, const std::string &metadata, + const std::string &kind) noexcept { + const sprint_coder::computer_use::InterlockEvaluation evaluation = + EvaluateWindowsInterlockOnLane(session, metadata, kind, kWindowsInterlockRuleset); + if (!evaluation.Refuses() && + evaluation.no_verdict != sprint_coder::computer_use::InterlockNoVerdict::kNone) + CountWindowsInterlockNoVerdict(evaluation.no_verdict); + return evaluation; +} // END sync interlock entry bool HandleWindowsPreflight(HANDLE pipe, const Frame &request, @@ -5082,6 +5140,28 @@ bool HandleWindowsPreflight(HANDLE pipe, const Frame &request, ScheduleWindowsEffectShadow(session, metadata, kind, false, false); return denied_sent; } + // The v2 interlocks (#500 S5-c-2 PR-5): AFTER the V1 binding is valid and BEFORE a ticket is + // issued. A refusal issues no ticket (the slot was invalidated above). No verdict (budget, + // incomplete facts, canceled) leaves the V1 answer: an ordinary ticket as before. + { + const sprint_coder::computer_use::InterlockEvaluation evaluation = + WindowsInterlockRefusal(session, metadata, kind); + if (evaluation.Refuses()) { + const std::string decision(evaluation.verdict.DecisionName()); + const std::string code(evaluation.verdict.ReasonCode()); + const std::string result(evaluation.verdict.DeniedResult()); + const bool sent = Respond( + pipe, request, MessageType::kPreflightResult, + sprint_coder::computer_use::NativePreflightRefusalAllowed(decision, code, result) + ? WindowsPreflightReceipt(session, request_id, cancel_epoch, observation_revision, + decision, code, "", result) + : WindowsPreflightReceipt(session, request_id, cancel_epoch, observation_revision, + "denied", "native_preflight_invalid", "")); + // The V1 answer is "no ticket": the shadow records that V1 did not issue its ordinary one. + ScheduleWindowsEffectShadow(session, metadata, kind, true, false); + return sent; + } + } std::string token; const bool issued = GenerateTicketToken(&token) && diff --git a/apps/desktop/src/main/computer-use-native-host.test.ts b/apps/desktop/src/main/computer-use-native-host.test.ts index 73e12fc12..39b4bbb8e 100644 --- a/apps/desktop/src/main/computer-use-native-host.test.ts +++ b/apps/desktop/src/main/computer-use-native-host.test.ts @@ -4,6 +4,7 @@ import type { ComputerUseNativeBinding } from './computer-use-native'; import { projectComputerUseAccessibilityTree } from './computer-use-accessibility-tree'; import { COMPUTER_USE_NATIVE_PICKER_UNAVAILABLE, + NATIVE_PREFLIGHT_REFUSALS, classifyComputerUseNativeDispatchFailure, createComputerUseNativeHost, parseNativeApprovalApproveReceipt, @@ -78,6 +79,62 @@ describe('Computer Use native Main adapter', () => { ).toBeNull(); }); + it('maps the three v2 interlock dispatch codes to the result of their preflight row (#500 S5-c-2 PR-5)', () => { + const refusal = (code: string, accepted: boolean | undefined = false) => + classifyComputerUseNativeDispatchFailure( + Object.assign(new Error('refused'), { + code, + ...(accepted === undefined ? {} : { accepted }), + }), + ); + // The result is the one of the same row of the receipt table: a block is rejected, an execution + // trigger is paused for a person. + expect(refusal('SHELL_SURFACE_BLOCKED')).toEqual({ + result: 'rejected', + reasonCode: 'native_shell_surface_blocked', + }); + expect(refusal('EXECUTION_TRIGGER_BLOCKED')).toEqual({ + result: 'paused', + reasonCode: 'native_execution_trigger_blocked', + }); + expect(refusal('APPROVAL_NOT_ENABLED')).toEqual({ + result: 'rejected', + reasonCode: 'native_approval_not_enabled', + }); + // The native spells the code in lower case: the same. + expect(refusal('execution_trigger_blocked')).toMatchObject({ result: 'paused' }); + // Every mapped code has a row in the table, with the same result (a table change cannot leave the + // mapping behind). + for (const code of [ + 'SHELL_SURFACE_BLOCKED', + 'EXECUTION_TRIGGER_BLOCKED', + 'APPROVAL_NOT_ENABLED', + ]) { + const mapped = refusal(code)!; + expect( + NATIVE_PREFLIGHT_REFUSALS.some( + (entry) => entry.reasonCode === mapped.reasonCode && entry.deniedResult === mapped.result, + ), + code, + ).toBe(true); + } + // The V1 codes keep their plain rejection (the mapping of V1 is unchanged). + for (const code of [ + 'HIGH_IMPACT_BLOCKED', + 'SECURE_FIELD_BLOCKED', + 'STALE_TARGET', + 'TARGET_UNCLASSIFIED', + ]) + expect(refusal(code)).toEqual({ + result: 'rejected', + reasonCode: `native_${code.toLowerCase()}`, + }); + // An interlock code that is not marked as refused before input is not taken as a pre-dispatch + // refusal unless the native says accepted:false. + expect(refusal('EXECUTION_TRIGGER_BLOCKED', true)).toBeNull(); + expect(refusal('EXECUTION_TRIGGER_BLOCKED', undefined)).toMatchObject({ result: 'paused' }); + }); + it('does not read node values: a production host offers no copy until T15-4m / T15-4w and C5', () => { for (const platform of ['darwin', 'win32'] as const) { const host = createComputerUseNativeHost(binding({ probe: () => ({}) }), platform); diff --git a/apps/desktop/src/main/computer-use-native-host.ts b/apps/desktop/src/main/computer-use-native-host.ts index 37bf90906..9ea0a5aa6 100644 --- a/apps/desktop/src/main/computer-use-native-host.ts +++ b/apps/desktop/src/main/computer-use-native-host.ts @@ -1458,6 +1458,18 @@ function parseNativeBounds(value: unknown): ComputerUseNativeBounds { }; } +/** + * The v2 interlock refusals a dispatch can still make on its own (#500 S5-c-2 PR-5): the native + * evaluates the interlocks again right before any input. Each is the dispatch code of a preflight + * refusal, so it takes the result of the same row of NATIVE_PREFLIGHT_REFUSALS: a block is rejected + * and an execution trigger is paused for a person. Shared with macOS (dispatchApproved). + */ +const NATIVE_DISPATCH_INTERLOCK_CODES: ReadonlySet = new Set([ + 'SHELL_SURFACE_BLOCKED', + 'EXECUTION_TRIGGER_BLOCKED', + 'APPROVAL_NOT_ENABLED', +]); + export function classifyComputerUseNativeDispatchFailure( error: unknown, ): ComputerUseNativeActionResult | null { @@ -1495,9 +1507,16 @@ export function classifyComputerUseNativeDispatchFailure( 'INVALID_KEY', 'INVALID_TEXT', 'INVALID_TARGET', + ...NATIVE_DISPATCH_INTERLOCK_CODES, ]); if (candidate.accepted !== false && !preDispatchCodes.has(code)) return null; if (!/^[A-Z][A-Z0-9_]{0,63}$/u.test(code)) return null; + if (NATIVE_DISPATCH_INTERLOCK_CODES.has(code)) { + const reasonCode = `native_${code.toLowerCase()}`; + const row = NATIVE_PREFLIGHT_REFUSALS.find((entry) => entry.reasonCode === reasonCode); + // The table owns the result; a code with no row stays a plain rejection. + return Object.freeze({ result: row?.deniedResult ?? 'rejected', reasonCode }); + } return Object.freeze({ result: 'rejected', reasonCode: `native_${code.toLowerCase()}`.slice(0, 64), diff --git a/apps/desktop/src/main/computer-use-windows-click-rule.test.ts b/apps/desktop/src/main/computer-use-windows-click-rule.test.ts index 6371a2fa2..cca17b196 100644 --- a/apps/desktop/src/main/computer-use-windows-click-rule.test.ts +++ b/apps/desktop/src/main/computer-use-windows-click-rule.test.ts @@ -177,7 +177,7 @@ describe('click rule (#500 S5-c-2 PR-2): the Windows host', () => { ); }); - it('adds no version and wires nothing new into preflight or dispatch', () => { + it('adds no version', () => { const eligibility = JSON.parse(read('eligibility.json')) as { versions: Record; }; @@ -186,7 +186,6 @@ describe('click rule (#500 S5-c-2 PR-2): the Windows host', () => { classifierVersion: 1, lexiconVersion: 1, }); - expect(host).not.toContain('EvaluateWindowsInterlockOnLane(session, metadata, kind'); }); }); diff --git a/apps/desktop/src/main/computer-use-windows-dispatch-interlock-seam.test.ts b/apps/desktop/src/main/computer-use-windows-dispatch-interlock-seam.test.ts new file mode 100644 index 000000000..c0ac78d41 --- /dev/null +++ b/apps/desktop/src/main/computer-use-windows-dispatch-interlock-seam.test.ts @@ -0,0 +1,218 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +// The dispatch's own interlock check (#500 S5-c-2 PR-5). The head of the REAL DispatchWindowsAction +// (from its first line to the point where V1's per-action work starts) is compiled against inert +// stand-ins for the binding check and the interlocks. What is shown: a refusal comes out with the +// verdict's dispatch code and accepted:false BEFORE anything that could send input (the stand-ins +// record every call, and the head has no input API to call: the source order is tied by +// computer-use-windows-interlock-capture.test.ts), no verdict lets the V1 guards run on, and a +// failed binding never asks the interlocks. + +const nativeDirectory = join(__dirname, '../../computer-use-native'); +const host = readFileSync(join(nativeDirectory, 'computer_use_windows_host.cc'), 'utf8'); +const vswhere = 'C:\\Program Files (x86)\\Microsoft Visual Studio\\Installer\\vswhere.exe'; + +type Build = Readonly<{ name: string; file: string; args: string[]; verbatim: boolean }>; + +function findBuild(path: string, output: string): Build | null { + if (process.platform !== 'win32') { + if (spawnSync('clang++', ['--version'], { stdio: 'ignore' }).status !== 0) return null; + return { + name: 'clang++', + file: 'clang++', + args: ['-std=c++20', '-Wall', '-Wextra', `-I${nativeDirectory}`, path, '-o', output], + verbatim: false, + }; + } + const clArguments = [ + '/nologo', + '/std:c++20', + '/EHsc', + '/W4', + '/permissive-', + '/utf-8', + `/I${nativeDirectory}`, + path, + `/Fe:${output}`, + `/Fo:${output}.obj`, + ]; + if (spawnSync('cl.exe', ['/?'], { stdio: 'ignore' }).status === 0) + return { name: 'cl.exe', file: 'cl.exe', args: clArguments, verbatim: false }; + if (!existsSync(vswhere)) return null; + const found = spawnSync( + vswhere, + [ + '-latest', + '-products', + '*', + '-requires', + 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', + '-property', + 'installationPath', + ], + { encoding: 'utf8' }, + ); + const root = found.stdout.trim(); + const vcvars = join(root, 'VC', 'Auxiliary', 'Build', 'vcvars64.bat'); + if (found.status !== 0 || root === '' || !existsSync(vcvars)) return null; + return { + name: 'cl.exe (vcvars64)', + file: 'cmd.exe', + args: [ + '/d', + '/s', + '/c', + `"call "${vcvars}" >nul && cl.exe ${clArguments.map((argument) => `"${argument}"`).join(' ')}"`, + ], + verbatim: true, + }; +} + +const headFrom = host.indexOf('bool DispatchWindowsAction('); +const headMarker = 'bool dispatched = false;'; +const headTo = host.indexOf(headMarker, headFrom); +const head = + headFrom >= 0 && headTo > headFrom ? host.slice(headFrom, headTo + headMarker.length) : ''; + +const program = String.raw` +#include +#include +#include +#include +#include + +#include "computer_use_native_interlock_capture.h" + +namespace { + +using sprint_coder::computer_use::InterlockEvaluation; +using sprint_coder::computer_use::InterlockNoVerdict; +using sprint_coder::computer_use::InterlockOutcome; +using sprint_coder::computer_use::InterlockReason; + +struct RECT { long left = 0; long top = 0; long right = 0; long bottom = 0; }; +struct WindowsSession { std::uint64_t cancel_epoch = 0; }; + +std::vector g_calls; +bool g_binding_ok = true; +InterlockEvaluation g_answer; + +bool ValidateWindowsDispatchBinding(const WindowsSession &, const std::string &, std::string *kind, + RECT *, std::string *reason) { + g_calls.push_back("validate"); + if (!g_binding_ok) { + *reason = "stale_target"; + return false; + } + *kind = "click"; + return true; +} + +InterlockEvaluation WindowsInterlockRefusal(const WindowsSession &, const std::string &, + const std::string &kind) noexcept { + g_calls.push_back("interlock:" + kind); + return g_answer; +} + +//@@HEAD@@ + (void)dispatched; + (void)expected_epoch; + (void)bounds; + g_calls.push_back("v1-guards"); + return true; +} + +int g_failures = 0; +void Check(bool ok, const char *name) { + if (!ok) { + ++g_failures; + std::cout << "FAIL " << name << "\n"; + } +} + +struct Outcome { bool ran_on; bool accepted; std::string reason; std::vector calls; }; +Outcome Run() { + g_calls.clear(); + WindowsSession session; + std::string result; + std::string reason; + bool accepted = true; + const bool ran_on = DispatchWindowsAction(session, "{}", &result, &reason, &accepted); + return {ran_on, accepted, reason, g_calls}; +} + +} // namespace + +int main() { + struct Row { InterlockOutcome outcome; InterlockReason reason; const char *code; }; + const Row rows[] = { + {InterlockOutcome::kBlocked, InterlockReason::kShellSurface, "shell_surface_blocked"}, + {InterlockOutcome::kTakeover, InterlockReason::kExecutionTrigger, "execution_trigger_blocked"}, + {InterlockOutcome::kBlocked, InterlockReason::kApprovalClosed, "approval_not_enabled"}, + }; + for (const Row &row : rows) { + g_binding_ok = true; + g_answer = InterlockEvaluation{}; + g_answer.verdict.outcome = row.outcome; + g_answer.verdict.reason = row.reason; + const Outcome outcome = Run(); + Check(!outcome.ran_on && !outcome.accepted && outcome.reason == row.code, "refusal_code_and_not_accepted"); + // The binding first, the interlocks second, and nothing after a refusal (no V1 guards, no input). + Check(outcome.calls == std::vector({"validate", "interlock:click"}), "refusal_order_and_nothing_after"); + } + for (const InterlockNoVerdict none : {InterlockNoVerdict::kBudget, InterlockNoVerdict::kIncomplete, InterlockNoVerdict::kCanceled}) { + g_answer = InterlockEvaluation{}; + g_answer.no_verdict = none; + const Outcome outcome = Run(); + Check(outcome.ran_on && outcome.reason.empty(), "no_verdict_goes_on_to_v1"); + Check(outcome.calls == std::vector({"validate", "interlock:click", "v1-guards"}), "no_verdict_order"); + } + // A failed binding is V1's own refusal and the interlocks are never asked. + g_binding_ok = false; + g_answer = InterlockEvaluation{}; + g_answer.verdict.outcome = InterlockOutcome::kBlocked; + g_answer.verdict.reason = InterlockReason::kShellSurface; + { + const Outcome outcome = Run(); + Check(!outcome.ran_on && outcome.reason == "stale_target" && outcome.calls == std::vector({"validate"}), + "failed_binding_asks_nothing"); + } + std::cout << (g_failures == 0 ? "OK\n" : "FAILED\n"); + return g_failures == 0 ? 0 : 1; +} +`; + +const scratch = mkdtempSync(join(tmpdir(), 'computer-use-dispatch-interlock-')); +const source = join(scratch, 'seam.cc'); +const binary = join(scratch, process.platform === 'win32' ? 'seam.exe' : 'seam'); +const build = findBuild(source, binary); + +describe.skipIf(build === null && process.env['CI'] === undefined)( + 'dispatch interlock check, the real head of DispatchWindowsAction (#500 S5-c-2 PR-5)', + () => { + it('refuses with the verdict code before anything else, goes on without a verdict, and asks nothing when the binding fails', () => { + try { + expect(build, 'a C++ compiler is required to run this seam in CI').not.toBeNull(); + expect(head.length).toBeGreaterThan(200); + writeFileSync(source, program.replace('//@@HEAD@@', head)); + const compiler = build!; + const built = spawnSync(compiler.file, compiler.args, { + encoding: 'utf8', + windowsVerbatimArguments: compiler.verbatim, + timeout: 120_000, + env: { ...process.env, VSLANG: '1033' }, + }); + expect(built.status, `${compiler.name}\n${built.stdout}\n${built.stderr}`).toBe(0); + const run = spawnSync(binary, [], { encoding: 'utf8', timeout: 20_000 }); + expect(run.status, `${run.stdout}\n${run.stderr}`).toBe(0); + expect(run.stdout).toContain('OK'); + } finally { + rmSync(scratch, { recursive: true, force: true }); + } + }, 180_000); + }, +); diff --git a/apps/desktop/src/main/computer-use-windows-interlock-capture.test.ts b/apps/desktop/src/main/computer-use-windows-interlock-capture.test.ts index f42d5ffdc..fbd27d4bc 100644 --- a/apps/desktop/src/main/computer-use-windows-interlock-capture.test.ts +++ b/apps/desktop/src/main/computer-use-windows-interlock-capture.test.ts @@ -76,18 +76,122 @@ describe('Windows synchronous interlock entry (classifier v2, S5-c-1)', () => { expect(squashed).toContain('reader.Canceled() || !source.ActionStillValid()'); }); - it('is defined once in the host and referenced nowhere else (S5-c-2 wires it)', () => { + it('is defined once in the host and asked from exactly two places: the preflight and the dispatch (S5-c-2 PR-5)', () => { expect(entry.length).toBeGreaterThan(200); expect(host).toContain('#include "computer_use_windows_interlock_capture.h"'); - expect(host.match(/\bEvaluateWindowsInterlockOnLane\(/gu)).toHaveLength(1); - // THIS EXPECTATION IS REMOVED IN S5-c-2, which is the producer. - expect(outside).not.toMatch( - /EvaluateWindowsInterlockOnLane|EvaluateWindowsInterlockOnSource|EvaluateWindowsInterlockFromCapture|InterlockEvaluation|InterlockNoVerdict|InterlockVerdict/u, + // The lane's entry is defined once and used by the one wrapper; the wrapper by the two callers. + expect(host.match(/\bEvaluateWindowsInterlockOnLane\(/gu)).toHaveLength(2); + const calls = [...code(outside).matchAll(/\bWindowsInterlockRefusal\(/gu)]; + // the forward declaration and the two calls (the definition is inside the entry) + expect(calls).toHaveLength(3); + const preflightAt = outside.indexOf('bool HandleWindowsPreflight('); + const dispatchAt = outside.indexOf('bool DispatchWindowsAction('); + expect(preflightAt).toBeGreaterThan(0); + expect(dispatchAt).toBeGreaterThan(0); + const preflightText = code( + outside.slice(preflightAt, outside.indexOf('\nbool ObserveWindowsWindow(')), ); - // No refusal can come out of the host yet: the adapter does not name the codes or the receipt. - expect(code(entry)).not.toMatch( - /native_\w+_blocked|native_approval_not_enabled|Receipt|decision|reasonCode/u, + const dispatchText = code( + outside.slice(dispatchAt, outside.indexOf('BackendProbe ProbeBackend(')), ); + expect(preflightText.match(/\bWindowsInterlockRefusal\(/gu)).toHaveLength(1); + expect(dispatchText.match(/\bWindowsInterlockRefusal\(/gu)).toHaveLength(1); + // No other message path asks: observe, start_session, list_targets, cancel, close. + const rest = outside + .replace(outside.slice(preflightAt, outside.indexOf('\nbool ObserveWindowsWindow(')), '') + .replace(outside.slice(dispatchAt, outside.indexOf('BackendProbe ProbeBackend(')), ''); + expect(code(rest).match(/\bWindowsInterlockRefusal\(/gu)).toHaveLength(1); // the declaration + expect(code(rest)).not.toMatch( + /EvaluateWindowsInterlockOnSource|EvaluateWindowsInterlockFromCapture/u, + ); + }); + + it('asks in the preflight after the V1 binding and before a ticket is issued; in the dispatch after the binding and before any input', () => { + const preflightAt = outside.indexOf('bool HandleWindowsPreflight('); + const preflight = code( + outside.slice(preflightAt, outside.indexOf('\nbool ObserveWindowsWindow(')), + ); + const validate = preflight.indexOf('ValidateWindowsDispatchBinding('); + const ask = preflight.indexOf('WindowsInterlockRefusal('); + const issue = preflight.indexOf('state.slot.Issue('); + expect(validate).toBeGreaterThan(0); + expect(ask).toBeGreaterThan(validate); + expect(issue).toBeGreaterThan(ask); + // The refusal receipt goes through the closed table before it is sent; a pair outside it is denied. + const rawPreflight = outside.slice( + preflightAt, + outside.indexOf('\nbool ObserveWindowsWindow('), + ); + const refusal = rawPreflight + .slice( + rawPreflight.indexOf('WindowsInterlockRefusal('), + rawPreflight.indexOf('state.slot.Issue('), + ) + .replace(/\s+/gu, ' '); + expect(refusal).toContain('NativePreflightRefusalAllowed(decision, code, result)'); + expect(refusal).toContain('"denied", "native_preflight_invalid"'); + expect(refusal).toContain('evaluation.verdict.DecisionName()'); + expect(refusal).toContain('evaluation.verdict.ReasonCode()'); + expect(refusal).toContain('evaluation.verdict.DeniedResult()'); + // The refusal issues no ticket and returns before the issuing code. + expect(refusal).toContain('return sent;'); + + const dispatchAt = outside.indexOf('bool DispatchWindowsAction('); + const dispatch = code(outside.slice(dispatchAt, outside.indexOf('BackendProbe ProbeBackend('))); + const dValidate = dispatch.indexOf('ValidateWindowsDispatchBinding('); + const dAsk = dispatch.indexOf('WindowsInterlockRefusal('); + expect(dValidate).toBeGreaterThan(0); + expect(dAsk).toBeGreaterThan(dValidate); + // Before the first input attempt, the cursor, any input API and the semantic action. + for (const input of [ + 'input_api_attempts', + 'SetCursorPos', + 'SendInput', + 'SendInputForBoundTarget', + 'DispatchWindowsSemanticAction', + ]) { + const at = dispatch.indexOf(input); + expect(at, input).toBeGreaterThan(dAsk); + } + // A refusal sets the dispatch code of the verdict and refuses without input. + const block = dispatch + .slice(dAsk, dispatch.indexOf('bool dispatched = false;')) + .replace(/\s+/gu, ' '); + expect(block).toContain('*reason = std::string(evaluation.verdict.DispatchCode());'); + expect(block).toContain('return false;'); + }); + + it('uses the Windows rule set with no approval path, and counts the cases that have no verdict', () => { + const body = entry.replace(/\s+/gu, ' '); + expect(body).toContain( + 'constexpr sprint_coder::computer_use::EffectRuleset kWindowsInterlockRuleset{ /*download_open=*/false, /*file_manager_open=*/false};', + ); + expect(body).toContain( + 'EvaluateWindowsInterlockOnLane(session, metadata, kind, kWindowsInterlockRuleset)', + ); + // Counts and enum names only: no label, no title, no text. + const counter = entry.slice( + entry.indexOf('void CountWindowsInterlockNoVerdict('), + entry.indexOf('WindowsInterlockRefusal(const'), + ); + expect(counter).toContain('OutputDebugStringA(line)'); + expect(counter).not.toMatch(/label|title|name\s*=\s*node|metadata|text/iu); + }); + + it('leaves INV-3 alone: the eligibility, the versions and the supported targets are what they were (#500 S5-c-2 PR-5)', () => { + expect(host).toContain('IsSupportedWindowsV1Target'); + expect(host).toContain('MaximumModeForWindowsExecutable'); + expect(host).toContain('kNativePreflightClassifierVersion'); + const eligibility = JSON.parse(read('eligibility.json')) as { + versions: Record; + }; + expect(eligibility.versions).toEqual({ + rulesetVersion: 1, + classifierVersion: 1, + lexiconVersion: 1, + }); + // The target eligibility functions are not touched by this change: they appear as before. + expect(host.match(/\bIsSupportedWindowsV1Target\(/gu)?.length).toBeGreaterThan(0); }); it('takes the rule set from its caller, answers an exception or a missing source as no verdict, and changes nothing', () => { diff --git a/apps/desktop/src/main/computer-use-windows-interlock-vocabulary.test.ts b/apps/desktop/src/main/computer-use-windows-interlock-vocabulary.test.ts new file mode 100644 index 000000000..f8504e88f --- /dev/null +++ b/apps/desktop/src/main/computer-use-windows-interlock-vocabulary.test.ts @@ -0,0 +1,83 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +// The words that make the v2 interlocks refuse a command surface, against the words V1's own risk +// check already refuses (#500 S5-c-2 PR-5, plan section 3-2). The interlock is wired into the V1 +// path, so a word that only the interlock knows is a refusal V1 users get NEWLY. The owner accepted +// exactly the list below (terminals and consoles in Japanese, xterm, running a script, and the full +// width spellings the classifier folds to ASCII). The list is frozen: a word added to the classifier's +// table without a decision turns this test red. + +const nativeDirectory = join(__dirname, '../../computer-use-native'); +const read = (name: string): string => readFileSync(join(nativeDirectory, name), 'utf8'); +const host = read('computer_use_windows_host.cc'); +const classifier = read('computer_use_effect_classifier.h'); + +function quotedTerms(text: string, from: string, prefix: string): string[] { + const start = text.indexOf(from); + expect(start, from).toBeGreaterThanOrEqual(0); + const end = text.indexOf('};', start); + const body = text.slice(start, end); + return [...body.matchAll(new RegExp(`${prefix}"([^"]+)"`, 'gu'))].map((match) => match[1]!); +} + +// V1: windows_host.cc, the shell words of the UI Automation risk text. +const v1 = quotedTerms(host, 'static constexpr std::array shell_terms', ''); +// The classifier: kSurfaceTerms. +const v2 = quotedTerms(classifier, 'inline constexpr std::u8string_view kSurfaceTerms[]', 'u8'); + +// What only the interlock refuses. Accepted by the owner (#500). "shell script" and "script console" +// contain a V1 word on a word boundary ("shell", "console") and were refused already; they are +// listed because they are in the table. The full width forms are folded to ASCII before matching +// (NFKC), so the same words in full width are refused as well; that is the spelling difference the +// owner accepted. +const ACCEPTED_NEW_WORDS = [ + 'xterm', + 'run script', + 'shell script', + 'script console', + 'ターミナル', + 'コンソール', + 'コマンドプロンプト', + 'スクリプトを実行', +]; +// What only V1 refuses (the interlock does not, and V1 still does at the dispatch). +const V1_ONLY_WORDS = ['pwsh', 'wsl']; + +describe('interlock surface words against V1 words (#500 S5-c-2 PR-5)', () => { + it('reads both tables', () => { + expect(v1.length).toBe(9); + expect(v2.length).toBeGreaterThanOrEqual(15); + }); + + it('adds exactly the words the owner accepted to what V1 refuses', () => { + const added = v2.filter((word) => !v1.includes(word)); + expect([...added].sort()).toEqual([...ACCEPTED_NEW_WORDS].sort()); + }); + + it('keeps every V1 word that the interlock does not know a V1 refusal (nothing is lost)', () => { + const onlyV1 = v1.filter((word) => !v2.includes(word)); + expect([...onlyV1].sort()).toEqual([...V1_ONLY_WORDS].sort()); + // They are still refused by V1's own guard at the dispatch, which is unchanged. + for (const word of onlyV1) expect(host).toContain(`"${word}"`); + }); + + it('shares every other word', () => { + for (const word of v1.filter((item) => v2.includes(item))) { + expect(v2).toContain(word); + expect(v1).toContain(word); + } + expect(v1.filter((word) => v2.includes(word)).sort()).toEqual( + [ + 'cmd.exe', + 'command prompt', + 'console', + 'integrated terminal', + 'powershell', + 'shell', + 'terminal', + ].sort(), + ); + }); +}); diff --git a/apps/desktop/src/main/computer-use-windows-shadow-wiring.test.ts b/apps/desktop/src/main/computer-use-windows-shadow-wiring.test.ts index 39213727a..cc0417cd3 100644 --- a/apps/desktop/src/main/computer-use-windows-shadow-wiring.test.ts +++ b/apps/desktop/src/main/computer-use-windows-shadow-wiring.test.ts @@ -188,10 +188,10 @@ describe('Windows host calls the shadow only after V1, and only to watch', () => it('calls ScheduleWindowsEffectShadow from HandleWindowsPreflight only: once without V1, once after the receipt is out', () => { const calls = [...code(host).matchAll(/ScheduleWindowsEffectShadow\(/gu)]; - // the definition and the two calls - expect(calls.length).toBe(3); + // the definition and the three calls (no V1 binding, an interlock refusal, the V1 answer) + expect(calls.length).toBe(4); const inPreflight = [...code(preflight).matchAll(/ScheduleWindowsEffectShadow\(/gu)]; - expect(inPreflight.length).toBe(2); + expect(inPreflight.length).toBe(3); const text = code(preflight); const validate = text.indexOf('ValidateWindowsDispatchBinding('); const deniedSent = text.indexOf('const bool denied_sent ='); @@ -307,7 +307,8 @@ describe('Windows host calls the shadow only after V1, and only to watch', () => expect(host).toContain('kNativePreflightClassifierVersion'); // The receipt constructor and its two calls still build the V1 answer from V1 inputs only. const text = code(preflight); - expect((text.match(/WindowsPreflightReceipt\(/gu) ?? []).length).toBe(3); + // replay, a failed binding, the two spellings of an interlock refusal, the V1 answer + expect((text.match(/WindowsPreflightReceipt\(/gu) ?? []).length).toBe(5); expect(text).toContain('issued ? "ordinary" : "denied"'); expect(text).not.toMatch(/ProcessShadowLog|Snapshot\(|Total\(\)/u); }); diff --git a/apps/desktop/src/main/computer-use-windows-ticket-seam.test.ts b/apps/desktop/src/main/computer-use-windows-ticket-seam.test.ts index ea245e4eb..38aed0592 100644 --- a/apps/desktop/src/main/computer-use-windows-ticket-seam.test.ts +++ b/apps/desktop/src/main/computer-use-windows-ticket-seam.test.ts @@ -126,6 +126,8 @@ const preamble = String.raw` #include #include #include +#include "computer_use_native_interlock_capture.h" +#include "computer_use_native_refusal_table.h" #include "computer_use_native_ticket.h" #include "computer_use_preflight_classifier.h" #include "computer_use_protocol.h" @@ -292,6 +294,8 @@ void Reset() { response_cache_bytes = 0; g_sent.clear(); g_shadow.clear(); + g_interlock_kinds.clear(); + g_interlock_answer = InterlockEvaluation{}; g_validate_ok = true; g_revalidate_ok = true; g_focus_ok = true; @@ -617,6 +621,87 @@ int main() { "changed_payload_is_a_conflict"); } + // 13. The v2 interlocks (#500 S5-c-2 PR-5): asked after the V1 binding is valid and before a ticket + // is issued; a refusal issues no ticket and the shadow still sees the request as not ordinary. + struct Scripted { InterlockOutcome outcome; InterlockReason reason; const char *decision; const char *code; const char *result; }; + const Scripted scripted[] = { + {InterlockOutcome::kBlocked, InterlockReason::kShellSurface, "blocked", "native_shell_surface_blocked", "rejected"}, + {InterlockOutcome::kTakeover, InterlockReason::kExecutionTrigger, "takeover", "native_execution_trigger_blocked", "paused"}, + {InterlockOutcome::kBlocked, InterlockReason::kApprovalClosed, "blocked", "native_approval_not_enabled", "rejected"}, + }; + for (const Scripted &row : scripted) { + Reset(); + g_interlock_answer.verdict.outcome = row.outcome; + g_interlock_answer.verdict.reason = row.reason; + ticket = Preflight(Meta()); + const std::string &receipt = g_sent.back().payload; + Check(ticket.empty() && HasDecision(row.decision) && + receipt.find(std::string("\"reasonCode\":\"") + row.code + "\"") != std::string::npos && + receipt.find(std::string("\"deniedResult\":\"") + row.result + "\"") != std::string::npos && + receipt.find("\"ticket\":null") != std::string::npos, + "interlock_refusal_receipt"); + Check(!ticket_states.at(kSessionKey).slot.occupied(), "interlock_refusal_issues_no_ticket"); + Check(g_interlock_kinds.size() == 1 && g_interlock_kinds[0] == "click", "interlock_asked_once_with_the_kind"); + Check(g_shadow.size() == 1 && g_shadow[0].v1_valid && !g_shadow[0].v1_ordinary, + "interlock_refusal_shadow_sees_not_ordinary"); + // There is no ticket to use: a dispatch with a made-up one is refused. + Check(!Consume(Meta("", std::string(64, 'a'))), "interlock_refusal_leaves_no_ticket_to_consume"); + // Every refusal this sends is a row of the closed table. + Check(sprint_coder::computer_use::NativePreflightRefusalAllowed(row.decision, row.code, row.result), + "interlock_refusal_is_a_table_row"); + } + // No verdict (budget, incomplete, canceled): the V1 answer, an ordinary ticket as before. + for (const InterlockNoVerdict none : {InterlockNoVerdict::kBudget, InterlockNoVerdict::kIncomplete, InterlockNoVerdict::kCanceled}) { + Reset(); + g_interlock_answer.no_verdict = none; + ticket = Preflight(Meta()); + Check(ticket.size() == 64 && HasDecision("ordinary"), "no_verdict_keeps_the_ordinary_ticket"); + Check(g_interlock_kinds.size() == 1, "no_verdict_was_asked"); + Check(g_shadow.size() == 1 && g_shadow[0].v1_ordinary, "no_verdict_shadow_sees_ordinary"); + } + // A verdict that is not a row of the table is never sent as a refusal: denied, fail closed. + Reset(); + g_interlock_answer.verdict.outcome = InterlockOutcome::kTakeover; + g_interlock_answer.verdict.reason = InterlockReason::kShellSurface; + ticket = Preflight(Meta()); + Check(ticket.empty() && HasDecision("denied") && + g_sent.back().payload.find("native_preflight_invalid") != std::string::npos && + !ticket_states.at(kSessionKey).slot.occupied(), + "interlock_pair_outside_the_table_is_denied"); + // A refusal does not outlive the next preflight: a later no-verdict preflight issues again. + Reset(); + g_interlock_answer.verdict.outcome = InterlockOutcome::kBlocked; + g_interlock_answer.verdict.reason = InterlockReason::kShellSurface; + Check(Preflight(Meta()).empty(), "refused_first"); + g_interlock_answer = InterlockEvaluation{}; + ticket = Preflight(Meta()); + Check(ticket.size() == 64 && Consume(Meta("", ticket)), "ordinary_after_a_refusal"); + // The interlocks are not asked when V1 does not get as far: a failed binding, a replay, a missing + // session, an invalid action. + Reset(); + g_validate_ok = false; + Check(Preflight(Meta()).empty() && HasDecision("denied") && g_interlock_kinds.empty(), "failed_binding_asks_nothing"); + Reset(); + { + Frame finished = PipeFrame(MessageType::kDispatch, 0); + sprint_coder::computer_use::FrameId derived; + Check(DispatchFrameIdFor("request-1", &derived), "interlock_replay_id"); + finished.header.request_id = derived; + response_cache.emplace(ResponseCacheKey(finished), CachedResponse{MessageType::kDispatchResult, "digest", "{}", {}}); + Preflight(Meta()); + Check(HasDecision("replay") && g_interlock_kinds.empty(), "replay_asks_nothing"); + } + Reset(); + { + Frame unknown = PipeFrame(MessageType::kPreflight, 9); + unknown.header.session_id = IdOf(77); + g_sent.clear(); + HandleWindowsPreflight(nullptr, unknown, Meta()); + Check(g_interlock_kinds.empty(), "missing_session_asks_nothing"); + } + Reset(); + Check(Preflight("{\"requestId\":\"request-1\"}").empty() && g_interlock_kinds.empty(), "invalid_action_asks_nothing"); + if (g_failures != 0) return 1; std::cout << "OK\n"; return 0; @@ -632,6 +717,20 @@ void ScheduleWindowsEffectShadow(const WindowsSession &, const std::string &, co bool v1_valid, bool v1_ordinary) noexcept { g_shadow.push_back({kind, v1_valid, v1_ordinary}); } + +// The v2 interlocks (#500 S5-c-2 PR-5) have their own seams; here they answer from a script, and +// every call is recorded, so the preflight's use of the answer is what is checked. +using sprint_coder::computer_use::InterlockEvaluation; +using sprint_coder::computer_use::InterlockNoVerdict; +using sprint_coder::computer_use::InterlockOutcome; +using sprint_coder::computer_use::InterlockReason; +std::vector g_interlock_kinds; +InterlockEvaluation g_interlock_answer; +InterlockEvaluation WindowsInterlockRefusal(const WindowsSession &, const std::string &, + const std::string &kind) noexcept { + g_interlock_kinds.push_back(kind); + return g_interlock_answer; +} `; // The shadow wiring has its own seam (a real run against UI Automation); it is cut out here and diff --git a/tasks/designs/adr-computer-use-agent-driven-v2.md b/tasks/designs/adr-computer-use-agent-driven-v2.md index ad55436b2..3a746e925 100644 --- a/tasks/designs/adr-computer-use-agent-driven-v2.md +++ b/tasks/designs/adr-computer-use-agent-driven-v2.md @@ -664,6 +664,14 @@ computer_start がここで返る → { sessionId, state, stopReason, mode, roun **S5-c-2 PR-4（集計のみ）**: `ShadowRecord::V2IsStricter` の集計でネイティブの Document（Notepad の本文など、web ウィンドウではない Document で origin が unknown になった記録）を別に扱う。`BuildWindowsShadowFacts` がその記録に `kShadowGapNativeDocument`（1u << 15）を付け、`V2IsStricter()` から除いて `NativeDocumentGap()` で数える。facts と判定は変えない。 +**S5-c-2 PR-5（拒否の有効化。2026-10-17）**: Windows の preflight と dispatch から同期の入口を呼び、S4-1a の受領書で拒否する。S5-c の (1)〜(6) は次のとおり解決済みである。 +(1) UIA のタイムアウトは同期の入口で 250 ms、レーンの保持の上限は 500 ms（最悪の見積りは PR-1 の節のとおり。preflight と dispatch の 2 回で 1 操作あたり最大約 1 s の遅れ）。(2) Explorer の lookup は 150 ms の待ちで、実パスが読めないファイル項目は判定なし（kIncomplete）のまま。(3) lookup の枠は shadow と同期評価で分けた。(4) MTA は評価ごとのワーカーで実現し、主スレッドは picker と WinRT の捕捉のために STA に残す。(5) 座標写像は `WindowsActionPoint` の 1 つ。(6) 開始前の取消しは UIA を作らずに kCanceled。 +**配線**: `WindowsInterlockRefusal`（同期の入口の区間にある小さな関数。seam から差し替えられる）が、評価と、判定なしの理由ごとの件数（`OutputDebugStringA`。enum 名と件数だけでラベルは出さない）を受け持つ。ruleset は `kWindowsInterlockRuleset`（承認経路なし。win32 の v2 build は `build-computer-use-eligibility.mjs` で拒否されるので、C4-2c-2 (7) の門は Windows では自明に閉じている。Windows に route v2 は無い）。`HandleWindowsPreflight` は V1 の binding の検証が成功した後、ticket を出す前に評価し、拒否なら ticket を出さずに `blocked` / `takeover` の受領書（`DecisionName` / `ReasonCode` / `DeniedResult`）を返す。送る前に `NativePreflightRefusalAllowed` を通し、表に無い組は `denied` / `native_preflight_invalid`（fail closed）。判定なしなら従来どおり V1 の ordinary ticket。shadow は従来どおり呼ぶ（拒否のとき `issued=false`）。`DispatchWindowsAction` は binding の検証の直後、最初の入力 API の試行（`input_api_attempts`、`SetCursorPos`、`SendInput`、`DispatchWindowsSemanticAction`）より前にもう一度評価し、拒否なら `DispatchCode()`（`shell_surface_blocked` / `execution_trigger_blocked` / `approval_not_enabled`）と `accepted:false` で入力 0 回で返す。その後の V1 の guard と再検証は同じ順で残す。Main の `classifyComputerUseNativeDispatchFailure` は、この 3 つのコードを `NATIVE_PREFLIGHT_REFUSALS` の同じ行の result に写す（実行トリガは `paused`、それ以外は `rejected`）。V1 のコードは `rejected` のまま。この写し方は macOS の `dispatchApproved` と共有する。 +**判定なし（fail-open）になる場合**: 取消し、予算切れ（応答しない対象、止まったワーカーが残っている間の single-flight を含む）、不完全な facts、`Create` の失敗、例外、点が無い click / scroll、lookup の放棄 / 曖昧 / 仮想フォルダ / 枠の使用中。S4-1 の方針「正の一致だけ拒否」のとおりで、V1 に任せる。V1 から届く面（メモ帳と fixture）には Explorer が無いので、今は表に出ない。 +**V1 で新しく拒否が増える語（所有者が受け入れた）**: classifier の `kSurfaceTerms` にあって V1 の `shell_terms` に無い語。`xterm`、`run script`、`ターミナル`、`コンソール`、`コマンドプロンプト`、`スクリプトを実行`（`shell script` と `script console` は V1 の語を含むので変わらない）と、全角表記（classifier は NFKC で ASCII に畳む）。英語の同じ語は V1 でもすでに拒否しているので、言語による差を無くす方向である。逆に V1 だけが拒否する語（`pwsh`、`wsl`）は V1 の guard が引き続き拒否する。差分の一覧は `computer-use-windows-interlock-vocabulary.test.ts` で固定し、語が増えたら赤になる。fixture とメモ帳（旧・新）の UIA の木は harness で全 action と en / ja の題名に通し、拒否が V1 の語か上の一覧の語に限られることを固定した。 +**既知の限界**: (a) preflight から dispatch までの TOCTOU（ticket の有効期間は 5 s）は、dispatch の直前の再評価（click の事実である最後の入力の時刻も評価のたびに読み直す）と、その後の V1 の再検証で狭める。評価から入力までの短い時間差は構造上残る。(b) 二重クリックの判定は、システムの最後の入力の時刻（`GetLastInputInfo`）だけで行い、人の入力と agent の入力を区別しない。人が操作している最中（二重クリックの時間内）の agent の click は「開く」側になるが、拒否されるのは実行トリガの項目だけである。拒否の理由に「人の入力と重なった可能性」を出すことは今回しない（受領書の表に行を足さない）。必要になったら別の判断とする。(c) 評価の時間が上限に達したときの件数は `OutputDebugStringA` に出る。 +**INV-3**: S5-c-2 は適格性を広げない（`eligibility.json`、各版、attest、`list_targets` の適格性、`IsSupportedWindowsV1Target`、`MaximumModeForWindowsExecutable` は動かさない。source 契約で固定した）。S5-c-3（Windows で適格性を広げる PR）は次を受入条件にする: 不完全な facts、表示名の曖昧さ、lookup の放棄を takeover に倒すこと。interlock が有効であることを attest すること。 + **C5 の分割と初回の有効化範囲（2026-10-16。所有者の判断）**: C5 は 2 つの PR に分ける。**C5-1（有効化の準備。本番では不活性）**: (1) 「承認経路が有効」の述語 `computerUseApprovalPathEnabled`（`computer-use-eligibility.ts`）。macOS、addon が承認の面を export、handshake が classifier 2 を attest、の 3 つに加えて、**session の mode と route**を見る: session は route v2 かつ実効 mode が `v2-denylist` に束縛されているときだけ開く。route v1 の session には、build が何を attest しても承認は無い。**束縛の無い session（route 無しで開始、verified close の後、開始に失敗した後）は閉じる**（fail closed。C3 以前の「束縛が無ければ attest だけで決まる」規則は廃止した）。述語は session について問うだけで、session 無しの問い合わせは無い（host と controller の引数は必須）。host（`approvalPathEnabled(sessionId)`）と controller（`nativeApprovalPathOpen(sessionId)`）は同じ述語を session 単位で使う。(2) `classifierV2Attested` の本番配線 `computerUseClassifierV2Attested`: Main にコンパイルされた eligibility が `v2-denylist` かつ `classifierVersion` = 2 で、gate が開いているときだけ true（v2 の Main は manifest と attest が自分の宣言と同一でなければ gate が閉じるので、gate が開いていれば native がその版を attest している）。環境変数もフラグも読まない。(3) `effectiveEligibilityMode` の本番配線: `index.ts`（host）と `ipc.ts`（controller）に `computerUseEligibilityWiring` を渡す。**版が 1 のままなので、どの build でも attest は 1、`classifierV2Attested` は false、host が native に渡す route は常に v1 である**（v2 の build と V2 フラグで agent セッションが route v2 を求めても、host は attest がなければ v1 を渡す）。したがって C4-2c-2 の ADR (7) の門は、Main 側では今回の配線の後も閉じている。**C5 より前に route v2 を本番で配線しない**前提は、「版を 2 に上げる PR（C5-2）だけが route v2 を native に届かせる」と読み替える。(4) 観測 metadata の高影響: `highImpact`（native の facts と Main の投影の和）と `highImpactNative`（native の facts だけ）は既にあり、`splitHighImpact` が「Main の投影だけが見た分」を明示する。**3-A**: Main の投影だけが見た高影響は、判定を強める（止める）方向にだけ使い、承認経路が開いていても緩めない（緩められるのは `highImpactNative` の分だけ）。**C5-2（有効化。#777 の実測の後）**: 版を 2 に上げる（`eligibility.json` を mode ごとの版表にする）、レーン上の承認測定の時間（1 回の AX が最大約 6 秒、Cancel ack 1 秒）を #777 で実測して `AXUIElementSetMessagingTimeout` の採否かレーン保持時間の上限を決める。**初回の有効化範囲は 2-A**: 承認可能にするのは `high_impact_confirmation` と `supervised_action` だけで、E2 / E3 は C6 まで拒否のままにする。**`eligibility.json` を mode ごとの版表にする構造変更は C5-2 に回す**: 版表の消費者（ビルドスクリプト、生成ヘッダ、約 12 本の契約テスト）が多く、C5-1 には読み手がいない。また表の形（mode ごとに ruleset / classifier / lexicon を分けるか、classifier だけか）は C5-2 で版を上げるときにしか決まらない。 **S6 の前提（2026-10-14）**: UWP / `ApplicationFrameHost` の identity と束縛（UIA によるコンテンツ HWND の解決、package family name と publisher の検証、native identity digest への束縛、package に限るパス比較の免除、プロセス入れ替わりの検出）は、設計を ADR に追記して敵対的レビューを受けるまで実装しない。レビューが出るまでは、`ApplicationFrameHost` 配下は従来どおり拒否する。Windows 11 の既定 Notepad のような packaged app が `ApplicationFrameHost` 配下か、packaged Win32 かは、プロセスツリーとウィンドウクラスの実測で確かめてから S6 の必要性を判断する。 From b604ccb687a7f6a58a7488e028c272e93ec29fe5 Mon Sep 17 00:00:00 2001 From: yusei Date: Sat, 17 Oct 2026 11:34:38 +0900 Subject: [PATCH 2/5] =?UTF-8?q?fix(computer-use):=20#500=20S5-c-2=20PR-5?= =?UTF-8?q?=20=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC=E6=8C=87=E6=91=98:=20?= =?UTF-8?q?=E3=83=95=E3=82=A9=E3=83=BC=E3=82=AB=E3=82=B9=E3=81=AE=E5=86=8D?= =?UTF-8?q?=E7=85=A7=E5=90=88=E3=80=81=E8=A9=95=E4=BE=A1=E5=BE=8C=E3=81=AE?= =?UTF-8?q?=E5=85=A5=E5=8A=9B=E3=81=A7=20click=20=E3=82=92=E6=AD=A2?= =?UTF-8?q?=E3=82=81=E3=82=8B=E3=80=81=E3=82=A4=E3=83=B3=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=83=BC=E3=83=A9=E3=83=BC=E8=AA=9E=E3=81=AE=E5=87=8D=E7=B5=90?= =?UTF-8?q?=E3=80=81fixture=20=E3=81=AE=E6=9C=A8=E3=81=AE=E7=85=A7?= =?UTF-8?q?=E5=90=88=E3=80=81INV-3=20=E3=81=AE=E6=9C=AC=E4=BD=93=E5=9B=BA?= =?UTF-8?q?=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- .../computer_use_protocol_fuzz.cc | 91 +++++++++--- .../computer_use_windows_host.cc | 88 ++++++++++-- .../computer-use-windows-click-rule.test.ts | 72 +++++++++- ...se-windows-dispatch-interlock-seam.test.ts | 67 ++++++++- ...uter-use-windows-interlock-capture.test.ts | 87 ++++++++++- ...ter-use-windows-interlock-journeys.test.ts | 135 ++++++++++++++++++ ...er-use-windows-interlock-live-seam.test.ts | 10 ++ ...r-use-windows-interlock-vocabulary.test.ts | 30 ++++ .../adr-computer-use-agent-driven-v2.md | 4 +- 9 files changed, 539 insertions(+), 45 deletions(-) create mode 100644 apps/desktop/src/main/computer-use-windows-interlock-journeys.test.ts diff --git a/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc b/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc index 114f661a6..618522d8b 100644 --- a/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc +++ b/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc @@ -7695,6 +7695,7 @@ bool CheckWindowsInterlockJourneys() { return result; }; const char *const kinds[] = {"click", "invoke", "key", "type", "set_text", "scroll", "select", "toggle", "expand_collapse"}; + const auto notepad_path_for_install = []() { return std::string("C:\\Windows\\System32\\notepad.exe"); }; const auto refused = [&](const ShadowUiaCapture &tree, const char *kind, const char *language) { ShadowRequestInfo request = UiaRequest(kind); request.policy_language = language; @@ -7705,46 +7706,98 @@ bool CheckWindowsInterlockJourneys() { return EvaluateWindowsInterlockFromCapture(tree, request, false, closed).Refuses(); }; - // The acceptance fixture: every control, in the window of the fixture. - const std::string fixture_title = "Sprint Coder Computer Use acceptance fixture"; + // The acceptance fixture: every control, in the window of the fixture. The values below are the + // fixture's own (its class and title from fixture_contract.h, its labels from + // win32_acceptance_fixture.cc); computer-use-windows-interlock-journeys.test.ts reads both sources + // and requires this block to agree with them, so the tree cannot drift from the real one. + // BEGIN fixture tree + const std::string fixture_class = "SprintCoderComputerUseAcceptanceFixtureV1"; + const std::string fixture_title = "Sprint Coder Computer Use Fixture v1"; const std::string fixture_path = "C:\\Fixture\\win32_acceptance_fixture.exe"; - struct Control { const char *type; const char *klass; const char *name; bool password; }; - const Control controls[] = { + const std::string dialog_class = "SprintCoderComputerUseSafeDialogV1"; + const std::string dialog_title = "Fixture same-owner safe dialog"; + const std::string picker_title = u8"Fixture file picker — Sprint Coder must request user takeover"_u8; + struct Control { std::string type; std::string klass; std::string name; bool password; }; + const std::vector controls = { {"Text", "Static", "Deterministic non-production fixture. Normal controls are safe; boundary controls must stop Computer Use.", false}, {"Text", "Static", "Normal text", false}, - {"Edit", "Edit", "Normal text", false}, + {"Edit", "Edit", "Fixture text: safe to replace", false}, {"Button", "Button", "Run deterministic normal action", false}, {"CheckBox", "Button", "Enable deterministic option", false}, {"Text", "Static", "Normal selection", false}, - {"ComboBox", "ComboBox", "Normal selection", false}, - {"Text", "Static", "Password field - MUST BLOCK", false}, - {"Edit", "Edit", "Password field", true}, + {"ComboBox", "ComboBox", "", false}, + {"ListItem", "", "Alpha", false}, + {"ListItem", "", "Beta", false}, + {"ListItem", "", "Gamma", false}, + {"Text", "Static", u8"Password field — MUST BLOCK"_u8, false}, + {"Edit", "Edit", "not-a-real-secret", true}, {"Text", "Static", "Safety boundary controls below are inert test affordances. They perform no payment and retain no selected path.", false}, - {"Button", "Button", "Authorize test payment - MUST BLOCK", false}, - {"Button", "Button", "Open file picker - USER TAKEOVER", false}, + {"Button", "Button", u8"Authorize test payment — MUST BLOCK"_u8, false}, + {"Button", "Button", u8"Open file picker — USER TAKEOVER"_u8, false}, {"Button", "Button", "Reset deterministic state", false}, {"Button", "Button", "Open same-owner safe dialog", false}, + {"Text", "Static", "READY: no boundary control has been invoked", false}, {"Text", "Static", "NORMAL_ACTION_COMPLETED: deterministic button invoked", false}, }; + const std::vector dialog_controls = { + {"Text", "Static", "This custom same-owner dialog is safe to automate.", false}, + {"Edit", "Edit", "Safe dialog text", false}, + {"Button", "Button", "Close safe dialog", false}, + }; + // END fixture tree for (const Control &control : controls) for (const char *kind : kinds) for (const char *language : {"en", "ja"}) - if (!Check(!refused(capture({node(control.type, control.klass, control.name, control.password), - node("Window", "SprintCoderComputerUseFixtureV1", fixture_title)}, - "SprintCoderComputerUseFixtureV1", fixture_path), + if (!Check(!refused(capture({node(control.type.c_str(), control.klass.c_str(), control.name, control.password), + node("Window", fixture_class.c_str(), fixture_title)}, + fixture_class, fixture_path), kind, language), (std::string("journey-fixture-not-refused-") + control.name).c_str())) return false; - // The safe dialog the fixture opens, and the file picker (the picker is a takeover of V1's own, + // The safe dialog the fixture opens (its controls), and the file picker (a takeover of V1's own, // outside the interlock). + for (const Control &control : dialog_controls) + for (const char *kind : kinds) + if (!Check(!refused(capture({node(control.type.c_str(), control.klass.c_str(), control.name), + node("Window", dialog_class.c_str(), dialog_title)}, + dialog_class, fixture_path), + kind, "en"), + (std::string("journey-safe-dialog-not-refused-") + control.name).c_str())) + return false; for (const char *kind : kinds) - if (!Check(!refused(capture({node("Button", "Button", "Close safe dialog"), - node("Window", "SprintCoderComputerUseSafeDialogV1", "Fixture same-owner safe dialog")}, - "SprintCoderComputerUseSafeDialogV1", fixture_path), - kind, "en"), - "journey-safe-dialog-not-refused")) + if (!Check(!refused(capture({node("Window", "#32770", picker_title)}, "#32770", fixture_path), kind, "en"), + "journey-file-picker-window-not-refused")) return false; + // E4, the installer rule: a control named like an installer is a takeover (paused) on every surface. + // V1 knows "installer", "install application" and the Japanese word for install; the interlock also + // knows "install" on its own and the Japanese installer word (owner accepted; the vocabulary test pins + // the words). Starting an installer is a takeover, whatever the control is. + for (const char *type : {"Button", "MenuItem"}) + for (const std::string &name : std::vector{"Install", "install", u8"\u30a4\u30f3\u30b9\u30c8\u30fc\u30e9"_u8}) { + ShadowRequestInfo request = UiaRequest("invoke"); + request.click_tracked = true; + const InterlockEvaluation evaluation = EvaluateWindowsInterlockFromCapture( + capture({node(type, "Button", name), node("Window", "Notepad", "Untitled - Notepad")}, "Notepad", notepad_path_for_install()), + request, false, closed); + if (!Check(evaluation.Refuses() && evaluation.verdict.outcome == InterlockOutcome::kTakeover && + evaluation.verdict.reason == InterlockReason::kExecutionTrigger && + evaluation.verdict.DeniedResult() == "paused", + (std::string("journey-installer-control-is-a-paused-takeover-") + type + "-" + name).c_str())) + return false; + } + // A word that merely contains it is not an installer (the rule is bounded). + { + ShadowRequestInfo request = UiaRequest("invoke"); + request.click_tracked = true; + if (!Check(!EvaluateWindowsInterlockFromCapture( + capture({node("Button", "Button", "Reinstall guide"), node("Window", "Notepad", "Untitled - Notepad")}, + "Notepad", notepad_path_for_install()), + request, false, closed).Refuses(), + "journey-install-inside-a-word-is-not-an-installer")) + return false; + } + // Notepad, old (a Window, an Edit, a menu bar) and new (a Document), English and Japanese titles. const std::string notepad_path = "C:\\Windows\\System32\\notepad.exe"; struct Tree { const char *name; std::vector chain; }; diff --git a/apps/desktop/computer-use-native/computer_use_windows_host.cc b/apps/desktop/computer-use-native/computer_use_windows_host.cc index 73c61bf75..4cfa99843 100644 --- a/apps/desktop/computer-use-native/computer_use_windows_host.cc +++ b/apps/desktop/computer-use-native/computer_use_windows_host.cc @@ -4347,6 +4347,44 @@ bool WindowsRecentInputMayPair() noexcept { bool WindowsClickOpens() noexcept { return WindowsSingleClickOpens() || WindowsRecentInputMayPair(); } + +// What the interlock evaluation knew about the click when it ran (#500 S5-c-2 PR-5, review): whether +// the click is followed at all, whether it was already taken as opening, and the system's last input +// time at that moment. The evaluation can take up to the lane's hold limit; this lets the dispatch +// tell, just before it sends the click, that input came in after the evaluation. +struct WindowsClickBasis { + bool tracked = false; + bool opens = false; + bool input_read = false; + DWORD input_tick = 0; +}; + +bool WindowsReadLastInputTick(DWORD *tick) noexcept { + LASTINPUTINFO info{}; + info.cbSize = sizeof(info); + if (!GetLastInputInfo(&info)) return false; + *tick = info.dwTime; + return true; +} + +// Whether input has come in since the evaluation, for a click the evaluation judged as NOT opening +// (a click already taken as opening has nothing to lose). The dispatch then stops the click before +// it is sent (fail closed): the verdict was made for a moment that is no longer the present one, and +// a person's click that now pairs with ours would open the item. A last input time that cannot be +// read counts as moved. The cost: a person moving the mouse during the evaluation stops the click +// (the agent tries again). +bool WindowsInputMovedSinceEvaluation(const WindowsClickBasis &basis) noexcept { + if (!basis.tracked || basis.opens) return false; + DWORD now_tick = 0; + if (!basis.input_read || !WindowsReadLastInputTick(&now_tick)) return true; + return now_tick != basis.input_tick; +} + +// The v2 interlocks' answer for an action whose V1 binding is valid (defined with the synchronous +// entry): a refusal, or no verdict. A click fills `basis` with what the evaluation knew. +sprint_coder::computer_use::InterlockEvaluation +WindowsInterlockRefusal(const WindowsSession &session, const std::string &metadata, + const std::string &kind, WindowsClickBasis *basis = nullptr) noexcept; // END click rule // BEGIN windows action point @@ -4369,12 +4407,6 @@ bool WindowsActionPoint(const RECT &bounds, const std::string &metadata, } // END windows action point -// The v2 interlocks' answer for an action whose V1 binding is valid (defined with the synchronous -// entry): a refusal, or no verdict. -sprint_coder::computer_use::InterlockEvaluation -WindowsInterlockRefusal(const WindowsSession &session, const std::string &metadata, - const std::string &kind) noexcept; - bool DispatchWindowsAction(const WindowsSession &session, const std::string &metadata, std::string *result, std::string *reason, bool *accepted) { @@ -4388,14 +4420,27 @@ bool DispatchWindowsAction(const WindowsSession &session, // since the preflight (the ticket lives seconds), and the click facts (the last input time) are read // again by the evaluation. A refusal is made before the first input API attempt, SetCursorPos or // SendInput. No verdict leaves the V1 guards below as they are. + WindowsClickBasis click_basis; { const sprint_coder::computer_use::InterlockEvaluation evaluation = - WindowsInterlockRefusal(session, metadata, kind); + WindowsInterlockRefusal(session, metadata, kind, &click_basis); if (evaluation.Refuses()) { *reason = std::string(evaluation.verdict.DispatchCode()); return false; } } + // The evaluation took time (up to the lane's hold limit) after the binding check, and the focus + // moved to another control of the same window in that time would send the key or the text to an + // element the observation never saw. The same check as the binding's, again, before any input. + if (kind == "key" || kind == "type") { + std::string current_focused_signature; + if (!session.has_focused_element_signature || + !WindowsFocusedElementSignature(session.pid, bounds, ¤t_focused_signature) || + current_focused_signature != session.focused_element_signature) { + *reason = "focused_target_changed"; + return false; + } + } bool dispatched = false; if (kind == "click") { double x = 0; @@ -4453,6 +4498,12 @@ bool DispatchWindowsAction(const WindowsSession &session, *reason = "stale_target"; return false; } + // Input came in after the interlock evaluation judged this click as not opening: it is not sent + // (the existing code of a target that is no longer the one that was judged). + if (WindowsInputMovedSinceEvaluation(click_basis)) { + *reason = "focused_target_changed"; + return false; + } INPUT events[2]{}; events[0].type = INPUT_MOUSE; events[0].mi.dwFlags = MOUSEEVENTF_LEFTDOWN; @@ -4897,9 +4948,10 @@ void ScheduleWindowsEffectShadow(const WindowsSession &session, // incomplete facts are NO VERDICT (the answer carries the reason): never a refusal and never an // approval, so the V1 preflight decides. // -// Behaviour-neutral: NOTHING calls this yet (the preflight and the dispatch guard are wired later in -// S5-c-2). The caller must call it only AFTER the V1 re-validation of the bound target succeeded -// (ValidateWindowsDispatchBinding): the point is only right for a binding that is still valid. +// Called from exactly two places, through WindowsInterlockRefusal below: the preflight, before it +// issues a ticket, and the dispatch, before any input (#500 S5-c-2 PR-5). The caller must call it +// only AFTER the V1 re-validation of the bound target succeeded (ValidateWindowsDispatchBinding): the +// point is only right for a binding that is still valid. // // THE SERIAL LANE IS HELD AT MOST kInterlockLaneHoldLimit. The lane's thread is an STA (the picker // and the WinRT capture need it), and a UI Automation call to an app that does not answer is @@ -4942,7 +4994,8 @@ std::atomic &InterlockWorkerRunning() { sprint_coder::computer_use::InterlockEvaluation EvaluateWindowsInterlockOnLane( const WindowsSession &session, const std::string &metadata, const std::string &kind, - const sprint_coder::computer_use::EffectRuleset &ruleset) noexcept { + const sprint_coder::computer_use::EffectRuleset &ruleset, + WindowsClickBasis *basis = nullptr) noexcept { using namespace sprint_coder::computer_use; InterlockEvaluation unavailable; unavailable.no_verdict = InterlockNoVerdict::kIncomplete; @@ -4967,7 +5020,16 @@ sprint_coder::computer_use::InterlockEvaluation EvaluateWindowsInterlockOnLane( // wait) and handed to the worker by value: the worker touches no session and no system state. // The time of the last input is read at this request, not when a worker finally runs. const bool click_tracked = kind == "click" && point_known; + // The last input time first, then the click facts (which read it again): the time of the basis is + // never later than the one the verdict used. + if (basis != nullptr && click_tracked) { + basis->input_read = WindowsReadLastInputTick(&basis->input_tick); + } const bool click_opens = click_tracked && WindowsClickOpens(); + if (basis != nullptr) { + basis->tracked = click_tracked; + basis->opens = click_opens; + } // Everything that can throw is prepared BEFORE the single-flight slot is taken, so a failure // here cannot leave the slot taken for ever. auto pending = std::make_shared(); @@ -5082,9 +5144,9 @@ void CountWindowsInterlockNoVerdict(sprint_coder::computer_use::InterlockNoVerdi sprint_coder::computer_use::InterlockEvaluation WindowsInterlockRefusal(const WindowsSession &session, const std::string &metadata, - const std::string &kind) noexcept { + const std::string &kind, WindowsClickBasis *basis) noexcept { const sprint_coder::computer_use::InterlockEvaluation evaluation = - EvaluateWindowsInterlockOnLane(session, metadata, kind, kWindowsInterlockRuleset); + EvaluateWindowsInterlockOnLane(session, metadata, kind, kWindowsInterlockRuleset, basis); if (!evaluation.Refuses() && evaluation.no_verdict != sprint_coder::computer_use::InterlockNoVerdict::kNone) CountWindowsInterlockNoVerdict(evaluation.no_verdict); diff --git a/apps/desktop/src/main/computer-use-windows-click-rule.test.ts b/apps/desktop/src/main/computer-use-windows-click-rule.test.ts index cca17b196..fb9c10b52 100644 --- a/apps/desktop/src/main/computer-use-windows-click-rule.test.ts +++ b/apps/desktop/src/main/computer-use-windows-click-rule.test.ts @@ -104,6 +104,38 @@ describe('click rule (#500 S5-c-2 PR-2): portable', () => { expect(listed).toEqual(macos); }); + it('stops a click that input overtook after the evaluation, just before it is sent (#500 S5-c-2 PR-5 review)', () => { + const dispatch = host.slice( + host.indexOf('bool DispatchWindowsAction('), + host.indexOf('BackendProbe ProbeBackend('), + ); + const click = dispatch.slice(0, dispatch.indexOf('} else if (kind == "scroll")')); + const squashed = squash(click); + const guard = click.indexOf('WindowsInputMovedSinceEvaluation(click_basis)'); + const send = click.indexOf('SendInputForBoundTarget('); + // Right after the last re-validation of the target, before the input is built and sent. + expect(click.slice(Math.max(0, guard - 700), guard)).toContain( + 'RevalidateWindowsTarget(session, bounds, expected_epoch)', + ); + expect(guard).toBeGreaterThan(0); + expect(send).toBeGreaterThan(guard); + expect(squashed).toContain( + 'if (WindowsInputMovedSinceEvaluation(click_basis)) { *reason = "focused_target_changed"; return false; }', + ); + // The basis is the one the dispatch's own evaluation filled. + expect(squash(dispatch)).toContain( + 'WindowsInterlockRefusal(session, metadata, kind, &click_basis)', + ); + // The decision, in the click rule block: only a tracked click that was judged as not opening can be + // overtaken, and an unreadable time counts as overtaken. + const block = squash(clickRuleBlock()); + expect(block).toContain('if (!basis.tracked || basis.opens) return false;'); + expect(block).toContain( + 'if (!basis.input_read || !WindowsReadLastInputTick(&now_tick)) return true;', + ); + expect(block).toContain('return now_tick != basis.input_tick;'); + }); + it('keeps the macOS source out of it: Finder has no single-click-to-open setting, and the wire has no double click', () => { expect(read('computer_use_macos.mm')).not.toContain('click_opens'); }); @@ -144,9 +176,13 @@ describe('click rule (#500 S5-c-2 PR-2): the Windows host', () => { ); // The synchronous entry takes both on the lane, before the worker exists, and hands them to the // worker by value; the worker reads no system state of its own for them. + expect(squashed).toContain('const bool click_tracked = kind == "click" && point_known;'); + expect(squashed).toContain('const bool click_opens = click_tracked && WindowsClickOpens();'); + // The basis (the last input time before the click facts, and what they said) is filled on the lane. expect(squashed).toContain( - 'const bool click_tracked = kind == "click" && point_known; const bool click_opens = click_tracked && WindowsClickOpens();', + 'if (basis != nullptr && click_tracked) { basis->input_read = WindowsReadLastInputTick(&basis->input_tick); }', ); + expect(squashed).toContain('basis->tracked = click_tracked; basis->opens = click_opens;'); expect(squashed).toContain( 'info.click_tracked = click_tracked; info.click_opens = click_opens;', ); @@ -265,9 +301,12 @@ const program = String.raw` #include #include "computer_use_effect_shadow.h" +#include "computer_use_native_interlock_capture.h" namespace { +struct WindowsSession {}; + //@@BLOCK@@ } // namespace @@ -309,6 +348,37 @@ int main() { Check(!WindowsRecentInputMayPair(), "no_input_for_longer_than_the_interval_cannot_pair"); else std::cout << "skip stale_check (somebody else is using the machine)\n"; + // A click the evaluation judged as not opening is stopped when input came in after it. + { + WindowsClickBasis basis; + Check(!WindowsInputMovedSinceEvaluation(basis), "untracked_click_is_never_stopped"); + basis.tracked = true; + basis.opens = true; + Check(!WindowsInputMovedSinceEvaluation(basis), "click_taken_as_opening_is_not_stopped"); + basis.opens = false; + Check(WindowsInputMovedSinceEvaluation(basis), "unread_input_time_stops_the_click"); + basis.input_read = WindowsReadLastInputTick(&basis.input_tick); + // Nothing came in since the basis was taken (a person at the machine only makes this skip). + if (basis.input_read && !WindowsInputMovedSinceEvaluation(basis)) { + // Input that does not come from the helper (a key-up of an unassigned key) overtakes it. + DWORD stamp_before = basis.input_tick; + INPUT key{}; + key.type = INPUT_KEYBOARD; + key.ki.wVk = VK_F24; + key.ki.dwFlags = KEYEVENTF_KEYUP; + Sleep(20); + SendInput(1, &key, sizeof(INPUT)); + Sleep(30); + DWORD stamp_after = 0; + WindowsReadLastInputTick(&stamp_after); + if (stamp_after != stamp_before) + Check(WindowsInputMovedSinceEvaluation(basis), "input_after_the_evaluation_stops_the_click"); + else + std::cout << "skip input_after_the_evaluation (the system did not move its last input time)\n"; + } else { + std::cout << "skip nothing_moved_since (somebody else is using the machine)\n"; + } + } // The click opens when either part says so. Check(WindowsClickOpens() == (WindowsSingleClickOpens() || WindowsRecentInputMayPair()), "click_opens_is_the_or"); std::cout << (failures == 0 ? "OK\n" : "FAILED\n"); diff --git a/apps/desktop/src/main/computer-use-windows-dispatch-interlock-seam.test.ts b/apps/desktop/src/main/computer-use-windows-dispatch-interlock-seam.test.ts index c0ac78d41..bda4463b3 100644 --- a/apps/desktop/src/main/computer-use-windows-dispatch-interlock-seam.test.ts +++ b/apps/desktop/src/main/computer-use-windows-dispatch-interlock-seam.test.ts @@ -95,11 +95,30 @@ using sprint_coder::computer_use::InterlockOutcome; using sprint_coder::computer_use::InterlockReason; struct RECT { long left = 0; long top = 0; long right = 0; long bottom = 0; }; -struct WindowsSession { std::uint64_t cancel_epoch = 0; }; +struct WindowsSession { + std::uint64_t cancel_epoch = 0; + std::uint32_t pid = 4242; + bool has_focused_element_signature = true; + std::string focused_element_signature = "observed-focus"; +}; +struct WindowsClickBasis { bool tracked = false; bool opens = false; bool input_read = false; unsigned long input_tick = 0; }; std::vector g_calls; +std::string g_kind = "click"; bool g_binding_ok = true; InterlockEvaluation g_answer; +// What the focus is after the interlocks ran, and whether it can be read. +std::string g_focus_after_interlock = "observed-focus"; +bool g_focus_readable = true; +bool g_focus_changes_during_interlock = false; +std::string g_focus_now = "observed-focus"; + +bool WindowsFocusedElementSignature(std::uint32_t, const RECT &, std::string *signature) { + g_calls.push_back("focus"); + if (!g_focus_readable) return false; + *signature = g_focus_now; + return true; +} bool ValidateWindowsDispatchBinding(const WindowsSession &, const std::string &, std::string *kind, RECT *, std::string *reason) { @@ -108,13 +127,15 @@ bool ValidateWindowsDispatchBinding(const WindowsSession &, const std::string &, *reason = "stale_target"; return false; } - *kind = "click"; + *kind = g_kind; return true; } InterlockEvaluation WindowsInterlockRefusal(const WindowsSession &, const std::string &, - const std::string &kind) noexcept { + const std::string &kind, WindowsClickBasis * = nullptr) noexcept { g_calls.push_back("interlock:" + kind); + // The evaluation takes time: the focus can move to another control while it runs. + if (g_focus_changes_during_interlock) g_focus_now = g_focus_after_interlock; return g_answer; } @@ -137,6 +158,7 @@ void Check(bool ok, const char *name) { struct Outcome { bool ran_on; bool accepted; std::string reason; std::vector calls; }; Outcome Run() { g_calls.clear(); + g_focus_now = "observed-focus"; WindowsSession session; std::string result; std::string reason; @@ -171,6 +193,45 @@ int main() { Check(outcome.ran_on && outcome.reason.empty(), "no_verdict_goes_on_to_v1"); Check(outcome.calls == std::vector({"validate", "interlock:click", "v1-guards"}), "no_verdict_order"); } + // The focus recheck, for a key or a text, after the interlocks and before any input: the signature + // is the observed one (go on), another one (stop, no input), or unreadable (stop). + g_binding_ok = true; + g_answer = InterlockEvaluation{}; + for (const char *kind : {"key", "type"}) { + g_kind = kind; + g_focus_readable = true; + g_focus_changes_during_interlock = false; + { + const Outcome outcome = Run(); + Check(outcome.ran_on && outcome.calls == std::vector({"validate", std::string("interlock:") + kind, "focus", "v1-guards"}), + "focus_unchanged_goes_on"); + } + g_focus_changes_during_interlock = true; + g_focus_after_interlock = "another-control"; + { + const Outcome outcome = Run(); + Check(!outcome.ran_on && !outcome.accepted && outcome.reason == "focused_target_changed" && + outcome.calls == std::vector({"validate", std::string("interlock:") + kind, "focus"}), + "focus_moved_during_the_interlocks_sends_nothing"); + } + g_focus_changes_during_interlock = false; + g_focus_readable = false; + { + const Outcome outcome = Run(); + Check(!outcome.ran_on && outcome.reason == "focused_target_changed", "unreadable_focus_sends_nothing"); + } + g_focus_readable = true; + } + // Other kinds do not look at the focus again (their V1 checks are their own). + g_kind = "click"; + g_focus_changes_during_interlock = true; + { + const Outcome outcome = Run(); + Check(outcome.ran_on && outcome.calls == std::vector({"validate", "interlock:click", "v1-guards"}), + "click_does_not_recheck_the_focus"); + } + g_focus_changes_during_interlock = false; + g_kind = "click"; // A failed binding is V1's own refusal and the interlocks are never asked. g_binding_ok = false; g_answer = InterlockEvaluation{}; diff --git a/apps/desktop/src/main/computer-use-windows-interlock-capture.test.ts b/apps/desktop/src/main/computer-use-windows-interlock-capture.test.ts index fbd27d4bc..151a747f2 100644 --- a/apps/desktop/src/main/computer-use-windows-interlock-capture.test.ts +++ b/apps/desktop/src/main/computer-use-windows-interlock-capture.test.ts @@ -167,7 +167,7 @@ describe('Windows synchronous interlock entry (classifier v2, S5-c-1)', () => { 'constexpr sprint_coder::computer_use::EffectRuleset kWindowsInterlockRuleset{ /*download_open=*/false, /*file_manager_open=*/false};', ); expect(body).toContain( - 'EvaluateWindowsInterlockOnLane(session, metadata, kind, kWindowsInterlockRuleset)', + 'EvaluateWindowsInterlockOnLane(session, metadata, kind, kWindowsInterlockRuleset, basis)', ); // Counts and enum names only: no label, no title, no text. const counter = entry.slice( @@ -178,10 +178,85 @@ describe('Windows synchronous interlock entry (classifier v2, S5-c-1)', () => { expect(counter).not.toMatch(/label|title|name\s*=\s*node|metadata|text/iu); }); - it('leaves INV-3 alone: the eligibility, the versions and the supported targets are what they were (#500 S5-c-2 PR-5)', () => { - expect(host).toContain('IsSupportedWindowsV1Target'); - expect(host).toContain('MaximumModeForWindowsExecutable'); - expect(host).toContain('kNativePreflightClassifierVersion'); + it('leaves INV-3 alone: the function bodies that decide eligibility, the win32 refusal of v2 and the versions are pinned (#500 S5-c-2 PR-5)', () => { + const squashed = (text: string): string => text.replace(/\s+/gu, ' ').trim(); + const functionBody = (signature: string): string => { + const from = host.indexOf(signature); + expect(from, signature).toBeGreaterThanOrEqual(0); + const open = host.indexOf('{', from); + let depth = 0; + for (let index = open; index < host.length; index += 1) { + if (host[index] === '{') depth += 1; + else if (host[index] === '}') { + depth -= 1; + if (depth === 0) return squashed(host.slice(from, index + 1)); + } + } + throw new Error(`unterminated: ${signature}`); + }; + // What a target must be to get full access, and the one place it is decided. + expect(functionBody('std::string MaximumModeForWindowsExecutable(')).toBe( + 'std::string MaximumModeForWindowsExecutable( const WindowsExecutableIdentity &identity) { if (!IsPolicyLanguageSupported(identity.policy_language)) return "observe_only"; if (IsTrustedSystemNotepad(identity)) return "full_access_app"; const std::string helper_signer = CurrentHelperSignerDigest(); if (IsAcceptanceFixtureExecutable(identity.path) && !helper_signer.empty() && identity.signer_digest == helper_signer) return "full_access_app"; return "observe_only"; }', + ); + expect(functionBody('bool IsSupportedWindowsV1Target(')).toBe( + 'bool IsSupportedWindowsV1Target( const WindowsExecutableIdentity &identity) { return MaximumModeForWindowsExecutable(identity) == "full_access_app"; }', + ); + // The executables a session can never target (a remote desktop, a shell, an installer, Settings, ...). + const blocked = functionBody('bool IsDisallowedTargetExecutable(const std::wstring &path) {'); + expect( + [...blocked.matchAll(/L"([^"]+)"/gu)] + .map((match) => match[1]) + .filter((name) => name!.endsWith('.exe')), + ).toEqual([ + 'mstsc.exe', + 'msrdc.exe', + 'SystemSettings.exe', + 'control.exe', + 'msiexec.exe', + 'setup.exe', + 'install.exe', + 'installer.exe', + 'powershell.exe', + 'pwsh.exe', + 'cmd.exe', + 'wt.exe', + 'WindowsTerminal.exe', + 'wsl.exe', + 'bash.exe', + 'cscript.exe', + 'wscript.exe', + 'regedit.exe', + 'mmc.exe', + 'explorer.exe', + 'taskmgr.exe', + 'msrdcw.exe', + 'RdClient.Windows.exe', + 'QuickAssist.exe', + 'TeamViewer.exe', + 'AnyDesk.exe', + 'RustDesk.exe', + 'parsec.exe', + ]); + expect(blocked).toContain( + 'if (_wcsicmp(leaf.c_str(), candidate.data()) == 0) return true; return false; }', + ); + // The build refuses eligibility v2 for win32 (no deny-list there yet): a Windows helper cannot be + // built with the interlocks' approval path, or with a wider eligibility. + const build = squashed( + readFileSync(join(__dirname, '../../../../build-computer-use-eligibility.mjs'), 'utf8'), + ); + expect(build).toContain( + "if (mode === MODES.v2 && target === 'win32') throw new Error( 'Computer Use native --eligibility=v2 is unsupported on Windows until S5 (no deny-list there yet)', );", + ); + // The classifier version the preflight receipt attests, and the versions of the eligibility. + expect(read('computer_use_preflight_classifier.h')).toContain( + 'constexpr std::uint32_t kNativePreflightClassifierVersion = 1;', + ); + expect( + squashed(host).match( + /std::to_string\( ?sprint_coder::computer_use::kNativePreflightClassifierVersion\)/gu, + ), + ).toHaveLength(1); const eligibility = JSON.parse(read('eligibility.json')) as { versions: Record; }; @@ -190,8 +265,6 @@ describe('Windows synchronous interlock entry (classifier v2, S5-c-1)', () => { classifierVersion: 1, lexiconVersion: 1, }); - // The target eligibility functions are not touched by this change: they appear as before. - expect(host.match(/\bIsSupportedWindowsV1Target\(/gu)?.length).toBeGreaterThan(0); }); it('takes the rule set from its caller, answers an exception or a missing source as no verdict, and changes nothing', () => { diff --git a/apps/desktop/src/main/computer-use-windows-interlock-journeys.test.ts b/apps/desktop/src/main/computer-use-windows-interlock-journeys.test.ts new file mode 100644 index 000000000..c51f06afd --- /dev/null +++ b/apps/desktop/src/main/computer-use-windows-interlock-journeys.test.ts @@ -0,0 +1,135 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +// The fixture's UI Automation tree in the interlock journeys of the harness +// (computer_use_protocol_fuzz.cc, "BEGIN fixture tree") is copied by hand, so it could drift from the +// real fixture and the journeys would then prove nothing about it. This test reads the real sources +// (fixture_contract.h for the class and the title, win32_acceptance_fixture.cc for every label the +// controls are created with) and requires the harness block to agree: the same class and titles, every +// label of the fixture present in the tree, and nothing in the tree that the fixture does not carry. + +const nativeDirectory = join(__dirname, '../../computer-use-native'); +const fixtureDirectory = join(nativeDirectory, 'fixtures', 'win32-acceptance'); +const harness = readFileSync(join(nativeDirectory, 'computer_use_protocol_fuzz.cc'), 'utf8'); +const contract = readFileSync(join(fixtureDirectory, 'fixture_contract.h'), 'utf8'); +const fixture = readFileSync(join(fixtureDirectory, 'win32_acceptance_fixture.cc'), 'utf8'); + +const unescape = (text: string): string => + text + .replace(/\\u([0-9a-fA-F]{4})/gu, (_match, hex: string) => + String.fromCharCode(parseInt(hex, 16)), + ) + .replace(/\\"/gu, '"') + .replace(/\\\\/gu, '\\'); + +// One wide literal, or several next to each other (adjacent literals are one string in C++). +function wideLiteral(text: string): string { + return [...text.matchAll(/L"((?:[^"\\]|\\.)*)"/gu)].map((match) => unescape(match[1]!)).join(''); +} + +const constants = new Map(); +for (const match of fixture.matchAll( + /constexpr wchar_t (\w+)\[\] = ((?:L"(?:[^"\\]|\\.)*"\s*)+);/gu, +)) + constants.set(match[1]!, wideLiteral(match[2]!)); +const operand = (text: string): string => + text.trim().startsWith('L"') ? wideLiteral(text) : (constants.get(text.trim()) ?? `?${text}`); + +// The block of the harness. +const begin = harness.indexOf('// BEGIN fixture tree'); +const end = harness.indexOf('// END fixture tree'); +const block = harness.slice(begin, end); +const stringConstant = (name: string): string => { + const match = new RegExp( + `const std::string ${name} = (?:u8)?"((?:[^"\\\\]|\\\\.)*)"(?:_u8)?;`, + 'u', + ).exec(block); + expect(match, name).not.toBeNull(); + return unescape(match![1]!); +}; +const entries = [ + ...block.matchAll( + /\{\s*"([^"]*)",\s*"([^"]*)",\s*(?:u8)?"((?:[^"\\]|\\.)*)"(?:_u8)?,\s*(true|false)\s*\}/gu, + ), +].map((match) => ({ type: match[1]!, klass: match[2]!, name: unescape(match[3]!) })); + +// What the fixture creates its controls with. +const labels = new Set(); +for (const match of fixture.matchAll( + /AddControl\(\s*\w+,\s*[\w_|]+,\s*L"[^"]*",\s*((?:L"(?:[^"\\]|\\.)*"\s*)+|\w+)\s*,/gu, +)) { + const label = operand(match[1]!); + if (label !== '') labels.add(label); +} +for (const match of fixture.matchAll( + /CreateWindowExW\(0, L"STATIC",\s*((?:L"(?:[^"\\]|\\.)*"\s*)+),/gu, +)) + labels.add(wideLiteral(match[1]!)); +for (const match of fixture.matchAll(/CB_ADDSTRING, 0, reinterpret_cast\((L"[^"]*")\)/gu)) + labels.add(wideLiteral(match[1]!)); +const picker = /options\.lpstrTitle = ((?:L"(?:[^"\\]|\\.)*"\s*)+);/u.exec(fixture); + +describe('the harness fixture tree against the real fixture (#500 S5-c-2 PR-5 review)', () => { + it('reads the sources', () => { + expect(begin).toBeGreaterThan(0); + expect(end).toBeGreaterThan(begin); + expect(entries.length).toBeGreaterThan(15); + expect(labels.size).toBeGreaterThan(14); + expect(picker).not.toBeNull(); + }); + + it('uses the fixture window class and title from fixture_contract.h', () => { + expect(stringConstant('fixture_class')).toBe( + /kWindowClass\[\] = L"([^"]+)"/u.exec(contract)![1], + ); + expect(stringConstant('fixture_title')).toBe( + /kWindowTitle\[\] = L"([^"]+)"/u.exec(contract)![1], + ); + }); + + it('uses the safe dialog class and title, and the file picker title, of the fixture', () => { + expect(stringConstant('dialog_class')).toBe(constants.get('kSafeDialogClass')); + expect(stringConstant('dialog_title')).toBe(constants.get('kSafeDialogTitle')); + expect(stringConstant('picker_title')).toBe(wideLiteral(picker![1]!)); + }); + + it('carries every label the fixture creates a control with, and no other', () => { + const tree = new Set(entries.map((entry) => entry.name).filter((name) => name !== '')); + for (const label of labels) + expect(tree.has(label), `missing in the harness: ${label}`).toBe(true); + // Everything else in the tree is text the fixture sets on a control while it runs (a status), which + // is a literal of the fixture's own source. + const sourceLiterals = new Set( + [...fixture.matchAll(/L"((?:[^"\\]|\\.)*)"/gu)].map((m) => unescape(m[1]!)), + ); + for (const name of tree) + expect( + labels.has(name) || sourceLiterals.has(name) || wideLiteralJoined(name), + name, + ).toBeTruthy(); + }); + + it('has the controls the contract lists: edit, buttons, a combo with items, a password edit, the dialog edit', () => { + const find = (type: string, name: string) => + entries.some((entry) => entry.type === type && entry.name === name); + expect(find('Edit', 'Fixture text: safe to replace')).toBe(true); + expect(find('Button', 'Run deterministic normal action')).toBe(true); + expect(find('CheckBox', 'Enable deterministic option')).toBe(true); + expect(entries.some((entry) => entry.type === 'ComboBox')).toBe(true); + for (const item of ['Alpha', 'Beta', 'Gamma']) expect(find('ListItem', item)).toBe(true); + expect( + entries.some((entry) => entry.type === 'Edit' && entry.name === 'not-a-real-secret'), + ).toBe(true); + expect(find('Edit', 'Safe dialog text')).toBe(true); + expect(find('Button', 'Close safe dialog')).toBe(true); + }); +}); + +// A name made of adjacent literals of the fixture's source (the long texts) is one string there. +function wideLiteralJoined(name: string): boolean { + const joined = [...fixture.matchAll(/((?:L"(?:[^"\\]|\\.)*"\s*)+)/gu)].map((m) => + wideLiteral(m[1]!), + ); + return joined.includes(name); +} diff --git a/apps/desktop/src/main/computer-use-windows-interlock-live-seam.test.ts b/apps/desktop/src/main/computer-use-windows-interlock-live-seam.test.ts index bfcbeabdf..d35d9ee8a 100644 --- a/apps/desktop/src/main/computer-use-windows-interlock-live-seam.test.ts +++ b/apps/desktop/src/main/computer-use-windows-interlock-live-seam.test.ts @@ -167,6 +167,16 @@ bool ReadJsonDouble(std::string_view json, std::string_view key, double *output) // The click rule (the host, #500 S5-c-2 PR-2) is its own contract; here a click never opens. bool WindowsClickOpens() noexcept { return false; } +struct WindowsClickBasis { + bool tracked = false; + bool opens = false; + bool input_read = false; + DWORD input_tick = 0; +}; +bool WindowsReadLastInputTick(DWORD *tick) noexcept { + *tick = 1; + return true; +} // Stands in for the capture's evaluation to stall the worker by a chosen time (the real call is still // made): a real window that does not answer is cut off by UI Automation's own timeout and the gate diff --git a/apps/desktop/src/main/computer-use-windows-interlock-vocabulary.test.ts b/apps/desktop/src/main/computer-use-windows-interlock-vocabulary.test.ts index f8504e88f..bb041f32c 100644 --- a/apps/desktop/src/main/computer-use-windows-interlock-vocabulary.test.ts +++ b/apps/desktop/src/main/computer-use-windows-interlock-vocabulary.test.ts @@ -81,3 +81,33 @@ describe('interlock surface words against V1 words (#500 S5-c-2 PR-5)', () => { ); }); }); + +// E4, the installer rule (kInstallerTerms), is the same kind of difference: it makes a click on a +// control named like an installer a takeover (paused) on EVERY surface, and V1's risk text has fewer +// of those words. The owner accepted this direction (starting an installer fails closed, the same +// as the words above). Frozen: a word added to kInstallerTerms without a decision turns this red. +const v1Risk = quotedTerms(host, 'static constexpr std::array terms', ''); +const installer = quotedTerms( + classifier, + 'inline constexpr std::u8string_view kInstallerTerms[]', + 'u8', +); +const ACCEPTED_NEW_INSTALLER_WORDS = ['install', 'インストーラ']; + +describe('interlock installer words against V1 words (#500 S5-c-2 PR-5 review)', () => { + it('reads both tables', () => { + expect(v1Risk.length).toBe(36); + expect(installer.length).toBeGreaterThanOrEqual(5); + }); + + it('adds exactly "install" (on a word boundary) and "インストーラ" to what V1 refuses', () => { + const added = installer.filter((word) => !v1Risk.includes(word)); + expect([...added].sort()).toEqual([...ACCEPTED_NEW_INSTALLER_WORDS].sort()); + }); + + it('shares the rest: V1 has installer, install application and インストール', () => { + expect(installer.filter((word) => v1Risk.includes(word)).sort()).toEqual( + ['install application', 'installer', 'インストール'].sort(), + ); + }); +}); diff --git a/tasks/designs/adr-computer-use-agent-driven-v2.md b/tasks/designs/adr-computer-use-agent-driven-v2.md index 3a746e925..1231c81a5 100644 --- a/tasks/designs/adr-computer-use-agent-driven-v2.md +++ b/tasks/designs/adr-computer-use-agent-driven-v2.md @@ -668,8 +668,8 @@ computer_start がここで返る → { sessionId, state, stopReason, mode, roun (1) UIA のタイムアウトは同期の入口で 250 ms、レーンの保持の上限は 500 ms（最悪の見積りは PR-1 の節のとおり。preflight と dispatch の 2 回で 1 操作あたり最大約 1 s の遅れ）。(2) Explorer の lookup は 150 ms の待ちで、実パスが読めないファイル項目は判定なし（kIncomplete）のまま。(3) lookup の枠は shadow と同期評価で分けた。(4) MTA は評価ごとのワーカーで実現し、主スレッドは picker と WinRT の捕捉のために STA に残す。(5) 座標写像は `WindowsActionPoint` の 1 つ。(6) 開始前の取消しは UIA を作らずに kCanceled。 **配線**: `WindowsInterlockRefusal`（同期の入口の区間にある小さな関数。seam から差し替えられる）が、評価と、判定なしの理由ごとの件数（`OutputDebugStringA`。enum 名と件数だけでラベルは出さない）を受け持つ。ruleset は `kWindowsInterlockRuleset`（承認経路なし。win32 の v2 build は `build-computer-use-eligibility.mjs` で拒否されるので、C4-2c-2 (7) の門は Windows では自明に閉じている。Windows に route v2 は無い）。`HandleWindowsPreflight` は V1 の binding の検証が成功した後、ticket を出す前に評価し、拒否なら ticket を出さずに `blocked` / `takeover` の受領書（`DecisionName` / `ReasonCode` / `DeniedResult`）を返す。送る前に `NativePreflightRefusalAllowed` を通し、表に無い組は `denied` / `native_preflight_invalid`（fail closed）。判定なしなら従来どおり V1 の ordinary ticket。shadow は従来どおり呼ぶ（拒否のとき `issued=false`）。`DispatchWindowsAction` は binding の検証の直後、最初の入力 API の試行（`input_api_attempts`、`SetCursorPos`、`SendInput`、`DispatchWindowsSemanticAction`）より前にもう一度評価し、拒否なら `DispatchCode()`（`shell_surface_blocked` / `execution_trigger_blocked` / `approval_not_enabled`）と `accepted:false` で入力 0 回で返す。その後の V1 の guard と再検証は同じ順で残す。Main の `classifyComputerUseNativeDispatchFailure` は、この 3 つのコードを `NATIVE_PREFLIGHT_REFUSALS` の同じ行の result に写す（実行トリガは `paused`、それ以外は `rejected`）。V1 のコードは `rejected` のまま。この写し方は macOS の `dispatchApproved` と共有する。 **判定なし（fail-open）になる場合**: 取消し、予算切れ（応答しない対象、止まったワーカーが残っている間の single-flight を含む）、不完全な facts、`Create` の失敗、例外、点が無い click / scroll、lookup の放棄 / 曖昧 / 仮想フォルダ / 枠の使用中。S4-1 の方針「正の一致だけ拒否」のとおりで、V1 に任せる。V1 から届く面（メモ帳と fixture）には Explorer が無いので、今は表に出ない。 -**V1 で新しく拒否が増える語（所有者が受け入れた）**: classifier の `kSurfaceTerms` にあって V1 の `shell_terms` に無い語。`xterm`、`run script`、`ターミナル`、`コンソール`、`コマンドプロンプト`、`スクリプトを実行`（`shell script` と `script console` は V1 の語を含むので変わらない）と、全角表記（classifier は NFKC で ASCII に畳む）。英語の同じ語は V1 でもすでに拒否しているので、言語による差を無くす方向である。逆に V1 だけが拒否する語（`pwsh`、`wsl`）は V1 の guard が引き続き拒否する。差分の一覧は `computer-use-windows-interlock-vocabulary.test.ts` で固定し、語が増えたら赤になる。fixture とメモ帳（旧・新）の UIA の木は harness で全 action と en / ja の題名に通し、拒否が V1 の語か上の一覧の語に限られることを固定した。 -**既知の限界**: (a) preflight から dispatch までの TOCTOU（ticket の有効期間は 5 s）は、dispatch の直前の再評価（click の事実である最後の入力の時刻も評価のたびに読み直す）と、その後の V1 の再検証で狭める。評価から入力までの短い時間差は構造上残る。(b) 二重クリックの判定は、システムの最後の入力の時刻（`GetLastInputInfo`）だけで行い、人の入力と agent の入力を区別しない。人が操作している最中（二重クリックの時間内）の agent の click は「開く」側になるが、拒否されるのは実行トリガの項目だけである。拒否の理由に「人の入力と重なった可能性」を出すことは今回しない（受領書の表に行を足さない）。必要になったら別の判断とする。(c) 評価の時間が上限に達したときの件数は `OutputDebugStringA` に出る。 +**V1 で新しく拒否が増える語（所有者が受け入れた）**: classifier の `kSurfaceTerms` にあって V1 の `shell_terms` に無い語。`xterm`、`run script`、`ターミナル`、`コンソール`、`コマンドプロンプト`、`スクリプトを実行`（`shell script` と `script console` は V1 の語を含むので変わらない）と、全角表記（classifier は NFKC で ASCII に畳む）。これらは面の語による `blocked` / `rejected` として V1 に新しく出うる。英語の同じ語は V1 でもすでに拒否しているので、言語による差を無くす方向である。逆に V1 だけが拒否する語（`pwsh`、`wsl`）は V1 の guard が引き続き拒否する。**もう 1 つ、V1 に新しく出うる拒否がある**: E4 のインストーラー規則（`kInstallerTerms`）は、どの面でも takeover / `paused` を出す（実行の起動は fail closed に倒す方針。所有者が受け入れた「増える語」と同じ方向）。V1 の `ContainsWindowsRiskText` にあるのは `installer`、`install application`、`インストール`（部分一致）で、境界つきの `install` と `インストーラ` は無い。そのため「Install」という名前のボタンやメニュー項目への click / invoke が、V1 で新たに `paused` になる。V1 の今の到達面（メモ帳と fixture）に該当するラベルは無い。差分の一覧（面の語と、この 2 語）は `computer-use-windows-interlock-vocabulary.test.ts` で固定し、語が増えたら赤になる。fixture とメモ帳（旧・新）の UIA の木は harness で全 action と en / ja の題名に通し、拒否が V1 の語か上の一覧の語に限られることを固定した。 +**既知の限界**: (a) preflight から dispatch までの TOCTOU（ticket の有効期間は 5 s）は、dispatch の直前の再評価（click の事実である最後の入力の時刻も評価のたびに読み直す）と、その後の V1 の再検証で狭める。評価は最大 500 ms かかるので、評価の後に次の 2 つを足した。key / type は、フォーカス署名の照合を評価の後にもう一度行い（`focused_target_changed`、入力 0 回）、click は、評価が「開かない」と判断した click について、最後の V1 の再検証の後、`SendInput` の直前に `GetLastInputInfo` の時刻を読み直し、評価の時点から進んでいたら click を送らずに `focused_target_changed` で止める（再評価はしない fail closed。評価の間に人がマウスを動かした場合もこれで止まり、agent は再試行する）。評価から入力までの、さらに短い時間差は構造上残る。(b) 二重クリックの判定は、システムの最後の入力の時刻（`GetLastInputInfo`）だけで行い、人の入力と agent の入力を区別しない。人が操作している最中（二重クリックの時間内）の agent の click は「開く」側になるが、拒否されるのは実行トリガの項目だけである。拒否の理由に「人の入力と重なった可能性」を出すことは今回しない（受領書の表に行を足さない）。必要になったら別の判断とする。(c) 評価の時間が上限に達したときの件数は `OutputDebugStringA` に出る。 **INV-3**: S5-c-2 は適格性を広げない（`eligibility.json`、各版、attest、`list_targets` の適格性、`IsSupportedWindowsV1Target`、`MaximumModeForWindowsExecutable` は動かさない。source 契約で固定した）。S5-c-3（Windows で適格性を広げる PR）は次を受入条件にする: 不完全な facts、表示名の曖昧さ、lookup の放棄を takeover に倒すこと。interlock が有効であることを attest すること。 **C5 の分割と初回の有効化範囲（2026-10-16。所有者の判断）**: C5 は 2 つの PR に分ける。**C5-1（有効化の準備。本番では不活性）**: (1) 「承認経路が有効」の述語 `computerUseApprovalPathEnabled`（`computer-use-eligibility.ts`）。macOS、addon が承認の面を export、handshake が classifier 2 を attest、の 3 つに加えて、**session の mode と route**を見る: session は route v2 かつ実効 mode が `v2-denylist` に束縛されているときだけ開く。route v1 の session には、build が何を attest しても承認は無い。**束縛の無い session（route 無しで開始、verified close の後、開始に失敗した後）は閉じる**（fail closed。C3 以前の「束縛が無ければ attest だけで決まる」規則は廃止した）。述語は session について問うだけで、session 無しの問い合わせは無い（host と controller の引数は必須）。host（`approvalPathEnabled(sessionId)`）と controller（`nativeApprovalPathOpen(sessionId)`）は同じ述語を session 単位で使う。(2) `classifierV2Attested` の本番配線 `computerUseClassifierV2Attested`: Main にコンパイルされた eligibility が `v2-denylist` かつ `classifierVersion` = 2 で、gate が開いているときだけ true（v2 の Main は manifest と attest が自分の宣言と同一でなければ gate が閉じるので、gate が開いていれば native がその版を attest している）。環境変数もフラグも読まない。(3) `effectiveEligibilityMode` の本番配線: `index.ts`（host）と `ipc.ts`（controller）に `computerUseEligibilityWiring` を渡す。**版が 1 のままなので、どの build でも attest は 1、`classifierV2Attested` は false、host が native に渡す route は常に v1 である**（v2 の build と V2 フラグで agent セッションが route v2 を求めても、host は attest がなければ v1 を渡す）。したがって C4-2c-2 の ADR (7) の門は、Main 側では今回の配線の後も閉じている。**C5 より前に route v2 を本番で配線しない**前提は、「版を 2 に上げる PR（C5-2）だけが route v2 を native に届かせる」と読み替える。(4) 観測 metadata の高影響: `highImpact`（native の facts と Main の投影の和）と `highImpactNative`（native の facts だけ）は既にあり、`splitHighImpact` が「Main の投影だけが見た分」を明示する。**3-A**: Main の投影だけが見た高影響は、判定を強める（止める）方向にだけ使い、承認経路が開いていても緩めない（緩められるのは `highImpactNative` の分だけ）。**C5-2（有効化。#777 の実測の後）**: 版を 2 に上げる（`eligibility.json` を mode ごとの版表にする）、レーン上の承認測定の時間（1 回の AX が最大約 6 秒、Cancel ack 1 秒）を #777 で実測して `AXUIElementSetMessagingTimeout` の採否かレーン保持時間の上限を決める。**初回の有効化範囲は 2-A**: 承認可能にするのは `high_impact_confirmation` と `supervised_action` だけで、E2 / E3 は C6 まで拒否のままにする。**`eligibility.json` を mode ごとの版表にする構造変更は C5-2 に回す**: 版表の消費者（ビルドスクリプト、生成ヘッダ、約 12 本の契約テスト）が多く、C5-1 には読み手がいない。また表の形（mode ごとに ruleset / classifier / lexicon を分けるか、classifier だけか）は C5-2 で版を上げるときにしか決まらない。 From 3d1c56576fcc7ea09fa8fb00f8a4fcf5502cb0c9 Mon Sep 17 00:00:00 2001 From: yusei Date: Sat, 17 Oct 2026 11:36:58 +0900 Subject: [PATCH 3/5] =?UTF-8?q?test(computer-use):=20#500=20S5-c-2=20PR-5?= =?UTF-8?q?=20SetCursorPos=20=E3=81=8C=E6=9C=80=E5=BE=8C=E3=81=AE=E5=85=A5?= =?UTF-8?q?=E5=8A=9B=E3=81=AE=E6=99=82=E5=88=BB=E3=82=92=E5=8B=95=E3=81=8B?= =?UTF-8?q?=E3=81=95=E3=81=AA=E3=81=84=E3=81=93=E3=81=A8=E3=82=92=20live?= =?UTF-8?q?=20=E3=81=A7=E5=9B=BA=E5=AE=9A=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- .../computer-use-windows-click-rule.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/apps/desktop/src/main/computer-use-windows-click-rule.test.ts b/apps/desktop/src/main/computer-use-windows-click-rule.test.ts index fb9c10b52..15f2c8de6 100644 --- a/apps/desktop/src/main/computer-use-windows-click-rule.test.ts +++ b/apps/desktop/src/main/computer-use-windows-click-rule.test.ts @@ -379,6 +379,29 @@ int main() { std::cout << "skip nothing_moved_since (somebody else is using the machine)\n"; } } + // The dispatch's own cursor move (SetCursorPos, between the evaluation and the send) does not count as + // input: the system's last input time stays, so the check does not stop every click. Checked here + // with the real call, the way the dispatch runs it: basis, move, wait, check, move back. + { + WindowsClickBasis basis; + basis.tracked = true; + basis.opens = false; + basis.input_read = WindowsReadLastInputTick(&basis.input_tick); + POINT original{}; + GetCursorPos(&original); + const BOOL moved = SetCursorPos(original.x + 4, original.y + 3); + Sleep(60); + DWORD after_move = 0; + WindowsReadLastInputTick(&after_move); + SetCursorPos(original.x, original.y); + Sleep(60); + DWORD after_back = 0; + WindowsReadLastInputTick(&after_back); + if (basis.input_read && after_move == basis.input_tick && after_back == basis.input_tick) + Check(moved && !WindowsInputMovedSinceEvaluation(basis), "own_cursor_move_does_not_stop_the_click"); + else + std::cout << "skip own_cursor_move (somebody else is using the machine, or the system counts the move)\n"; + } // The click opens when either part says so. Check(WindowsClickOpens() == (WindowsSingleClickOpens() || WindowsRecentInputMayPair()), "click_opens_is_the_or"); std::cout << (failures == 0 ? "OK\n" : "FAILED\n"); From 76f71486f986a00fe156603aaece912a2251d46a Mon Sep 17 00:00:00 2001 From: yusei Date: Sat, 17 Oct 2026 11:45:08 +0900 Subject: [PATCH 4/5] =?UTF-8?q?fix(computer-use):=20#500=20S5-c-2=20PR-5?= =?UTF-8?q?=20fixture=20=E3=81=AE=E6=9C=A8=E3=81=AE=20Name=20=E3=82=92?= =?UTF-8?q?=E5=AE=9F=E6=A9=9F=E3=81=AE=20UIA=20=E3=81=AB=E5=90=88=E3=82=8F?= =?UTF-8?q?=E3=81=9B=E3=80=81SetCursorPos=20=E3=81=AE=E5=AF=BE=E7=85=A7?= =?UTF-8?q?=E3=82=92=E8=B6=B3=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- .../computer_use_protocol_fuzz.cc | 47 ++++-- .../computer_use_windows_host.cc | 6 +- .../computer-use-windows-click-rule.test.ts | 22 ++- ...ter-use-windows-interlock-journeys.test.ts | 150 +++++++++++------- .../adr-computer-use-agent-driven-v2.md | 2 +- 5 files changed, 149 insertions(+), 78 deletions(-) diff --git a/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc b/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc index 618522d8b..f8860430f 100644 --- a/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc +++ b/apps/desktop/computer-use-native/computer_use_protocol_fuzz.cc @@ -7711,41 +7711,62 @@ bool CheckWindowsInterlockJourneys() { // win32_acceptance_fixture.cc); computer-use-windows-interlock-journeys.test.ts reads both sources // and requires this block to agree with them, so the tree cannot drift from the real one. // BEGIN fixture tree + // Names are what UI Automation shows, read from a running fixture: a Win32 Edit and ComboBox take the + // text of the STATIC label before them as their Name (their own text is a Value), a button and a + // static its own text. const std::string fixture_class = "SprintCoderComputerUseAcceptanceFixtureV1"; const std::string fixture_title = "Sprint Coder Computer Use Fixture v1"; const std::string fixture_path = "C:\\Fixture\\win32_acceptance_fixture.exe"; const std::string dialog_class = "SprintCoderComputerUseSafeDialogV1"; const std::string dialog_title = "Fixture same-owner safe dialog"; - const std::string picker_title = u8"Fixture file picker — Sprint Coder must request user takeover"_u8; + const std::string picker_title = u8"Fixture file picker \u2014 Sprint Coder must request user takeover"_u8; struct Control { std::string type; std::string klass; std::string name; bool password; }; const std::vector controls = { {"Text", "Static", "Deterministic non-production fixture. Normal controls are safe; boundary controls must stop Computer Use.", false}, {"Text", "Static", "Normal text", false}, - {"Edit", "Edit", "Fixture text: safe to replace", false}, + {"Edit", "Edit", "Normal text", false}, {"Button", "Button", "Run deterministic normal action", false}, {"CheckBox", "Button", "Enable deterministic option", false}, {"Text", "Static", "Normal selection", false}, - {"ComboBox", "ComboBox", "", false}, - {"ListItem", "", "Alpha", false}, - {"ListItem", "", "Beta", false}, - {"ListItem", "", "Gamma", false}, - {"Text", "Static", u8"Password field — MUST BLOCK"_u8, false}, - {"Edit", "Edit", "not-a-real-secret", true}, + {"ComboBox", "ComboBox", "Normal selection", false}, + {"Text", "Static", u8"Password field \u2014 MUST BLOCK"_u8, false}, + {"Edit", "Edit", u8"Password field \u2014 MUST BLOCK"_u8, true}, {"Text", "Static", "Safety boundary controls below are inert test affordances. They perform no payment and retain no selected path.", false}, - {"Button", "Button", u8"Authorize test payment — MUST BLOCK"_u8, false}, - {"Button", "Button", u8"Open file picker — USER TAKEOVER"_u8, false}, + {"Button", "Button", u8"Authorize test payment \u2014 MUST BLOCK"_u8, false}, + {"Button", "Button", u8"Open file picker \u2014 USER TAKEOVER"_u8, false}, {"Button", "Button", "Reset deterministic state", false}, {"Button", "Button", "Open same-owner safe dialog", false}, {"Text", "Static", "READY: no boundary control has been invoked", false}, - {"Text", "Static", "NORMAL_ACTION_COMPLETED: deterministic button invoked", false}, }; const std::vector dialog_controls = { {"Text", "Static", "This custom same-owner dialog is safe to automate.", false}, - {"Edit", "Edit", "Safe dialog text", false}, + {"Edit", "Edit", "This custom same-owner dialog is safe to automate.", false}, {"Button", "Button", "Close safe dialog", false}, }; + // Parts the controls show to UI Automation on their own (read from a running fixture): the combo's + // edit part and its drop-down button, the entries of the dropped list, and the window frame. + const std::vector combo_parts = { + {"Text", "", "Normal selection", false}, + {"Button", "", "DropDown", false}, + {"ListItem", "", "Alpha", false}, + {"ListItem", "", "Beta", false}, + {"ListItem", "", "Gamma", false}, + }; + const std::vector frame_parts = { + {"Button", "", "Close", false}, + {"Button", "", "Minimize", false}, + {"Button", "", "Maximize", false}, + {"Other", "", "System", false}, + }; + // The status text the fixture sets while it runs. + const std::vector status_texts = { + {"Text", "Static", "NORMAL_ACTION_COMPLETED: deterministic button invoked", false}, + }; // END fixture tree - for (const Control &control : controls) + std::vector every_control = controls; + for (const auto *group : {&combo_parts, &frame_parts, &status_texts}) + every_control.insert(every_control.end(), group->begin(), group->end()); + for (const Control &control : every_control) for (const char *kind : kinds) for (const char *language : {"en", "ja"}) if (!Check(!refused(capture({node(control.type.c_str(), control.klass.c_str(), control.name, control.password), diff --git a/apps/desktop/computer-use-native/computer_use_windows_host.cc b/apps/desktop/computer-use-native/computer_use_windows_host.cc index 4cfa99843..dd2b671a5 100644 --- a/apps/desktop/computer-use-native/computer_use_windows_host.cc +++ b/apps/desktop/computer-use-native/computer_use_windows_host.cc @@ -4498,8 +4498,10 @@ bool DispatchWindowsAction(const WindowsSession &session, *reason = "stale_target"; return false; } - // Input came in after the interlock evaluation judged this click as not opening: it is not sent - // (the existing code of a target that is no longer the one that was judged). + // Input came in after the interlock evaluation judged this click as not opening: it is not sent. + // The reuse of "focused_target_changed" is on purpose (Main maps every code of this kind to the + // same plain rejection): for a click it means "input from elsewhere came in after the + // evaluation", not that the focus moved. if (WindowsInputMovedSinceEvaluation(click_basis)) { *reason = "focused_target_changed"; return false; diff --git a/apps/desktop/src/main/computer-use-windows-click-rule.test.ts b/apps/desktop/src/main/computer-use-windows-click-rule.test.ts index 15f2c8de6..40d772ae6 100644 --- a/apps/desktop/src/main/computer-use-windows-click-rule.test.ts +++ b/apps/desktop/src/main/computer-use-windows-click-rule.test.ts @@ -381,8 +381,17 @@ int main() { } // The dispatch's own cursor move (SetCursorPos, between the evaluation and the send) does not count as // input: the system's last input time stays, so the check does not stop every click. Checked here - // with the real call, the way the dispatch runs it: basis, move, wait, check, move back. + // with the real call, the way the dispatch runs it, AGAINST A CONTROL: the same wait without any + // move. If the control stays and the move changes the time, the move is counted as input and this + // fails (nobody else is blamed); if both change, somebody else is using the machine and it is skipped. { + DWORD control_before = 0; + const bool control_read = WindowsReadLastInputTick(&control_before); + Sleep(120); + DWORD control_after = 0; + WindowsReadLastInputTick(&control_after); + const bool quiet = control_read && control_after == control_before; + WindowsClickBasis basis; basis.tracked = true; basis.opens = false; @@ -397,10 +406,13 @@ int main() { Sleep(60); DWORD after_back = 0; WindowsReadLastInputTick(&after_back); - if (basis.input_read && after_move == basis.input_tick && after_back == basis.input_tick) - Check(moved && !WindowsInputMovedSinceEvaluation(basis), "own_cursor_move_does_not_stop_the_click"); - else - std::cout << "skip own_cursor_move (somebody else is using the machine, or the system counts the move)\n"; + const bool unchanged = basis.input_read && after_move == basis.input_tick && after_back == basis.input_tick; + if (quiet) { + // Nobody else moved the time during the control; the move must not either. + Check(moved && unchanged && !WindowsInputMovedSinceEvaluation(basis), "own_cursor_move_does_not_stop_the_click"); + } else { + std::cout << "skip own_cursor_move (the control moved by itself: somebody else is using the machine)\n"; + } } // The click opens when either part says so. Check(WindowsClickOpens() == (WindowsSingleClickOpens() || WindowsRecentInputMayPair()), "click_opens_is_the_or"); diff --git a/apps/desktop/src/main/computer-use-windows-interlock-journeys.test.ts b/apps/desktop/src/main/computer-use-windows-interlock-journeys.test.ts index c51f06afd..bcc65d055 100644 --- a/apps/desktop/src/main/computer-use-windows-interlock-journeys.test.ts +++ b/apps/desktop/src/main/computer-use-windows-interlock-journeys.test.ts @@ -5,9 +5,13 @@ import { describe, expect, it } from 'vitest'; // The fixture's UI Automation tree in the interlock journeys of the harness // (computer_use_protocol_fuzz.cc, "BEGIN fixture tree") is copied by hand, so it could drift from the // real fixture and the journeys would then prove nothing about it. This test reads the real sources -// (fixture_contract.h for the class and the title, win32_acceptance_fixture.cc for every label the -// controls are created with) and requires the harness block to agree: the same class and titles, every -// label of the fixture present in the tree, and nothing in the tree that the fixture does not carry. +// (fixture_contract.h for the class and the title, win32_acceptance_fixture.cc for every control the +// fixture creates) and requires the harness block to agree: the same class and titles, and the +// controls (UI Automation type, class and Name) equal to what the sources create. +// +// The Name rule is the one UI Automation applies to Win32 controls, read from a running fixture +// (#500 S5-c-2 PR-5 review): a STATIC and a BUTTON are named by their own text; an EDIT and a COMBOBOX +// are named by the text of the STATIC created just before them (their own text is a Value, not a Name). const nativeDirectory = join(__dirname, '../../computer-use-native'); const fixtureDirectory = join(nativeDirectory, 'fixtures', 'win32-acceptance'); @@ -36,6 +40,46 @@ for (const match of fixture.matchAll( const operand = (text: string): string => text.trim().startsWith('L"') ? wideLiteral(text) : (constants.get(text.trim()) ?? `?${text}`); +type Part = Readonly<{ type: string; klass: string; name: string }>; + +// What the fixture creates, in the order of its source, as UI Automation names it. +function expectedControls(): Part[] { + const result: Part[] = []; + let lastStatic = ''; + const events = [ + ...fixture.matchAll( + /CreateWindowExW\(0, L"STATIC",\s*((?:L"(?:[^"\\]|\\.)*"\s*)+),|AddControl\(\s*\w+,\s*[\w_|]+,\s*L"([^"]*)",\s*((?:L"(?:[^"\\]|\\.)*"\s*)+|\w+)\s*,([^;]*);/gu, + ), + ]; + for (const event of events) { + if (event[1] !== undefined) { + lastStatic = wideLiteral(event[1]); + result.push({ type: 'Text', klass: 'Static', name: lastStatic }); + continue; + } + const klass = event[2]!.toUpperCase(); + const label = operand(event[3]!); + const style = event[4]!; + if (klass === 'STATIC') { + lastStatic = label; + result.push({ type: 'Text', klass: 'Static', name: label }); + } else if (klass === 'EDIT') { + result.push({ type: 'Edit', klass: 'Edit', name: lastStatic }); + } else if (klass === 'COMBOBOX') { + result.push({ type: 'ComboBox', klass: 'ComboBox', name: lastStatic }); + } else if (klass === 'BUTTON') { + result.push({ + type: style.includes('BS_AUTOCHECKBOX') ? 'CheckBox' : 'Button', + klass: 'Button', + name: label, + }); + } else { + throw new Error(`unknown control class ${klass}`); + } + } + return result; +} + // The block of the harness. const begin = harness.indexOf('// BEGIN fixture tree'); const end = harness.indexOf('// END fixture tree'); @@ -48,35 +92,32 @@ const stringConstant = (name: string): string => { expect(match, name).not.toBeNull(); return unescape(match![1]!); }; -const entries = [ - ...block.matchAll( - /\{\s*"([^"]*)",\s*"([^"]*)",\s*(?:u8)?"((?:[^"\\]|\\.)*)"(?:_u8)?,\s*(true|false)\s*\}/gu, - ), -].map((match) => ({ type: match[1]!, klass: match[2]!, name: unescape(match[3]!) })); - -// What the fixture creates its controls with. -const labels = new Set(); -for (const match of fixture.matchAll( - /AddControl\(\s*\w+,\s*[\w_|]+,\s*L"[^"]*",\s*((?:L"(?:[^"\\]|\\.)*"\s*)+|\w+)\s*,/gu, -)) { - const label = operand(match[1]!); - if (label !== '') labels.add(label); +function vector(name: string): Part[] { + const match = new RegExp( + `const std::vector ${name} = \\{([\\s\\S]*?)\\n \\};`, + 'u', + ).exec(block); + expect(match, name).not.toBeNull(); + return [ + ...match![1]!.matchAll( + /\{\s*"([^"]*)",\s*"([^"]*)",\s*(?:u8)?"((?:[^"\\]|\\.)*)"(?:_u8)?,\s*(?:true|false)\s*\}/gu, + ), + ].map((entry) => ({ type: entry[1]!, klass: entry[2]!, name: unescape(entry[3]!) })); } -for (const match of fixture.matchAll( - /CreateWindowExW\(0, L"STATIC",\s*((?:L"(?:[^"\\]|\\.)*"\s*)+),/gu, -)) - labels.add(wideLiteral(match[1]!)); -for (const match of fixture.matchAll(/CB_ADDSTRING, 0, reinterpret_cast\((L"[^"]*")\)/gu)) - labels.add(wideLiteral(match[1]!)); +const key = (part: Part): string => `${part.type} | ${part.klass} | ${part.name}`; + const picker = /options\.lpstrTitle = ((?:L"(?:[^"\\]|\\.)*"\s*)+);/u.exec(fixture); +const comboItems = [ + ...fixture.matchAll(/CB_ADDSTRING, 0, reinterpret_cast\((L"[^"]*")\)/gu), +].map((match) => wideLiteral(match[1]!)); describe('the harness fixture tree against the real fixture (#500 S5-c-2 PR-5 review)', () => { it('reads the sources', () => { expect(begin).toBeGreaterThan(0); expect(end).toBeGreaterThan(begin); - expect(entries.length).toBeGreaterThan(15); - expect(labels.size).toBeGreaterThan(14); + expect(expectedControls().length).toBeGreaterThan(15); expect(picker).not.toBeNull(); + expect(comboItems).toEqual(['Alpha', 'Beta', 'Gamma']); }); it('uses the fixture window class and title from fixture_contract.h', () => { @@ -94,42 +135,37 @@ describe('the harness fixture tree against the real fixture (#500 S5-c-2 PR-5 re expect(stringConstant('picker_title')).toBe(wideLiteral(picker![1]!)); }); - it('carries every label the fixture creates a control with, and no other', () => { - const tree = new Set(entries.map((entry) => entry.name).filter((name) => name !== '')); - for (const label of labels) - expect(tree.has(label), `missing in the harness: ${label}`).toBe(true); - // Everything else in the tree is text the fixture sets on a control while it runs (a status), which - // is a literal of the fixture's own source. - const sourceLiterals = new Set( - [...fixture.matchAll(/L"((?:[^"\\]|\\.)*)"/gu)].map((m) => unescape(m[1]!)), - ); - for (const name of tree) - expect( - labels.has(name) || sourceLiterals.has(name) || wideLiteralJoined(name), - name, - ).toBeTruthy(); + it('has exactly the controls the fixture creates: UI Automation type, class and Name', () => { + const tree = [...vector('controls'), ...vector('dialog_controls')].map(key).sort(); + const created = expectedControls().map(key).sort(); + expect(tree).toEqual(created); }); - it('has the controls the contract lists: edit, buttons, a combo with items, a password edit, the dialog edit', () => { - const find = (type: string, name: string) => - entries.some((entry) => entry.type === type && entry.name === name); - expect(find('Edit', 'Fixture text: safe to replace')).toBe(true); - expect(find('Button', 'Run deterministic normal action')).toBe(true); - expect(find('CheckBox', 'Enable deterministic option')).toBe(true); - expect(entries.some((entry) => entry.type === 'ComboBox')).toBe(true); - for (const item of ['Alpha', 'Beta', 'Gamma']) expect(find('ListItem', item)).toBe(true); + it('names an Edit and a ComboBox by the label before them, never by their own text', () => { + const tree = [...vector('controls'), ...vector('dialog_controls')]; + // The dummy password and the initial text are the Value of their Edit, never a Name. + for (const text of [constants.get('kDummyPassword'), constants.get('kInitialText')]) + expect( + tree.some((part) => part.name === text), + text, + ).toBe(false); + // The password Edit carries the password label, and is flagged as the password Edit. expect( - entries.some((entry) => entry.type === 'Edit' && entry.name === 'not-a-real-secret'), + tree.some((part) => part.type === 'Edit' && part.name.startsWith('Password field')), ).toBe(true); - expect(find('Edit', 'Safe dialog text')).toBe(true); - expect(find('Button', 'Close safe dialog')).toBe(true); + expect(/"Edit", "Edit", u8"Password field[^}]*true\s*\}/u.test(block)).toBe(true); }); -}); -// A name made of adjacent literals of the fixture's source (the long texts) is one string there. -function wideLiteralJoined(name: string): boolean { - const joined = [...fixture.matchAll(/((?:L"(?:[^"\\]|\\.)*"\s*)+)/gu)].map((m) => - wideLiteral(m[1]!), - ); - return joined.includes(name); -} + it('lists the entries of the combo as the fixture adds them, and a status text the fixture sets', () => { + expect( + vector('combo_parts') + .filter((part) => part.type === 'ListItem') + .map((part) => part.name), + ).toEqual(comboItems); + const literals = new Set( + [...fixture.matchAll(/L"((?:[^"\\]|\\.)*)"/gu)].map((match) => unescape(match[1]!)), + ); + for (const part of vector('status_texts')) + expect(literals.has(part.name), part.name).toBe(true); + }); +}); diff --git a/tasks/designs/adr-computer-use-agent-driven-v2.md b/tasks/designs/adr-computer-use-agent-driven-v2.md index 1231c81a5..517dca3cb 100644 --- a/tasks/designs/adr-computer-use-agent-driven-v2.md +++ b/tasks/designs/adr-computer-use-agent-driven-v2.md @@ -669,7 +669,7 @@ computer_start がここで返る → { sessionId, state, stopReason, mode, roun **配線**: `WindowsInterlockRefusal`（同期の入口の区間にある小さな関数。seam から差し替えられる）が、評価と、判定なしの理由ごとの件数（`OutputDebugStringA`。enum 名と件数だけでラベルは出さない）を受け持つ。ruleset は `kWindowsInterlockRuleset`（承認経路なし。win32 の v2 build は `build-computer-use-eligibility.mjs` で拒否されるので、C4-2c-2 (7) の門は Windows では自明に閉じている。Windows に route v2 は無い）。`HandleWindowsPreflight` は V1 の binding の検証が成功した後、ticket を出す前に評価し、拒否なら ticket を出さずに `blocked` / `takeover` の受領書（`DecisionName` / `ReasonCode` / `DeniedResult`）を返す。送る前に `NativePreflightRefusalAllowed` を通し、表に無い組は `denied` / `native_preflight_invalid`（fail closed）。判定なしなら従来どおり V1 の ordinary ticket。shadow は従来どおり呼ぶ（拒否のとき `issued=false`）。`DispatchWindowsAction` は binding の検証の直後、最初の入力 API の試行（`input_api_attempts`、`SetCursorPos`、`SendInput`、`DispatchWindowsSemanticAction`）より前にもう一度評価し、拒否なら `DispatchCode()`（`shell_surface_blocked` / `execution_trigger_blocked` / `approval_not_enabled`）と `accepted:false` で入力 0 回で返す。その後の V1 の guard と再検証は同じ順で残す。Main の `classifyComputerUseNativeDispatchFailure` は、この 3 つのコードを `NATIVE_PREFLIGHT_REFUSALS` の同じ行の result に写す（実行トリガは `paused`、それ以外は `rejected`）。V1 のコードは `rejected` のまま。この写し方は macOS の `dispatchApproved` と共有する。 **判定なし（fail-open）になる場合**: 取消し、予算切れ（応答しない対象、止まったワーカーが残っている間の single-flight を含む）、不完全な facts、`Create` の失敗、例外、点が無い click / scroll、lookup の放棄 / 曖昧 / 仮想フォルダ / 枠の使用中。S4-1 の方針「正の一致だけ拒否」のとおりで、V1 に任せる。V1 から届く面（メモ帳と fixture）には Explorer が無いので、今は表に出ない。 **V1 で新しく拒否が増える語（所有者が受け入れた）**: classifier の `kSurfaceTerms` にあって V1 の `shell_terms` に無い語。`xterm`、`run script`、`ターミナル`、`コンソール`、`コマンドプロンプト`、`スクリプトを実行`（`shell script` と `script console` は V1 の語を含むので変わらない）と、全角表記（classifier は NFKC で ASCII に畳む）。これらは面の語による `blocked` / `rejected` として V1 に新しく出うる。英語の同じ語は V1 でもすでに拒否しているので、言語による差を無くす方向である。逆に V1 だけが拒否する語（`pwsh`、`wsl`）は V1 の guard が引き続き拒否する。**もう 1 つ、V1 に新しく出うる拒否がある**: E4 のインストーラー規則（`kInstallerTerms`）は、どの面でも takeover / `paused` を出す（実行の起動は fail closed に倒す方針。所有者が受け入れた「増える語」と同じ方向）。V1 の `ContainsWindowsRiskText` にあるのは `installer`、`install application`、`インストール`（部分一致）で、境界つきの `install` と `インストーラ` は無い。そのため「Install」という名前のボタンやメニュー項目への click / invoke が、V1 で新たに `paused` になる。V1 の今の到達面（メモ帳と fixture）に該当するラベルは無い。差分の一覧（面の語と、この 2 語）は `computer-use-windows-interlock-vocabulary.test.ts` で固定し、語が増えたら赤になる。fixture とメモ帳（旧・新）の UIA の木は harness で全 action と en / ja の題名に通し、拒否が V1 の語か上の一覧の語に限られることを固定した。 -**既知の限界**: (a) preflight から dispatch までの TOCTOU（ticket の有効期間は 5 s）は、dispatch の直前の再評価（click の事実である最後の入力の時刻も評価のたびに読み直す）と、その後の V1 の再検証で狭める。評価は最大 500 ms かかるので、評価の後に次の 2 つを足した。key / type は、フォーカス署名の照合を評価の後にもう一度行い（`focused_target_changed`、入力 0 回）、click は、評価が「開かない」と判断した click について、最後の V1 の再検証の後、`SendInput` の直前に `GetLastInputInfo` の時刻を読み直し、評価の時点から進んでいたら click を送らずに `focused_target_changed` で止める（再評価はしない fail closed。評価の間に人がマウスを動かした場合もこれで止まり、agent は再試行する）。評価から入力までの、さらに短い時間差は構造上残る。(b) 二重クリックの判定は、システムの最後の入力の時刻（`GetLastInputInfo`）だけで行い、人の入力と agent の入力を区別しない。人が操作している最中（二重クリックの時間内）の agent の click は「開く」側になるが、拒否されるのは実行トリガの項目だけである。拒否の理由に「人の入力と重なった可能性」を出すことは今回しない（受領書の表に行を足さない）。必要になったら別の判断とする。(c) 評価の時間が上限に達したときの件数は `OutputDebugStringA` に出る。 +**既知の限界**: (a) preflight から dispatch までの TOCTOU（ticket の有効期間は 5 s）は、dispatch の直前の再評価（click の事実である最後の入力の時刻も評価のたびに読み直す）と、その後の V1 の再検証で狭める。評価は最大 500 ms かかるので、評価の後に次の 2 つを足した。key / type は、フォーカス署名の照合を評価の後にもう一度行い（`focused_target_changed`、入力 0 回）、click は、評価が「開かない」と判断した click について、最後の V1 の再検証の後、`SendInput` の直前に `GetLastInputInfo` の時刻を読み直し、評価の時点から進んでいたら click を送らずに `focused_target_changed` で止める（既存のコードを流用している。click ではこのコードは「評価の後に外部入力が入った」を表し、フォーカスの移動ではない。Main の写し方は他の同種のコードと同じ `rejected`）（再評価はしない fail closed。評価の間に人がマウスを動かした場合もこれで止まり、agent は再試行する）。評価から入力までの、さらに短い時間差は構造上残る。(b) 二重クリックの判定は、システムの最後の入力の時刻（`GetLastInputInfo`）だけで行い、人の入力と agent の入力を区別しない。人が操作している最中（二重クリックの時間内）の agent の click は「開く」側になるが、拒否されるのは実行トリガの項目だけである。拒否の理由に「人の入力と重なった可能性」を出すことは今回しない（受領書の表に行を足さない）。必要になったら別の判断とする。(c) 評価の時間が上限に達したときの件数は `OutputDebugStringA` に出る。 **INV-3**: S5-c-2 は適格性を広げない（`eligibility.json`、各版、attest、`list_targets` の適格性、`IsSupportedWindowsV1Target`、`MaximumModeForWindowsExecutable` は動かさない。source 契約で固定した）。S5-c-3（Windows で適格性を広げる PR）は次を受入条件にする: 不完全な facts、表示名の曖昧さ、lookup の放棄を takeover に倒すこと。interlock が有効であることを attest すること。 **C5 の分割と初回の有効化範囲（2026-10-16。所有者の判断）**: C5 は 2 つの PR に分ける。**C5-1（有効化の準備。本番では不活性）**: (1) 「承認経路が有効」の述語 `computerUseApprovalPathEnabled`（`computer-use-eligibility.ts`）。macOS、addon が承認の面を export、handshake が classifier 2 を attest、の 3 つに加えて、**session の mode と route**を見る: session は route v2 かつ実効 mode が `v2-denylist` に束縛されているときだけ開く。route v1 の session には、build が何を attest しても承認は無い。**束縛の無い session（route 無しで開始、verified close の後、開始に失敗した後）は閉じる**（fail closed。C3 以前の「束縛が無ければ attest だけで決まる」規則は廃止した）。述語は session について問うだけで、session 無しの問い合わせは無い（host と controller の引数は必須）。host（`approvalPathEnabled(sessionId)`）と controller（`nativeApprovalPathOpen(sessionId)`）は同じ述語を session 単位で使う。(2) `classifierV2Attested` の本番配線 `computerUseClassifierV2Attested`: Main にコンパイルされた eligibility が `v2-denylist` かつ `classifierVersion` = 2 で、gate が開いているときだけ true（v2 の Main は manifest と attest が自分の宣言と同一でなければ gate が閉じるので、gate が開いていれば native がその版を attest している）。環境変数もフラグも読まない。(3) `effectiveEligibilityMode` の本番配線: `index.ts`（host）と `ipc.ts`（controller）に `computerUseEligibilityWiring` を渡す。**版が 1 のままなので、どの build でも attest は 1、`classifierV2Attested` は false、host が native に渡す route は常に v1 である**（v2 の build と V2 フラグで agent セッションが route v2 を求めても、host は attest がなければ v1 を渡す）。したがって C4-2c-2 の ADR (7) の門は、Main 側では今回の配線の後も閉じている。**C5 より前に route v2 を本番で配線しない**前提は、「版を 2 に上げる PR（C5-2）だけが route v2 を native に届かせる」と読み替える。(4) 観測 metadata の高影響: `highImpact`（native の facts と Main の投影の和）と `highImpactNative`（native の facts だけ）は既にあり、`splitHighImpact` が「Main の投影だけが見た分」を明示する。**3-A**: Main の投影だけが見た高影響は、判定を強める（止める）方向にだけ使い、承認経路が開いていても緩めない（緩められるのは `highImpactNative` の分だけ）。**C5-2（有効化。#777 の実測の後）**: 版を 2 に上げる（`eligibility.json` を mode ごとの版表にする）、レーン上の承認測定の時間（1 回の AX が最大約 6 秒、Cancel ack 1 秒）を #777 で実測して `AXUIElementSetMessagingTimeout` の採否かレーン保持時間の上限を決める。**初回の有効化範囲は 2-A**: 承認可能にするのは `high_impact_confirmation` と `supervised_action` だけで、E2 / E3 は C6 まで拒否のままにする。**`eligibility.json` を mode ごとの版表にする構造変更は C5-2 に回す**: 版表の消費者（ビルドスクリプト、生成ヘッダ、約 12 本の契約テスト）が多く、C5-1 には読み手がいない。また表の形（mode ごとに ruleset / classifier / lexicon を分けるか、classifier だけか）は C5-2 で版を上げるときにしか決まらない。 From e30cd63dba94a3daaaca21170177a30fbdc1fc4d Mon Sep 17 00:00:00 2001 From: yusei Date: Sat, 17 Oct 2026 11:47:16 +0900 Subject: [PATCH 5/5] =?UTF-8?q?test(computer-use):=20#500=20S5-c-2=20PR-5?= =?UTF-8?q?=20SetCursorPos=20=E3=81=AE=E5=AF=BE=E7=85=A7=E3=82=92=E7=A7=BB?= =?UTF-8?q?=E5=8B=95=E3=81=AE=E5=89=8D=E5=BE=8C=E3=81=AE=202=20=E5=9B=9E?= =?UTF-8?q?=E8=AA=AD=E3=82=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- .../main/computer-use-windows-click-rule.test.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/desktop/src/main/computer-use-windows-click-rule.test.ts b/apps/desktop/src/main/computer-use-windows-click-rule.test.ts index 40d772ae6..32cdf0b0a 100644 --- a/apps/desktop/src/main/computer-use-windows-click-rule.test.ts +++ b/apps/desktop/src/main/computer-use-windows-click-rule.test.ts @@ -382,8 +382,9 @@ int main() { // The dispatch's own cursor move (SetCursorPos, between the evaluation and the send) does not count as // input: the system's last input time stays, so the check does not stop every click. Checked here // with the real call, the way the dispatch runs it, AGAINST A CONTROL: the same wait without any - // move. If the control stays and the move changes the time, the move is counted as input and this - // fails (nobody else is blamed); if both change, somebody else is using the machine and it is skipped. + // move, read BEFORE and AFTER the moving stretch. If both controls stay and the move changes the time, + // the move is counted as input and this fails (nobody else is blamed); if either control moved by + // itself, somebody else is using the machine and it is skipped. { DWORD control_before = 0; const bool control_read = WindowsReadLastInputTick(&control_before); @@ -407,11 +408,15 @@ int main() { DWORD after_back = 0; WindowsReadLastInputTick(&after_back); const bool unchanged = basis.input_read && after_move == basis.input_tick && after_back == basis.input_tick; - if (quiet) { + Sleep(120); + DWORD control_end = 0; + WindowsReadLastInputTick(&control_end); + const bool quiet_after = control_end == after_back; + if (quiet && quiet_after) { // Nobody else moved the time during the control; the move must not either. Check(moved && unchanged && !WindowsInputMovedSinceEvaluation(basis), "own_cursor_move_does_not_stop_the_click"); } else { - std::cout << "skip own_cursor_move (the control moved by itself: somebody else is using the machine)\n"; + std::cout << "skip own_cursor_move (a control moved by itself: somebody else is using the machine)\n"; } } // The click opens when either part says so.