From 88cbbb6af3e61ee616e67469d20e050f2574e411 Mon Sep 17 00:00:00 2001 From: longxboy Date: Sun, 7 Jun 2026 13:58:12 +0800 Subject: [PATCH 1/2] fix(send_message): pick active composer when ghost frames exist WeChat's accessibility tree can contain multiple edit+send pairs: the live main-window composer plus stale "ghost" frames left behind by chats previously detached into separate windows. The old depth-first "take the first pair" logic grabbed the wrong (ghost) composer, whose input never received text and whose Send button stayed DISABLED forever, causing the plan to loop and ultimately fail with "No action selected". Collect every candidate edit+send pair and rank them so the genuinely active composer wins: 1. editable currently FOCUSED (strongest signal) 2. Send button NOT disabled (composer already has text) 3. pair under the main "Weixin" frame (not a ghost/detached frame) DFS order breaks any remaining ties. This makes 'wx messages send' reliable even when stale frames are present, removing the need to restart the container to clear ghost windows. --- .../src/plans/send_message.rs | 108 +++++++++++++----- 1 file changed, 79 insertions(+), 29 deletions(-) diff --git a/packages/agent-server-rust/src/plans/send_message.rs b/packages/agent-server-rust/src/plans/send_message.rs index 815f8d64..9708c5a6 100644 --- a/packages/agent-server-rust/src/plans/send_message.rs +++ b/packages/agent-server-rust/src/plans/send_message.rs @@ -29,47 +29,97 @@ pub struct SendMessagePlanState { pub confirm_attempts: u32, } -fn find_edit_and_send_button(a11y: &A11yNode) -> Option<(&A11yNode, &A11yNode)> { - let send_btn = query_selector(a11y, r#"push-button[name="Send(S)"]"#)?; - // Find sibling EDITABLE text node via parent - // Since we don't have parent refs in the tree-based approach, - // we search the tree for the pattern - find_edit_near_send(a11y, send_btn) +fn node_has_state(node: &A11yNode, state: &str) -> bool { + node.states + .as_ref() + .map(|s| s.iter().any(|st| st == state)) + .unwrap_or(false) +} + +/// A candidate composer: the editable text input plus its sibling Send(S) button. +struct ComposerPair<'a> { + edit: &'a A11yNode, + send: &'a A11yNode, + /// True if this pair lives under the main "Weixin" application frame + /// (as opposed to a detached/ghost chat frame leftover in the a11y tree). + in_main_frame: bool, } -fn find_edit_near_send<'a>( - root: &'a A11yNode, - _send_btn: &A11yNode, -) -> Option<(&'a A11yNode, &'a A11yNode)> { - // Walk tree looking for a parent that has both an EDITABLE text and Send(S) button - find_edit_send_pair(root) +/// Find the composer (editable + Send button) to operate on. +/// +/// WeChat's accessibility tree can contain *multiple* edit+send pairs: +/// the live main-window composer plus stale "ghost" frames left behind by +/// chats that were previously detached into separate windows. A naive +/// depth-first "take the first pair" grabs the wrong (ghost) composer, whose +/// input never receives text and whose Send stays DISABLED forever, causing +/// the plan to loop and ultimately fail with "No action selected". +/// +/// To be robust we collect every candidate pair, then rank them so the +/// genuinely active composer wins: +/// 1. editable currently FOCUSED (strongest signal) +/// 2. Send button NOT disabled (composer already has text) +/// 3. pair under the main "Weixin" frame (not a ghost/detached frame) +/// The first pair (DFS order) breaks any remaining ties. +fn find_edit_and_send_button(a11y: &A11yNode) -> Option<(&A11yNode, &A11yNode)> { + let mut candidates: Vec = Vec::new(); + collect_edit_send_pairs(a11y, false, &mut candidates); + + if candidates.is_empty() { + return None; + } + + // Pick the best candidate by preference, preserving DFS order on ties. + let best = candidates.iter().enumerate().min_by_key(|(idx, c)| { + let focused = node_has_state(c.edit, "FOCUSED"); + let send_enabled = !node_has_state(c.send, "DISABLED"); + // Lower key = higher priority. Each desirable property subtracts rank. + let mut score: i32 = 0; + if focused { + score -= 100; + } + if send_enabled { + score -= 10; + } + if c.in_main_frame { + score -= 1; + } + // Tie-break: earlier DFS position wins. + (score, *idx as i32) + }); + + best.map(|(_, c)| (c.edit, c.send)) } -fn find_edit_send_pair(node: &A11yNode) -> Option<(&A11yNode, &A11yNode)> { +/// Recursively collect all edit+send composer pairs, tracking whether each +/// pair is inside the main "Weixin" frame. +fn collect_edit_send_pairs<'a>( + node: &'a A11yNode, + in_main_frame: bool, + out: &mut Vec>, +) { + // Once we enter the main "Weixin" frame, everything below it is in-main. + let in_main_frame = in_main_frame || (node.role == "frame" && node.name == "Weixin"); + if let Some(children) = &node.children { - let send_btn = children.iter().find(|c| { - c.role == "push-button" && c.name == "Send(S)" - }); - let edit_node = children.iter().find(|c| { - c.role == "text" - && c.states - .as_ref() - .map(|s| s.iter().any(|st| st == "EDITABLE")) - .unwrap_or(false) - }); + let send_btn = children + .iter() + .find(|c| c.role == "push-button" && c.name == "Send(S)"); + let edit_node = children + .iter() + .find(|c| c.role == "text" && node_has_state(c, "EDITABLE")); if let (Some(edit), Some(send)) = (edit_node, send_btn) { - return Some((edit, send)); + out.push(ComposerPair { + edit, + send, + in_main_frame, + }); } - // Recurse for child in children { - if let Some(result) = find_edit_send_pair(child) { - return Some(result); - } + collect_edit_send_pairs(child, in_main_frame, out); } } - None } #[async_trait::async_trait] From 209cb0230d98ddfb8c2a4d9e1720e3473979a6b1 Mon Sep 17 00:00:00 2001 From: longxboy Date: Thu, 11 Jun 2026 19:42:45 +0800 Subject: [PATCH 2/2] fix(chat-select): bounded non-blocking reads and shared composer finder - chat-select.py: replace blocking readline() loops with a select()-based read_lines_until() so deadlines hold even when the frida child stays alive but silent; short-circuit when the target chat is already selected regardless of force, since clicking the selected item never fires the selectSession hook - move the ghost-frame-aware edit+send composer finder from send_message.rs into ia::helpers and reuse it in chat_open.rs, ranking candidates lexicographically (focused > send-enabled > main frame) - send_message: always run chat-select even in "chat" state; the open chat may not be the target and chat-select skips when it already is Co-Authored-By: Claude Fable 5 --- docker/tools/chat-select.py | 94 +++++++++----- packages/agent-server-rust/src/ia/helpers.rs | 92 ++++++++++++++ .../agent-server-rust/src/plans/chat_open.rs | 34 +---- .../src/plans/send_message.rs | 116 ++---------------- 4 files changed, 165 insertions(+), 171 deletions(-) diff --git a/docker/tools/chat-select.py b/docker/tools/chat-select.py index 721dacb2..896305a5 100644 --- a/docker/tools/chat-select.py +++ b/docker/tools/chat-select.py @@ -20,6 +20,7 @@ import json import os import re +import select import shutil # ── Per-build constants ────────────────────────────────────────────────────── @@ -230,6 +231,50 @@ def write_js(path, content): """ +def read_lines_until(proc, timeout, stop_on=None): + """Read output lines from proc.stdout until stop_on appears, EOF, or the + wall-clock deadline expires. Returns the lines read (rstripped). + + A plain proc.stdout.readline() blocks indefinitely when the child stays + alive but silent (e.g. a frida hook that never fires), defeating any + time-based loop guard — that hang is what used to make `wx send` time out. + This reads the raw fd non-blocking behind select(), so the deadline holds + even for partial lines, and select() can never miss data stranded in a + Python-level buffer (we own the only buffer, kept on the proc object so + consecutive calls on the same proc don't lose bytes read past stop_on). + """ + fd = proc.stdout.fileno() + os.set_blocking(fd, False) + deadline = time.time() + timeout + buf = getattr(proc, "_read_buf", b"") + lines = [] + try: + while True: + while b"\n" in buf: + raw, buf = buf.split(b"\n", 1) + line = raw.decode("utf-8", errors="replace").rstrip() + lines.append(line) + if stop_on and stop_on in line: + return lines + remaining = deadline - time.time() + if remaining <= 0: + return lines # deadline expired + ready, _, _ = select.select([fd], [], [], remaining) + if not ready: + return lines # deadline expired waiting for output + try: + chunk = os.read(fd, 65536) + except BlockingIOError: + continue + except OSError: + return lines + if not chunk: + return lines # EOF + buf += chunk + finally: + proc._read_buf = buf + + def run_frida_script(pid, script_path, timeout=30, stop_on="SCRIPT_DONE"): """Run a frida script, return output lines.""" proc = subprocess.Popen( @@ -237,19 +282,8 @@ def run_frida_script(pid, script_path, timeout=30, stop_on="SCRIPT_DONE"): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, text=True, bufsize=1, ) - lines = [] - start = time.time() try: - while time.time() - start < timeout: - line = proc.stdout.readline() - if not line: - break - line = line.rstrip() - lines.append(line) - if stop_on and stop_on in line: - break - except Exception: - pass + lines = read_lines_until(proc, timeout, stop_on=stop_on) finally: try: proc.stdin.close() @@ -271,13 +305,7 @@ def run_frida_bg(pid, script_path): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, text=True, bufsize=1, ) - start = time.time() - while time.time() - start < 10: - line = proc.stdout.readline() - if not line: - break - if "READY" in line: - break + read_lines_until(proc, 10, stop_on="READY") return proc @@ -554,17 +582,10 @@ def select_by_index(pid, profile, target_index, click_coords, vector_base, vecto timeout=5, capture_output=True, text=True) log(f"[chat-select] Click result: {click_result.stdout.strip()}") - # Read output looking for DETACHED confirmation (hook fires once then detaches) - lines = [] - start = time.time() - while time.time() - start < 5: - line = proc.stdout.readline() - if not line: - break - line = line.rstrip() - lines.append(line) - if "DETACHED" in line: - break + # Read output looking for DETACHED confirmation (hook fires once then + # detaches). Bounded read: if the click did not produce a selectSession + # call the hook never fires and we give up after the deadline. + lines = read_lines_until(proc, 5, stop_on="DETACHED") kill_frida(proc) @@ -633,9 +654,16 @@ def main(): target_index = sessions[target] log(f"[chat-select] Target: {target} -> index {target_index}") - # Current-selection skip: if not forced and target already selected, skip - if not force and current_sel and current_sel == target: - log(f"[chat-select] Target already selected (current_sel={current_sel}), skipping") + # Already-selected short-circuit: if the target chat is ALREADY the current + # selection, the right-hand pane is already showing it — there is nothing + # to do. This holds even when force=True: clicking the already-selected + # chat list item does NOT trigger a selectSession() call, so the Frida + # hook never fires and select_by_index() would wait out its deadline and + # report a false "Hook did not fire" failure. current_sel is freshly read + # from WeChat's current-session pointer on every invocation, so there is + # no stale skip decision for force to override. + if current_sel == target: + log(f"[chat-select] Target already selected (current_sel={current_sel}), skipping (force={force})") result_json(True, username=target, index=target_index, skipped=True) # Find click coordinates: use --click-xy if provided, else fall back to a11y diff --git a/packages/agent-server-rust/src/ia/helpers.rs b/packages/agent-server-rust/src/ia/helpers.rs index 6fb7fa74..a6f9dc28 100644 --- a/packages/agent-server-rust/src/ia/helpers.rs +++ b/packages/agent-server-rust/src/ia/helpers.rs @@ -36,6 +36,98 @@ pub fn get_bounds_center(bounds: &Bounds) -> (f64, f64) { ) } +/// Check whether an a11y node carries the given state (e.g. "FOCUSED", +/// "DISABLED", "EDITABLE"). +pub fn node_has_state(node: &A11yNode, state: &str) -> bool { + node.states + .as_ref() + .map(|s| s.iter().any(|st| st == state)) + .unwrap_or(false) +} + +/// The main application frame is named "Weixin" or "WeChat" depending on +/// build/locale (the in-repo fixtures use "WeChat"; chat.rs matches both +/// names for the nav button for the same reason). +fn is_main_frame(node: &A11yNode) -> bool { + node.role == "frame" && (node.name == "Weixin" || node.name == "WeChat") +} + +/// A candidate composer: the editable text input plus its sibling Send(S) button. +struct ComposerPair<'a> { + edit: &'a A11yNode, + send: &'a A11yNode, + /// True if this pair lives under the main application frame + /// (as opposed to a detached/ghost chat frame leftover in the a11y tree). + in_main_frame: bool, +} + +/// Find the composer (editable + Send button) to operate on. +/// +/// WeChat's accessibility tree can contain *multiple* edit+send pairs: +/// the live main-window composer plus stale "ghost" frames left behind by +/// chats that were previously detached into separate windows. A naive +/// depth-first "take the first pair" grabs the wrong (ghost) composer, whose +/// input never receives text and whose Send stays DISABLED forever, causing +/// plans to loop and ultimately fail with "No action selected". +/// +/// To be robust we collect every candidate pair, then rank them so the +/// genuinely active composer wins: +/// 1. editable currently FOCUSED (strongest signal) +/// 2. Send button NOT disabled (composer already has text) +/// 3. pair under the main application frame (not a ghost/detached frame) +/// The first pair (DFS order) breaks any remaining ties. +pub fn find_edit_and_send_button(a11y: &A11yNode) -> Option<(&A11yNode, &A11yNode)> { + let mut candidates: Vec = Vec::new(); + collect_edit_send_pairs(a11y, false, &mut candidates); + + // Rank lexicographically; `false` sorts before `true`, so each criterion + // is written as "false = preferred". DFS index breaks ties. + candidates + .iter() + .enumerate() + .min_by_key(|(idx, c)| { + ( + !node_has_state(c.edit, "FOCUSED"), + node_has_state(c.send, "DISABLED"), + !c.in_main_frame, + *idx, + ) + }) + .map(|(_, c)| (c.edit, c.send)) +} + +/// Recursively collect all edit+send composer pairs, tracking whether each +/// pair is inside the main application frame. +fn collect_edit_send_pairs<'a>( + node: &'a A11yNode, + in_main_frame: bool, + out: &mut Vec>, +) { + // Once we enter the main frame, everything below it is in-main. + let in_main_frame = in_main_frame || is_main_frame(node); + + if let Some(children) = &node.children { + let send_btn = children + .iter() + .find(|c| c.role == "push-button" && c.name == "Send(S)"); + let edit_node = children + .iter() + .find(|c| c.role == "text" && node_has_state(c, "EDITABLE")); + + if let (Some(edit), Some(send)) = (edit_node, send_btn) { + out.push(ComposerPair { + edit, + send, + in_main_frame, + }); + } + + for child in children { + collect_edit_send_pairs(child, in_main_frame, out); + } + } +} + /// Extract a FrameHint from an a11y frame node. pub fn frame_hint_from_node(node: &A11yNode) -> Option { let bounds = node.bounds.clone()?; diff --git a/packages/agent-server-rust/src/plans/chat_open.rs b/packages/agent-server-rust/src/plans/chat_open.rs index df7970e3..0dbfb7d7 100644 --- a/packages/agent-server-rust/src/plans/chat_open.rs +++ b/packages/agent-server-rust/src/plans/chat_open.rs @@ -1,5 +1,6 @@ use super::Plan; use crate::ia::actions; +use crate::ia::helpers::find_edit_and_send_button; use crate::ia::selectors::{query_selector, query_selector_all}; use crate::ia::types::*; use crate::tools::chat_select::{open_chat, OpenChatResult}; @@ -24,36 +25,9 @@ pub enum ChatOpenPhase { } fn find_edit_area(a11y: &A11yNode) -> Option<&A11yNode> { - find_edit_near_send(a11y) -} - -fn find_edit_near_send(node: &A11yNode) -> Option<&A11yNode> { - if let Some(children) = &node.children { - let has_send = children.iter().any(|c| { - c.role == "push-button" && c.name == "Send(S)" - }); - let edit_node = children.iter().find(|c| { - c.role == "text" - && c.states - .as_ref() - .map(|s| s.iter().any(|st| st == "EDITABLE")) - .unwrap_or(false) - }); - - if has_send { - if let Some(edit) = edit_node { - return Some(edit); - } - } - - // Recurse - for child in children { - if let Some(result) = find_edit_near_send(child) { - return Some(result); - } - } - } - None + // Ghost-frame-aware: ranks all edit+send pairs so a stale detached-chat + // composer is not picked over the live one (see ia::helpers). + find_edit_and_send_button(a11y).map(|(edit, _)| edit) } #[async_trait::async_trait] diff --git a/packages/agent-server-rust/src/plans/send_message.rs b/packages/agent-server-rust/src/plans/send_message.rs index 9708c5a6..ee6b2a0d 100644 --- a/packages/agent-server-rust/src/plans/send_message.rs +++ b/packages/agent-server-rust/src/plans/send_message.rs @@ -1,5 +1,6 @@ use super::Plan; use crate::ia::actions; +use crate::ia::helpers::{find_edit_and_send_button, node_has_state}; use crate::ia::selectors::query_selector; use crate::ia::types::*; use crate::tools::chat_select::{open_chat, OpenChatResult}; @@ -29,99 +30,6 @@ pub struct SendMessagePlanState { pub confirm_attempts: u32, } -fn node_has_state(node: &A11yNode, state: &str) -> bool { - node.states - .as_ref() - .map(|s| s.iter().any(|st| st == state)) - .unwrap_or(false) -} - -/// A candidate composer: the editable text input plus its sibling Send(S) button. -struct ComposerPair<'a> { - edit: &'a A11yNode, - send: &'a A11yNode, - /// True if this pair lives under the main "Weixin" application frame - /// (as opposed to a detached/ghost chat frame leftover in the a11y tree). - in_main_frame: bool, -} - -/// Find the composer (editable + Send button) to operate on. -/// -/// WeChat's accessibility tree can contain *multiple* edit+send pairs: -/// the live main-window composer plus stale "ghost" frames left behind by -/// chats that were previously detached into separate windows. A naive -/// depth-first "take the first pair" grabs the wrong (ghost) composer, whose -/// input never receives text and whose Send stays DISABLED forever, causing -/// the plan to loop and ultimately fail with "No action selected". -/// -/// To be robust we collect every candidate pair, then rank them so the -/// genuinely active composer wins: -/// 1. editable currently FOCUSED (strongest signal) -/// 2. Send button NOT disabled (composer already has text) -/// 3. pair under the main "Weixin" frame (not a ghost/detached frame) -/// The first pair (DFS order) breaks any remaining ties. -fn find_edit_and_send_button(a11y: &A11yNode) -> Option<(&A11yNode, &A11yNode)> { - let mut candidates: Vec = Vec::new(); - collect_edit_send_pairs(a11y, false, &mut candidates); - - if candidates.is_empty() { - return None; - } - - // Pick the best candidate by preference, preserving DFS order on ties. - let best = candidates.iter().enumerate().min_by_key(|(idx, c)| { - let focused = node_has_state(c.edit, "FOCUSED"); - let send_enabled = !node_has_state(c.send, "DISABLED"); - // Lower key = higher priority. Each desirable property subtracts rank. - let mut score: i32 = 0; - if focused { - score -= 100; - } - if send_enabled { - score -= 10; - } - if c.in_main_frame { - score -= 1; - } - // Tie-break: earlier DFS position wins. - (score, *idx as i32) - }); - - best.map(|(_, c)| (c.edit, c.send)) -} - -/// Recursively collect all edit+send composer pairs, tracking whether each -/// pair is inside the main "Weixin" frame. -fn collect_edit_send_pairs<'a>( - node: &'a A11yNode, - in_main_frame: bool, - out: &mut Vec>, -) { - // Once we enter the main "Weixin" frame, everything below it is in-main. - let in_main_frame = in_main_frame || (node.role == "frame" && node.name == "Weixin"); - - if let Some(children) = &node.children { - let send_btn = children - .iter() - .find(|c| c.role == "push-button" && c.name == "Send(S)"); - let edit_node = children - .iter() - .find(|c| c.role == "text" && node_has_state(c, "EDITABLE")); - - if let (Some(edit), Some(send)) = (edit_node, send_btn) { - out.push(ComposerPair { - edit, - send, - in_main_frame, - }); - } - - for child in children { - collect_edit_send_pairs(child, in_main_frame, out); - } - } -} - #[async_trait::async_trait] impl Plan for SendMessagePlan { type PlanState = SendMessagePlanState; @@ -175,7 +83,11 @@ impl Plan for SendMessagePlan { )) }); - let force = main_state_id == Some("chat"); + // Always run chat-select, even in state "chat_open": the open + // chat may not be the target. This is cheap — chat-select + // itself short-circuits when the target is already the + // current selection. + let force = true; let result = open_chat(¶ms.chat_id, force, click_xy).await; if !result.ok { @@ -208,13 +120,7 @@ impl Plan for SendMessagePlan { plan_state.phase = SendMessagePhase::Inputting; - let is_focused = edit_node - .states - .as_ref() - .map(|s| s.iter().any(|st| st == "FOCUSED")) - .unwrap_or(false); - - if is_focused { + if node_has_state(edit_node, "FOCUSED") { continue; } @@ -286,13 +192,7 @@ impl Plan for SendMessagePlan { None => return None, }; - let is_disabled = send_btn - .states - .as_ref() - .map(|s| s.iter().any(|st| st == "DISABLED")) - .unwrap_or(false); - - if is_disabled { + if node_has_state(send_btn, "DISABLED") { plan_state.phase = SendMessagePhase::Done; return Some(SelectedAction { action: actions::wait_short(),