diff --git a/Cargo.lock b/Cargo.lock index 8ffb0c2..1d9aed2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2055,11 +2055,13 @@ dependencies = [ "chrono", "clipboard-win", "enigo", + "fd-lock", "image", "libc", "regex", "serde", "serde_json", + "sha2", "uiautomation", "windows-sys 0.61.2", "x11rb", diff --git a/Cargo.toml b/Cargo.toml index 416241b..8fd778e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,7 +90,7 @@ agent-client-protocol-schema = { version = "=1.9.1", default-features = false } # no new package; it adds the seccomp filter, the host-allowlist proxy and the # `.git` protection the built-in engine lacks. incord-sandbox = { path = "crates/sandbox" } -# Computer use (`crates/desktop`, binary `incu`): the 17-tool MCP server that +# Computer use (`crates/desktop`, binary `incu`): the 23-tool MCP server that # `incode --desktop-mcp` wraps, and the journal/stop-file/capture code the run # server and the toolbox call. A path dependency, so no new package: it carries # the screen-capture, accessibility and input dependencies that used to be diff --git a/README.md b/README.md index 2f7f781..654b2a9 100644 --- a/README.md +++ b/README.md @@ -850,8 +850,8 @@ anything runs. | `Screenshot` | observe | A picture of a public page or a workspace HTML/SVG file. Offered only when the headless browser is found. | | `RenderPdf` | change | A public page or a workspace HTML/SVG file printed to PDF. Offered only when a printer is found (below). | | `ConvertDocument` | change | An office document to PDF through LibreOffice. Offered only when LibreOffice is found. | -| `screen_info` · `list_windows` · `screenshot` · `ui_tree` · `cursor_position` · `wait_for` | observe | Look at the desktop, or wait (bounded) for a control, a window or a change of the screen. Offered only when `INCODE_DESKTOP=1` — see [Computer use](#computer-use). | -| `click_element` · `set_value` · `element_action` · `focus_window` · `type_text` · `paste_text` · `key` · `scroll` · `drag` · `move_mouse` · `click` | run | Act on the desktop. Same condition; refused in read-only. | +| `screen_info` · `list_windows` · `screenshot` · `ui_tree` · `cursor_position` · `wait_for` · `watch` · `routine_list` · `routine_match` | observe | Look at the desktop, wait (bounded) for a control, a window or a change of the screen, watch for one for hours, or look up a saved routine. Offered only when `INCODE_DESKTOP=1` — see [Computer use](#computer-use). | +| `click_element` · `set_value` · `element_action` · `focus_window` · `type_text` · `paste_text` · `key` · `scroll` · `drag` · `move_mouse` · `click` · `routine_save` · `routine_run` · `routine_forget` | run | Act on the desktop, or keep, replay and forget a routine of such actions. Same condition; refused in read-only. | | `Skill` | observe | Read one skill's text from the shared store. | | `Agent` | run | Hand a self-contained task to a sub-agent (a second headless run of the session's CLI, or the API route), with the session's mode, guards and cancel; at most 2 levels deep. Takes `agent: ` for a [user-defined sub-agent](#user-defined-sub-agents). | | `AskUserQuestion` | session | A multiple-choice question to the owner: answered in the owner's app through the memory engine, else at the terminal, else reported unanswered. | @@ -1079,6 +1079,9 @@ plain screenshots and pixel clicks still work. | `screenshot` | The primary display, or one `window`. The reply carries the region and origin, so a point in the picture maps back to the screen. | | `cursor_position` | Where the pointer is. | | `wait_for` | Wait, bounded, until a control matching a `selector` appears or goes, a window appears or closes, or the screen changes. Returns what it found, or a `timeout` reply with what it last saw. | +| `watch` | `wait_for` for up to 24 hours, looked at every second, with no model polling. When it ends, your post-tool hook for `watch` is run with the reply as `tool_response`: that is the notification or follow-up. **The call blocks** until then. See [Routines and `watch`](#routines-and-watch). | +| `routine_save` · `routine_run` · `routine_forget` | Keep what was done as a routine, run it again with no model, or forget it. | +| `routine_list` · `routine_match` | List the saved routines, or show which one a phrase would run without running it. | | `click_element` | Click a control by id or `selector`, at its centre as it is *now*. | | `set_value` | Set a text field's value (by id or `selector`) through the accessibility interface: no typing, no focus, no clipboard. Read back and marked `(verified)`, or `(unverified: why)`. A password field is never read back. | | `element_action` | Run one of a control's own actions (by id or `selector`) without the mouse: `press`, `toggle`, `expand`, `collapse`, `select` and `focus` everywhere, plus what the platform adds (`scroll_into_view` on Windows; `show_menu`, `increment`, `decrement`, `confirm`, `cancel` on macOS). An action the control lacks is refused, and the refusal lists the ones it offers. | @@ -1121,6 +1124,77 @@ conditions take only `app`/`window`), `timeout_ms` (default 5000, held to or `timeout`. The full grammar and what each platform supplies is in `docs/COMPUTER_USE.md`. +### Routines and `watch` + +A **routine** is a phrase and the desktop calls that carried it out, kept so +`routine_run` can do it again with no model. It is made from what Watchman +already recorded (`routine_save` with `records`, the `w-…` ids of the turns that +did the task, read from the local memory engine) or from an explicit `steps` +list; the engine is optional. + +- `ui_tree`, `screenshot` and the other looking calls are dropped; `wait_for` is + kept. +- **Nothing is kept** when the phrase says "this" or "that", when ANY call or + span in the records failed (a shell command too, not only a desktop call) or + the record says its run failed, when a `wait_for` timed out, or when any step uses an `eN` id, a pixel + coordinate (`click`, `move_mouse`, `drag`) or a window handle (`window`, + `focus_window`). Only selector-based steps replay, and each must say where: + a selector needs an `app=`/`window=` part, and a typing or key step carries + `in_window` (a recording has none, so it is given the window of the step before + it; the replay still refuses to type unless that window has the focus). + `scroll` is never replayable. A replay invokes links and buttons through + `element_action invoke` (no mouse, no focus) and falls back to the recorded + click only when the invoke was refused before it did anything (the control + does not offer it), and it stops when a `set_value` or an `element_action` + that changes a state (toggle, select, expand, collapse, increment, ...) + answers "unverified" and the next step acts on it; a button press (`invoke`, + `press`, `scroll_into_view`) always says "unverified" and never stops it. +- A phrase matches by its **words in any order, ignoring filler words**, except + that the words on each side of `to`, `from`, `into`, `onto`, `over`, `under`, + `before` or `after` stay on that side ("copy a to b" is not "copy b to a"); a + near miss runs nothing. `routine_match` shows what a phrase would run. +- A replay runs every step through the same checks as a live call (blocked apps, + hooks, stop file, journal) and **stops at the first failure**, saying which + step and why, with the paths of that step's before and after pictures (one of + each is taken for every step, through the same gates as `screenshot`, in a + folder of its own for each run beside `routines.json`, only the last run kept; + none is taken of a step that typed a secret; never sent to OCR). There is no + repair: do the task yourself and `routine_save` it under the same phrase to + replace the routine. +- A routine unused for 30 days is forgotten. Only **you** can keep one for good: + `incu routine keep ` (and `unkeep`, `list`) at a terminal; no tool has + an argument for it, and saving other steps over a kept routine un-keeps it. + That is a convenience, not a security boundary: an agent with a shell can run + the command too. +- The file is `routines.json` in the per-user data folder + (`%APPDATA%\incord-desktop`, `~/Library/Application Support/incord-desktop`, + or `$XDG_DATA_HOME`/`~/.local/share/incord-desktop`), overridden by + `INCU_ROUTINES` or `routines` in the config. It holds typed text as typed, and + `routine_save`, `routine_list` and `routine_match` show every step's full + text so you can read what a routine will type. **Typed text is stored only + when the typing step itself names a plain text box** (a `selector` of its own + with a window, role edit or combobox, no password-like name, and not a password + field on the desktop when saved; the replay focuses it first). **Every other + typed text is never stored**: it is a parameter (`{{password}}`), so a routine + saved from a recording has one for all its typing. You bind each at a terminal, + `incu routine bind INCU_PARAM_`, to a variable you set in + incu's environment; only the variable's name is stored, `routine_run` takes no + `params` and refuses to start until every one is bound for the steps as they + are, and saving other steps drops the bindings. The hooks are never shown a + value. Changes to the file take an OS file lock, so two servers do not lose + each other's saves. + +`watch` is `wait_for` with a long deadline (default 1 hour, at most 24, looked +at every second, every five for `screen_changes`): it blocks until the condition +holds, then runs your post-tool hook for the tool `watch` with the reply +(`result`, what it found, the `message` you passed) as `tool_response`. Put the +notification in that hook, for example `osascript -e 'display notification "…"'` +on macOS or `notify-send` on Linux. The stop file (`incu stop`) ends a watch +within a second; nothing else does but its deadline, not even the client going +away (end of input does not cut off a request the client already sent), so an +orphaned watch lasts until its deadline or `incu stop`; while it runs the server +answers nothing else. + ### What it will not do - **Act on a control it did not show you.** An id is a position in one diff --git a/crates/desktop/Cargo.toml b/crates/desktop/Cargo.toml index e914d57..182fb11 100644 --- a/crates/desktop/Cargo.toml +++ b/crates/desktop/Cargo.toml @@ -5,7 +5,7 @@ version = "0.1.0" edition = "2021" rust-version = "1.88" license = "Apache-2.0" -description = "Computer use for an agent: screenshots, accessibility trees, mouse and keyboard, as a 17-tool MCP server with a kill switch and a watch journal" +description = "Computer use for an agent: screenshots, accessibility trees, mouse and keyboard, as a 23-tool MCP server with a kill switch, a watch journal and replayable routines" repository = "https://github.com/incordai/Incode" keywords = ["computer-use", "mcp", "accessibility", "agents", "automation"] categories = ["command-line-utilities", "accessibility"] @@ -26,6 +26,14 @@ chrono = "0.4" base64 = "0.22" # The operator's hook `matcher` (`src/hooks.rs`). regex = "1" +# An OS lock on a file, for the routines file and a replay's pictures +# (`src/routine/oslock.rs`). The same call as `File::try_lock`, which needs Rust +# 1.89 where this crate's floor is 1.88. Already in the lock file (rustyline +# uses it): no new package. +fd-lock = "4" +# A digest of a routine's steps, stored with the person's parameter bindings +# (`src/routine/params.rs`). The root package already depends on it: no new package. +sha2 = "0.10" # Moved here from incode with the code that uses them, at the same pins; the # reasoning for each choice is recorded in incode's Cargo.toml history and in diff --git a/crates/desktop/src/cli.rs b/crates/desktop/src/cli.rs index 8f91d56..3f9789f 100644 --- a/crates/desktop/src/cli.rs +++ b/crates/desktop/src/cli.rs @@ -42,6 +42,19 @@ incu - computer use for an agent, as an MCP server over stdio USAGE: incu serve [OPTIONS] speak MCP on stdin/stdout (the default command) incu stop [OPTIONS] halt the desktop a running server drives + incu routine list show every learned routine, each step in full + incu routine keep keep a routine from expiring (only you can) + incu routine unkeep let it expire again, thirty days from now + incu routine bind + read a routine's secret {PARAM} from the + environment variable ENV_VAR (named INCU_PARAM_...) + of incu's own environment, for the steps as they are + now. Only you bind; the variable's NAME is stored. + Binding stops accidental or stale use. It does NOT + stop an agent that can write your app-data folder or + run incu itself: run agents sandboxed (insbx), where + that folder and the incu binary are out of reach. + incu routine unbind remove that binding incu --help | --version OPTIONS for serve: @@ -57,6 +70,19 @@ OPTIONS for serve: $INCU_CONFIG, else the platform config folder --display Linux: \":N\" or \"xvfb[:WxH]\" (overrides the config) +ENVIRONMENT for serve: + INCU_ROUTINES the file learned routines are kept in (else the + config's `routines`, else routines.json in the + per-user data folder) + INCORD_MEMORY_URL the local memory engine routine_save reads + Watchman records from (default + http://127.0.0.1:9180; loopback only) + +OPTIONS for routine: + --file the routines file (else the config's, else the + one serve uses) + --config the config to find it in + OPTIONS for stop: --journal stop the server journalling to PATH --workspace stop the server for DIR's .incode/ journal @@ -73,6 +99,41 @@ pub enum Command { Serve(ServeArgs), /// Ask a running server to stop. Stop(StopArgs), + /// List, keep or unkeep learned routines: what only the person does. + Routine(RoutineArgs), +} + +/// What `incu routine` does. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RoutineAction { + /// Show every routine, with every step in full. + List, + /// Keep a routine from expiring. + Keep, + /// Let a kept routine expire again (thirty days from now). + Unkeep, + /// Bind a parameter of a routine to an environment variable of incu's own. + Bind, + /// Remove such a binding. + Unbind, +} + +/// The arguments of `incu routine`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RoutineArgs { + /// What to do. + pub action: RoutineAction, + /// The phrase naming the routine, for keep, unkeep, bind and unbind. Matched + /// as a request is: by its words, in any order, filler ignored. + pub phrase: String, + /// The parameter, for bind and unbind. + pub param: String, + /// The environment variable (its NAME), for bind. + pub var: String, + /// The routines file, when named; else the config's, else the default. + pub file: Option, + /// The config file named on the command line, if any. + pub config: Option, } /// The flags of `incu serve`. @@ -120,6 +181,10 @@ pub fn parse>(args: I) -> Result args.remove(0); parse_stop(args) } + Some("routine") => { + args.remove(0); + parse_routine(args) + } Some("-h" | "--help" | "help") => Ok(Command::Print(USAGE.to_string())), Some("--version" | "-V") => Ok(Command::Print(format!( "incu {}", @@ -175,6 +240,86 @@ fn parse_serve(args: Vec) -> Result { })) } +fn parse_routine(args: Vec) -> Result { + let mut it = args.into_iter(); + let action = match it.next().as_deref() { + Some("list") => RoutineAction::List, + Some("keep") => RoutineAction::Keep, + Some("unkeep") => RoutineAction::Unkeep, + Some("bind") => RoutineAction::Bind, + Some("unbind") => RoutineAction::Unbind, + Some("-h" | "--help") => return Ok(Command::Print(USAGE.to_string())), + Some(other) => { + return Err(format!( + "unknown routine command `{other}` (list, keep, unkeep, bind or unbind)\n\n{USAGE}" + )) + } + None => { + return Err(format!( + "routine needs list, keep, unkeep, bind or unbind\n\n{USAGE}" + )) + } + }; + let mut words = Vec::new(); + let mut file = None; + let mut config = None; + while let Some(arg) = it.next() { + match arg.as_str() { + "-h" | "--help" => return Ok(Command::Print(USAGE.to_string())), + "--file" => file = Some(PathBuf::from(value(&arg, &mut it)?)), + "--config" => config = Some(PathBuf::from(value(&arg, &mut it)?)), + flag if flag.starts_with('-') => { + return Err(format!("unknown option `{flag}` for routine\n\n{USAGE}")) + } + _ => words.push(arg), + } + } + // Bind and unbind end with the parameter (and, for bind, the variable); what + // comes before is the words that name the routine. + let (mut param, mut var) = (String::new(), String::new()); + match action { + RoutineAction::Bind => { + if words.len() < 3 { + return Err( + "incu routine bind \ + " + .to_string(), + ); + } + var = words.pop().unwrap_or_default(); + param = words.pop().unwrap_or_default(); + } + RoutineAction::Unbind => { + if words.len() < 2 { + return Err( + "incu routine unbind ".to_string(), + ); + } + param = words.pop().unwrap_or_default(); + } + _ => {} + } + let phrase = words.join(" "); + match (action, phrase.trim().is_empty()) { + (RoutineAction::List, false) => Err("routine list takes no phrase".to_string()), + ( + RoutineAction::Keep + | RoutineAction::Unkeep + | RoutineAction::Bind + | RoutineAction::Unbind, + true, + ) => Err("name the routine: incu routine keep ".to_string()), + _ => Ok(Command::Routine(RoutineArgs { + action, + phrase, + param, + var, + file, + config, + })), + } +} + fn parse_stop(args: Vec) -> Result { let mut journal = None; let mut workspace = None; @@ -206,6 +351,9 @@ pub struct ServeOptions { pub display: Option, /// The operator's commands around every call, if any. pub hooks: Option>, + /// Where routines are kept; `None` is the default + /// ([`crate::routine::store::Store::default_path`]). + pub routines: Option, /// What the startup lines on stderr call this server (`incu`, or /// `incode-desktop` for incode's wrapper). pub name: &'static str, @@ -228,6 +376,7 @@ impl ServeOptions { limits, display, hooks, + routines, } = config; Ok(Self { tier: args.tier, @@ -235,11 +384,27 @@ impl ServeOptions { limits, display: args.display.or(display), hooks: shell_hooks(hooks), + routines, name: "incu", }) } } +/// The routines kit `incu` runs with: the file at `path`, else the default, and +/// Watchman's records from the local memory engine as the source. `None` when +/// there is no place to keep the file. +/// +/// The engine is only asked when a routine is saved from record ids, and an +/// engine that is not running is an error at that moment, not at startup: the +/// crate works without it. +fn routine_kit(path: Option) -> Option { + let path = path.or_else(crate::routine::store::Store::default_path)?; + Some(crate::mcp::RoutineKit::new( + crate::routine::store::Store::at(path), + Box::new(crate::routine::watchman::WatchmanSource::from_env()), + )) +} + /// [`ShellHooks`] for `config`, or none when it has nothing to run. fn shell_hooks(config: HookConfig) -> Option> { (!config.is_empty()).then(|| Box::new(ShellHooks::new(config)) as Box) @@ -269,6 +434,7 @@ pub fn serve(options: ServeOptions) -> Result<(), String> { limits, display, hooks, + routines, name, } = options; crate::limits::set( @@ -307,12 +473,162 @@ pub fn serve(options: ServeOptions) -> Result<(), String> { eprintln!("{name} journalling to {}", path.display()); server = server.with_journal(path); } + server = match routine_kit(routines) { + Some(kit) => { + eprintln!("{name}: routines in {}", kit.path().display()); + server.with_routines(kit) + } + None => { + eprintln!( + "{name}: routines are off: no data folder to keep them in (set INCU_ROUTINES)" + ); + server + } + }; // Named on stderr so an operator who asked for `read-only` can see that it // arrived. A capability restriction nobody can observe is one nobody trusts. eprintln!("{name} MCP server ready on stdio ({})", tier.label()); crate::mcp::run_stdio(server).map_err(|e| e.to_string()) } +/// The routines file `incu routine` works on: `--file`, else the config's +/// `routines`, else `$INCU_ROUTINES`, else the per-user data folder: the file +/// `incu serve` uses. +/// +/// # Errors +/// +/// The config cannot be read, or there is no place to keep routines. +pub fn routine_store( + file: Option, + config: Option, +) -> Result { + use crate::routine::store::Store; + let path = match file { + Some(path) => Some(path), + None => { + let config = match &config { + Some(path) => Config::load(path)?, + None => Config::load_default()?, + }; + config.routines.or_else(Store::default_path) + } + }; + path.map(Store::at) + .ok_or_else(|| "no place to keep routines: set INCU_ROUTINES or pass --file".to_string()) +} + +/// `incu routine`: what only the person does to the routines an agent saved. +/// +/// **Why this is a command and not a tool.** A routine a person has kept never +/// expires. If an agent could say "keep this", it could exempt its own saves from +/// the thirty days that exist so that nothing it learned lives on unreviewed. So +/// keeping is done here, at a terminal, by the person, and no tool has an +/// argument for it. +/// +/// # Errors +/// +/// The file cannot be read or written, or no routine matches the phrase. +pub fn run_routine(args: RoutineArgs) -> Result { + let store = routine_store(args.file, args.config)?; + let now = crate::routine::now_ms(); + match args.action { + RoutineAction::List => { + let file = store.load()?; + let mut out = format!("routines in {}\n", store.path().display()); + let mut any = false; + for routine in file.live(now) { + any = true; + let status = match routine.days_left(now) { + Some(days) => format!("expires in {days} days unless used"), + None => "kept: never expires".to_string(), + }; + out.push_str(&format!("\n\"{}\" ({status})\n", routine.phrase)); + for (i, step) in routine.steps.iter().enumerate() { + out.push_str(&format!(" {}. {}\n", i + 1, step.review())); + } + // What it needs the person to bind, and what is bound: the + // variable's NAME and whether it is set here, never its value. + let needs = crate::routine::params::names(&routine.steps); + let stale = !routine.bindings.is_empty() + && routine.bound_steps != crate::routine::params::digest(&routine.steps); + for param in &needs { + match routine.bindings.get(param) { + Some(var) if stale => out.push_str(&format!( + " {{{{{param}}}}} bound to {var}, but the steps are not the ones it \ + was bound for: bind it again\n" + )), + Some(var) => out.push_str(&format!( + " {{{{{param}}}}} bound to {var} ({})\n", + if std::env::var_os(var).is_some_and(|v| !v.is_empty()) { + "set in this environment" + } else { + "NOT set in this environment" + } + )), + None => out.push_str(&format!( + " {{{{{param}}}}} NOT BOUND: incu routine bind \"{}\" {param} {}{}\n", + routine.phrase, + crate::routine::params::ENV_PREFIX, + param.to_uppercase() + )), + } + } + } + if !any { + out.push_str("\n(none)\n"); + } + Ok(out.trim_end().to_string()) + } + RoutineAction::Bind => { + let (phrase, param, var) = (args.phrase, args.param, args.var); + store.update(|file| { + file.bind(&phrase, ¶m, &var, now)?; + Ok(format!( + "bound {{{{{param}}}}} of \"{phrase}\" to {var}. Set {var} in the \ + environment where incu is launched: {}", + if std::env::var_os(&var).is_some_and(|v| !v.is_empty()) { + "it is set in this environment" + } else { + "it is NOT set in this environment" + } + )) + }) + } + RoutineAction::Unbind => { + let (phrase, param) = (args.phrase, args.param); + store.update(|file| { + if file.unbind(&phrase, ¶m, now) { + Ok(format!( + "{{{{{param}}}}} of \"{phrase}\" is no longer bound" + )) + } else { + Err(format!( + "\"{phrase}\" has no binding for {param:?}; `incu routine list` shows \ + what is bound" + )) + } + }) + } + RoutineAction::Keep | RoutineAction::Unkeep => { + let keep = args.action == RoutineAction::Keep; + let phrase = args.phrase; + store.update(|file| { + if file.set_taught(&phrase, keep, now) { + Ok(if keep { + format!("kept \"{phrase}\": it will not expire") + } else { + format!("\"{phrase}\" will expire after thirty days unused, from now") + }) + } else { + Err(format!( + "no routine matches \"{phrase}\"; `incu routine list` shows what is saved" + )) + } + }) + } + } +} + /// Ask the server whose stop file is `path` to stop, and say so on stdout. /// /// REPORTED, never swallowed: a stop request that silently failed is worse than @@ -345,6 +661,10 @@ pub fn run(command: Command) -> Result<(), String> { Ok(()) } Command::Serve(args) => serve(ServeOptions::from_args(args)?), + Command::Routine(args) => { + println!("{}", run_routine(args)?); + Ok(()) + } Command::Stop(args) => { let path = match (args.journal, args.workspace) { (Some(journal), _) => crate::safety::stop_file_beside(&journal), @@ -441,6 +761,210 @@ mod tests { )); } + fn routine_args(words: &[&str]) -> RoutineArgs { + match parse(argv(words)).expect("parses") { + Command::Routine(args) => args, + other => panic!("expected routine, got {other:?}"), + } + } + + #[test] + fn routine_keep_and_unkeep_take_the_words_that_ask_for_the_routine() { + let keep = routine_args(&["routine", "keep", "open", "the", "weekly", "report"]); + assert_eq!(keep.action, RoutineAction::Keep); + assert_eq!(keep.phrase, "open the weekly report"); + let unkeep = routine_args(&["routine", "unkeep", "report", "--file", "r.json"]); + assert_eq!(unkeep.action, RoutineAction::Unkeep); + assert_eq!(unkeep.file.as_deref(), Some(Path::new("r.json"))); + assert_eq!( + routine_args(&["routine", "list"]).action, + RoutineAction::List + ); + // A routine is named, a list is not, and a typo is not guessed. + assert!(parse(argv(&["routine", "keep"])).is_err()); + assert!(parse(argv(&["routine", "list", "extra"])).is_err()); + assert!(parse(argv(&["routine", "teach", "x"])).is_err()); + assert!(parse(argv(&["routine"])).is_err()); + assert!(parse(argv(&["routine", "keep", "x", "--taught"])).is_err()); + } + + /// Only this command sets "kept"; keeping stops the expiry and unkeeping + /// starts it again. + #[test] + fn keeping_a_routine_is_done_here_and_stops_it_expiring() { + use crate::routine::store::{Routines, Store}; + let dir = std::env::temp_dir().join(format!("incu-cli-keep-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let file = dir.join("routines.json"); + let store = Store::at(file.clone()); + let mut routines = Routines::default(); + routines + .learn( + "open notes", + vec![crate::routine::Step::new( + "key", + serde_json::json!({"keys": ["ctrl", "n"], "in_window": "app=notes"}), + )], + false, + crate::routine::now_ms(), + ) + .unwrap(); + store.save(&routines).unwrap(); + let args = |action, phrase: &str| RoutineArgs { + action, + phrase: phrase.to_string(), + param: String::new(), + var: String::new(), + file: Some(file.clone()), + config: None, + }; + assert!(store.load().unwrap().routines[0] + .days_left(crate::routine::now_ms()) + .is_some()); + let said = run_routine(args(RoutineAction::Keep, "notes open")).unwrap(); + assert!(said.contains("will not expire"), "{said}"); + let kept = store.load().unwrap(); + assert!(kept.routines[0].taught); + assert_eq!(kept.routines[0].days_left(crate::routine::now_ms()), None); + let listed = run_routine(args(RoutineAction::List, "")).unwrap(); + assert!(listed.contains("kept: never expires"), "{listed}"); + assert!( + listed.contains("ctrl"), + "every step is shown in full: {listed}" + ); + run_routine(args(RoutineAction::Unkeep, "open notes")).unwrap(); + assert!(!store.load().unwrap().routines[0].taught); + let missing = run_routine(args(RoutineAction::Keep, "open mail")).unwrap_err(); + assert!(missing.contains("no routine matches"), "{missing}"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn routine_bind_takes_the_routine_words_then_the_param_then_the_variable() { + let bind = routine_args(&[ + "routine", + "bind", + "unlock", + "the", + "vault", + "password", + "INCU_PARAM_VAULT", + "--file", + "r.json", + ]); + assert_eq!(bind.action, RoutineAction::Bind); + assert_eq!(bind.phrase, "unlock the vault"); + assert_eq!( + (bind.param.as_str(), bind.var.as_str()), + ("password", "INCU_PARAM_VAULT") + ); + let unbind = routine_args(&["routine", "unbind", "unlock", "vault", "password"]); + assert_eq!(unbind.action, RoutineAction::Unbind); + assert_eq!(unbind.phrase, "unlock vault"); + assert_eq!(unbind.param, "password"); + // Too few words: a routine, a param and a variable are all needed. + assert!(parse(argv(&["routine", "bind", "vault", "password"])).is_err()); + assert!(parse(argv(&["routine", "bind"])).is_err()); + assert!(parse(argv(&["routine", "unbind", "password"])).is_err()); + } + + /// The person binds; the file keeps the variable's NAME and a digest of the + /// steps, never a value. + #[test] + fn binding_is_the_persons_command_and_stores_a_name_not_a_value() { + use crate::routine::store::{Routines, Store}; + let dir = std::env::temp_dir().join(format!("incu-cli-bind-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let file = dir.join("routines.json"); + let store = Store::at(file.clone()); + let mut routines = Routines::default(); + routines + .learn( + "unlock the vault", + vec![crate::routine::Step::new( + "type_text", + serde_json::json!({"text": "{{password}}", "in_window": "app=vault"}), + )], + false, + crate::routine::now_ms(), + ) + .unwrap(); + store.save(&routines).unwrap(); + let args = |action, param: &str, var: &str| RoutineArgs { + action, + phrase: "vault unlock".to_string(), + param: param.to_string(), + var: var.to_string(), + file: Some(file.clone()), + config: None, + }; + std::env::set_var("INCU_PARAM_CLI_TEST_BIND", "hunter2-secret"); + let said = run_routine(args( + RoutineAction::Bind, + "password", + "INCU_PARAM_CLI_TEST_BIND", + )) + .unwrap(); + assert!(said.contains("INCU_PARAM_CLI_TEST_BIND"), "{said}"); + assert!(!said.contains("hunter2"), "{said}"); + let on_disk = std::fs::read_to_string(&file).unwrap(); + assert!(on_disk.contains("INCU_PARAM_CLI_TEST_BIND"), "{on_disk}"); + assert!( + !on_disk.contains("hunter2"), + "a value reached the file: {on_disk}" + ); + let routine = &store.load().unwrap().routines[0]; + assert_eq!(routine.bindings["password"], "INCU_PARAM_CLI_TEST_BIND"); + assert_eq!( + routine.bound_steps, + crate::routine::params::digest(&routine.steps) + ); + // The listing says what is bound and whether it is set, with no value. + let listed = run_routine(args(RoutineAction::List, "", "")).unwrap(); + assert!( + listed.contains("bound to INCU_PARAM_CLI_TEST_BIND (set in this environment)"), + "{listed}" + ); + assert!(!listed.contains("hunter2"), "{listed}"); + // Refused: another variable of the environment, a param it does not take, + // a routine that is not there. + let home = run_routine(args(RoutineAction::Bind, "password", "HOME")).unwrap_err(); + assert!(home.contains("INCU_PARAM_"), "{home}"); + let nope = run_routine(args(RoutineAction::Bind, "pasword", "INCU_PARAM_X")).unwrap_err(); + assert!(nope.contains("takes: password"), "{nope}"); + let mut elsewhere = args(RoutineAction::Bind, "password", "INCU_PARAM_X"); + elsewhere.phrase = "open mail".to_string(); + assert!(run_routine(elsewhere) + .unwrap_err() + .contains("no routine matches")); + // Unbinding removes it. + run_routine(args(RoutineAction::Unbind, "password", "")).unwrap(); + assert!(store.load().unwrap().routines[0].bindings.is_empty()); + assert!(store.load().unwrap().routines[0].bound_steps.is_empty()); + let again = run_routine(args(RoutineAction::Unbind, "password", "")).unwrap_err(); + assert!(again.contains("no binding"), "{again}"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// What binding is and is not, said where the person reads about the command. + #[test] + fn the_bind_help_says_it_is_not_a_boundary_against_an_agent_that_can_write_files() { + let help = USAGE.split("OPTIONS for serve").next().unwrap(); + let bind = help.split("incu routine bind").nth(1).unwrap(); + let bind = bind.split("incu routine unbind").next().unwrap(); + for said in [ + "stops accidental or stale use", + "does NOT", + "app-data folder", + "insbx", + ] { + assert!( + bind.contains(said), + "the bind help does not say {said:?}: {bind}" + ); + } + } + /// The usage must name every flag the parser takes, or an operator cannot /// find the one that matters. #[test] diff --git a/crates/desktop/src/config.rs b/crates/desktop/src/config.rs index 0371b55..de57672 100644 --- a/crates/desktop/src/config.rs +++ b/crates/desktop/src/config.rs @@ -1,5 +1,5 @@ -//! The settings the crate owns: limits, the Linux display and the operator's -//! hooks. +//! The settings the crate owns: limits, the Linux display, the operator's +//! hooks and where routines are kept. //! //! # One file for either binary //! @@ -48,6 +48,11 @@ pub struct Config { /// The operator's commands around every call. See [`crate::hooks`]. #[serde(default, alias = "desktop_hooks", alias = "hooks")] pub hooks: HookConfig, + /// Where learned routines are kept. Unset: `$INCU_ROUTINES`, else + /// `routines.json` in the per-user data folder + /// ([`crate::routine::store::Store::default_path`]). + #[serde(default, alias = "routines_file")] + pub routines: Option, } impl Config { @@ -149,6 +154,18 @@ mod tests { assert_eq!(config.display.as_deref(), Some(":99")); } + /// Where routines are kept is overridable, under either name. + #[test] + fn the_routines_file_can_be_named_in_the_config() { + let config: Config = + serde_json::from_str(r#"{"routines": "/data/r.json"}"#).expect("config"); + assert_eq!(config.routines, Some(PathBuf::from("/data/r.json"))); + let alias: Config = + serde_json::from_str(r#"{"routines_file": "/data/s.json"}"#).expect("config"); + assert_eq!(alias.routines, Some(PathBuf::from("/data/s.json"))); + assert_eq!(Config::default().routines, None); + } + #[test] fn nothing_set_is_the_default() { let config: Config = serde_json::from_str("{}").expect("config"); diff --git a/crates/desktop/src/lib.rs b/crates/desktop/src/lib.rs index 493b12f..d05ffb0 100644 --- a/crates/desktop/src/lib.rs +++ b/crates/desktop/src/lib.rs @@ -8,7 +8,9 @@ //! //! - **This is offered to a DELEGATED agent over MCP.** It is not dispatched by //! a loop, because there is no such loop: the agent runs in its own process -//! and reaches the screen only through the 17 tools of [`mcp`]. +//! and reaches the screen only through the 23 tools of [`mcp`]: the 17 +//! that look and act, `watch`, and five that keep and replay what was done +//! ([`routine`]). //! //! - **This crate does not depend on incode.** It began as a module of incode //! (`src/desktop/`) and moved out whole. What it used to take from the rest @@ -51,6 +53,7 @@ pub mod journal; pub mod limits; pub mod mcp; pub mod permission; +pub mod routine; pub mod safety; pub mod selector; pub mod thread; diff --git a/crates/desktop/src/mcp.rs b/crates/desktop/src/mcp.rs index aa07719..9c839f5 100644 --- a/crates/desktop/src/mcp.rs +++ b/crates/desktop/src/mcp.rs @@ -52,8 +52,11 @@ //! - [`tools`]: `run_tool` and one step per tool. //! - [`guard`]: the blocked-application, focus, point and region refusals a //! tool asks before it acts. -//! - `wait_tool`: `wait_for`, the one tool that polls: its arguments and the -//! live desktop, over the timing rules in [`crate::wait`]. +//! - `wait_tool`: `wait_for` and `watch`, the two tools that poll: their +//! arguments and the live desktop, over the timing rules in [`crate::wait`]. +//! - `routine_tool` and `routine_definitions`: the five routine tools over +//! [`crate::routine`], and `Server::gated_call`, the one path through every +//! gate that a live call and each step of a replayed routine both take. //! - `tests`: the tests, at their old path `mcp::tests`. //! //! # Wired to @@ -75,6 +78,8 @@ use crate::permission::Tier; mod definitions; mod gate; mod guard; +mod routine_definitions; +mod routine_tool; mod tools; mod transport; mod wait_tool; @@ -85,7 +90,11 @@ mod tests; #[cfg(test)] mod selector_tests; +#[cfg(test)] +mod routine_tests; + pub use definitions::tool_definitions; +pub use routine_tool::RoutineKit; pub use transport::run_stdio; use transport::{code, Failure}; @@ -128,6 +137,9 @@ pub struct Server { /// the contract, and for why a hook sees the real input rather than the /// redacted description the journal gets. hooks: Option>, + /// The routines file and where records come from. `None` until + /// [`Server::with_routines`]; the routine tools refuse by name without it. + routines: Option, } impl Server { @@ -142,6 +154,7 @@ impl Server { stop_file: None, tier: Tier::Act, hooks: None, + routines: None, } } @@ -227,15 +240,10 @@ impl Server { // tool refused. Reporting it as a protocol error would make a refused // click look like a broken server, and the agent would retry the // transport rather than read the reason. - self.catch_stop_file(); - let permitted = self.authorise(name); - let raw_args = args.to_string(); - let vetoed = self.hook_veto(&permitted, name, &raw_args); - let outcome = self.outcome(&permitted, vetoed, name, &args); - self.tell_post_hooks(name, &raw_args, &outcome); - self.record_call(name, &args, &outcome); - - Ok(match outcome { + // + // The gates are `gated_call` (in `routine_tool`), the one path a + // replayed routine step takes too. + Ok(match self.gated_call(name, &args) { Ok(content) => json!({"content": content, "isError": false}), Err(why) => json!({ "content": [{"type": "text", "text": why}], diff --git a/crates/desktop/src/mcp/definitions.rs b/crates/desktop/src/mcp/definitions.rs index ea208f1..d0ae337 100644 --- a/crates/desktop/src/mcp/definitions.rs +++ b/crates/desktop/src/mcp/definitions.rs @@ -21,7 +21,8 @@ use serde_json::{json, Value}; -use super::{ACTING, UNTRUSTED}; +use super::tools::looks_after; +use super::UNTRUSTED; /// Every desktop tool, in MCP's shape. The toolbox offers these same /// definitions to a model (`toolbox::desktop`), so they are written once. @@ -31,6 +32,7 @@ pub fn tool_definitions() -> Vec { "screenshot", "ui_tree", "wait_for", + "watch", "click_element", "set_value", "element_action", @@ -42,6 +44,7 @@ pub fn tool_definitions() -> Vec { "drag", ]; let mut tools = base_definitions(); + tools.extend(super::routine_definitions::definitions()); for tool in &mut tools { let name = tool["name"].as_str().unwrap_or_default().to_string(); let properties = &mut tool["inputSchema"]["properties"]; @@ -52,7 +55,7 @@ pub fn tool_definitions() -> Vec { focused window the call is REFUSED rather than typing somewhere else." } "screenshot" => "Capture only this window, from list_windows.", - "wait_for" => { + "wait_for" | "watch" => { "The window to watch, from list_windows: for element_* the one the \ selector is looked for in, for screen_changes the one whose area is \ compared. Default: the window the selector names, else the focused one \ @@ -62,7 +65,7 @@ pub fn tool_definitions() -> Vec { }; properties["window"] = json!({"type": "string", "description": what}); } - if ACTING.contains(&name.as_str()) { + if looks_after(&name) { properties["screenshot_after"] = json!({ "type": "boolean", "description": "Also return a screenshot of the window afterwards, to check \ diff --git a/crates/desktop/src/mcp/gate.rs b/crates/desktop/src/mcp/gate.rs index f25ac50..40d8961 100644 --- a/crates/desktop/src/mcp/gate.rs +++ b/crates/desktop/src/mcp/gate.rs @@ -183,8 +183,19 @@ impl Server { // It cannot deny: the action is already done, and reporting a refusal // for it would be a lie to the agent. if let Some(hooks) = self.hooks.as_ref() { - if outcome.is_ok() { - let said = hooks.run_post_tool_use(name, raw_args, "", false); + if let Ok(content) = outcome { + // Every tool's hook is told the call and not its answer, as it + // always was: a screen's text is not the operator's to be + // handed by default. `watch` is the exception, because telling + // the operator WHAT it saw is the whole point of it: its hook + // is the notification, and `tool_response` is the reply + // (found or timeout, the control or window, the `message`). + let answer = if name == "watch" { + super::tools::text_of(content) + } else { + String::new() + }; + let said = hooks.run_post_tool_use(name, raw_args, &answer, false); for message in said.messages() { // stderr, never stdout: stdout is the protocol channel. eprintln!("hook after {name}: {message}"); @@ -193,6 +204,16 @@ impl Server { } } + /// A line in the journal that is not a finished call: a `watch` that has + /// begun, so an operator following the journal sees the wait. + /// + /// Called by `Server::run_tool` before a `watch` starts to poll. + pub(super) fn journal_note(&mut self, t: &str, detail: &str) { + if let Some(journal) = self.journal.as_mut() { + let _ = journal.record(t, &format!("started: {detail}"), true); + } + } + /// Write the call, ran or refused, to the operator's journal. /// /// Called by `Server::call_tool` last, after [`Server::tell_post_hooks`]. @@ -256,13 +277,21 @@ impl Server { "element_action" => format!("{} {}", element_ref(args), s("action")), // The condition and the selector as `Selector::render_redacted` // has it (no text field's name), and how long it may wait. - "wait_for" => format!( + // `watch` is the same, and its `message` (what its hook is told to + // say) stays out of the journal like any other free text. + "wait_for" | "watch" => format!( "{} {} timeout {}", s("condition"), selector_ref(args), args.get("timeout_ms") .map_or("default".to_string(), |ms| format!("{ms} ms")) ), + // The phrase a routine is asked for by, which is the operator's own + // name for it; never its steps, which may carry typed text. + "routine_save" | "routine_run" | "routine_forget" | "routine_match" => { + let phrase: String = s("phrase").chars().take(80).collect(); + format!("\"{phrase}\"") + } "focus_window" => format!("window {}", window_arg(args).unwrap_or_default()), "click" | "move_mouse" => format!("({}, {})", n("x"), n("y")), "drag" => format!( diff --git a/crates/desktop/src/mcp/routine_definitions.rs b/crates/desktop/src/mcp/routine_definitions.rs new file mode 100644 index 0000000..166e240 --- /dev/null +++ b/crates/desktop/src/mcp/routine_definitions.rs @@ -0,0 +1,283 @@ +//! What a client is told about `watch` and the five routine tools. +//! +//! Kept apart from [`super::definitions`] for the reason that file is apart +//! from the transport: these are paragraphs a model reads, and they change for a +//! different reason than the code does. [`super::definitions::tool_definitions`] +//! appends [`definitions`] to the base list and adds `window` and +//! `screenshot_after` the same way it does for every other tool. +//! +//! # Wired to +//! +//! - **Called by** [`super::definitions::tool_definitions`]. +//! - **Calls** nothing; reads [`super::UNTRUSTED`] and [`crate::wait`]'s limits. + +use serde_json::{json, Value}; + +use super::UNTRUSTED; + +/// `watch` and the routine tools, in MCP's shape. +pub(super) fn definitions() -> Vec { + vec![ + watch_definition(), + routine_save_definition(), + routine_run_definition(), + routine_forget_definition(), + routine_list_definition(), + routine_match_definition(), + ] +} + +/// The `phrase` argument shared by four of the tools. +fn phrase_property(what: &str) -> Value { + json!({"type": "string", "description": what}) +} + +fn watch_definition() -> Value { + json!({ + "name": "watch", + "description": format!( + "Watch for something to happen and tell the operator when it does, with no \ + model polling: the same conditions as wait_for (element_appears, \ + element_gone, window_appears, window_gone, screen_changes) looked for every \ + second (every {screen_poll} ms for screen_changes, which captures the whole \ + screen each look) for up to {max_hours} hours, then the operator's post-tool \ + hook for `watch` is run with the reply as its `tool_response`, which is how \ + it notifies (a desktop notification, a message, a follow-up command: \ + whatever the operator wrote). THIS CALL BLOCKS until the condition holds, \ + the time runs out or the operator stops the desktop (`incu stop`, the stop \ + file: it ends within a second), and your turn waits with it. Nothing else \ + ends it, not even your client going away: an orphaned watch lasts until its \ + deadline. IT ALSO HOLDS THIS SERVER: while it runs no other desktop call is \ + answered, including yours. Use it only when waiting is the task. For a few seconds \ + use wait_for. `timeout_ms` defaults to {default} \ + ms (one hour) and is held to {max} ms; `poll_ms` {min_poll} to {max_poll}. \ + `message` is what the notification should say; it is returned in the reply. \ + The reply is JSON whose `result` is `found` or `timeout`, as wait_for's. \ + {UNTRUSTED}", + max_hours = crate::wait::MAX_WATCH_MS / 3_600_000, + screen_poll = crate::wait::DEFAULT_WATCH_SCREEN_POLL_MS, + default = crate::wait::DEFAULT_WATCH_MS, + max = crate::wait::MAX_WATCH_MS, + min_poll = crate::wait::MIN_WATCH_POLL_MS, + max_poll = crate::wait::MAX_WATCH_POLL_MS, + ), + "inputSchema": { + "type": "object", + "properties": { + "condition": { + "type": "string", + "enum": [ + "element_appears", "element_gone", "window_appears", "window_gone", + "screen_changes" + ], + }, + "selector": { + "type": "string", + "description": "key=value pairs joined by ';' — control: role, name \ + (name~ for contains), aid, path, nth; window: app, \ + window (window~ for contains). E.g. \ + role=button;name=Save or app=notepad;window~=Save As. \ + For element_* conditions it must name a control; for \ + window_* only app and/or window.", + }, + "timeout_ms": { + "type": "integer", + "description": format!( + "Longest to watch. Default {}, at most {}.", + crate::wait::DEFAULT_WATCH_MS, + crate::wait::MAX_WATCH_MS + ), + }, + "poll_ms": { + "type": "integer", + "description": format!( + "How often to look. Default {} ({} for screen_changes), from {} to {}.", + crate::wait::DEFAULT_WATCH_POLL_MS, + crate::wait::DEFAULT_WATCH_SCREEN_POLL_MS, + crate::wait::MIN_WATCH_POLL_MS, + crate::wait::MAX_WATCH_POLL_MS + ), + }, + "baseline": { + "type": "string", + "description": "screen_changes: the `hash` of an earlier reply to compare \ + with. Default: the screen as the watch begins.", + }, + "tolerance": { + "type": "integer", + "description": format!( + "screen_changes: how many of the 128 bits may differ. Default {}.", + crate::wait::DEFAULT_TOLERANCE + ), + }, + "message": { + "type": "string", + "description": "What the notification should say, up to 200 characters. \ + Returned in the reply, so the hook can show it.", + }, + }, + "required": ["condition"], + }, + }) +} + +fn routine_save_definition() -> Value { + json!({ + "name": "routine_save", + "description": format!( + "Keep what was just done at the desktop as a routine that `routine_run` can \ + do again later with no model. Give the `phrase` that asks for it and EITHER \ + `records` (Watchman record ids of the turns that did the task, in any \ + order; needs the local memory engine) OR `steps` (a list of {{tool, args}} \ + you write). The looking calls (ui_tree, screenshot, list_windows, \ + screen_info, cursor_position) are dropped and wait_for is kept. NOTHING IS \ + KEPT, and the reason is given, when: the phrase says \"this\" or \"that\"; \ + ANY call or span of the records failed (a shell command too, not only a \ + desktop call), or the record itself says its run failed, or a wait_for \ + timed out; or any step uses an eN id, a pixel \ + coordinate (click, move_mouse, drag) or a window handle (`window`, \ + focus_window), because those hold for one screen only — name controls with \ + a `selector` and windows inside it (app=...;window=...). Every acting step \ + must say WHERE: a selector needs an app= or window= part, and a typing or \ + key step needs `in_window` (e.g. app=notepad;window~=Untitled; a recording \ + is given the window of the step before it, and the replay refuses to type \ + unless that window has focus); scroll is never kept. A replay invokes links \ + and buttons without the mouse and stops when a set_value (or an \ + increment/decrement) answers \"unverified\" and the next step depends on \ + it; a button press, which always says \"unverified\", never stops it. \ + Saving under a phrase that means the same as a saved one (same words, any \ + order, filler words ignored; but the words on each side of to, from, into, \ + onto, over, under, before or after stay on that side) REPLACES it: that is \ + how a routine that failed is redone. TYPED TEXT IS STORED AS IT IS in the \ + routines file, and the reply shows every step's full text so you and the \ + person can read it, ONLY WHEN THE TYPING STEP ITSELF NAMES THE FIELD: a \ + `selector` of its own (with app= or window=, role=edit or role=combobox, \ + no password-like name) that the desktop confirms is not a password field \ + when you save; the replay focuses that field before it types. EVERY OTHER \ + typed text (no selector of its own, because focus may be anywhere, a \ + sign-in dialog or a prompt included; a button; a secure field; a text a \ + recording kept only as a placeholder or digest) is stored as a parameter, \ + `{{{{name}}}}`, listed in the reply, and `routine_run` refuses to start \ + until the PERSON has bound each one (`incu routine bind`). So a routine you \ + save from a recording has parameters for all its typing. You may write \ + `{{{{name}}}}` in a typed text yourself to make one. A routine unused for 30 \ + days is forgotten; only the person can keep one for good (`incu routine \ + keep`), so there is no argument for it, and saving other steps over a kept \ + or bound routine un-keeps and un-binds it. {UNTRUSTED}" + ), + "inputSchema": { + "type": "object", + "properties": { + "phrase": phrase_property( + "The words that ask for this routine, e.g. \"open the weekly report\". \ + Not \"this\" or \"that\"." + ), + "records": { + "type": "array", + "items": {"type": "string"}, + "description": "Watchman record ids (w-...) of the turns that did the \ + task. Their desktop calls are joined in the order they \ + happened.", + }, + "steps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "tool": {"type": "string"}, + "args": {"type": "object"} + }, + "required": ["tool"], + }, + "description": "The steps themselves, instead of records.", + }, + }, + "required": ["phrase"], + }, + }) +} + +fn routine_run_definition() -> Value { + json!({ + "name": "routine_run", + "description": format!( + "Do a saved routine again, on the real desktop, now: the phrase is matched by \ + its words (any order, filler ignored; see routine_match). Every step runs \ + through the same checks as a call you make yourself (blocked applications, \ + the operator's hooks, the stop file, the journal), in order, and stops at the \ + first one that fails. There is NO repair: on a failure you are told which \ + step and why, and what was already done stays done. Then do the whole task \ + yourself, from the screen as it is, and routine_save it under the same \ + phrase to replace the routine (or routine_forget it). A picture of the \ + window each step is for (else the screen) is taken before and after the \ + step through the same gates as `screenshot` (a hook that refuses \ + `screenshot` refuses them), kept for the last run only, and the paths of the \ + failed step's two pictures are in the failure; none is taken after a step \ + that typed a secret or before the next. THERE ARE NO ARGUMENTS FOR SECRETS: \ + a routine whose list shows `params_needed` reads each from the environment \ + variable the PERSON bound it to, at a terminal (`incu routine bind \ + \"\" INCU_PARAM_`); until every one is bound, for the \ + steps as they are now, it refuses before step 1 and says what to bind. \ + Passing `params` is refused. The value is used in memory for that run only. \ + The routine moves the mouse and keyboard of the machine the operator is \ + using. {UNTRUSTED}" + ), + "inputSchema": { + "type": "object", + "properties": { + "phrase": phrase_property("The words that ask for the routine."), + }, + "required": ["phrase"], + }, + }) +} + +fn routine_forget_definition() -> Value { + json!({ + "name": "routine_forget", + "description": "Delete the saved routine a phrase asks for. Use it for a routine \ + that no longer does the right thing.", + "inputSchema": { + "type": "object", + "properties": { + "phrase": phrase_property("The words that ask for the routine."), + }, + "required": ["phrase"], + }, + }) +} + +fn routine_list_definition() -> Value { + json!({ + "name": "routine_list", + "description": "Every saved routine that has not expired: its phrase, each step \ + with ALL the text it types and its full selector (read it: it is \ + what will run), the parameters it needs (`params_needed`), which \ + the person has bound to which variable (`params_bound`) and which \ + are not (`params_unbound`), when \ + it was learned and last used, whether the person kept it (never \ + expires), and how many days it has left.", + "inputSchema": {"type": "object", "properties": {}}, + }) +} + +fn routine_match_definition() -> Value { + json!({ + "name": "routine_match", + "description": "Which saved routine a phrase would run, without running it: the \ + routine and its steps (all the text each types, in full), or that \ + none matches. A phrase matches a routine when it has the same \ + words in any order, ignoring filler words (a, the, please, ...); \ + the words on each side of to, from, into, onto, over, under, \ + before or after must stay on that side (copy a to b is not copy \ + b to a); a near miss does not match. Call this before \ + routine_run to see what it would do.", + "inputSchema": { + "type": "object", + "properties": { + "phrase": phrase_property("The words to match."), + }, + "required": ["phrase"], + }, + }) +} diff --git a/crates/desktop/src/mcp/routine_tests.rs b/crates/desktop/src/mcp/routine_tests.rs new file mode 100644 index 0000000..72bbe1b --- /dev/null +++ b/crates/desktop/src/mcp/routine_tests.rs @@ -0,0 +1,1817 @@ +//! The routine tools and `watch` on a live server: the gates a replayed step +//! passes, stop-at-first-failure, the tiers, and what the journal and the +//! operator's hook are told. +//! +//! The "desktop" is the real one of the machine running the tests, as in +//! `selector_tests::live`, and every step is one that touches nothing: a wait +//! for a window that does not exist to be gone (found at once) or to appear (a +//! timeout of 100 ms). +//! +//! A test that starts a [`Server`] starts the input thread, which on Linux +//! needs a display: there it is ignored, as CI's Linux test job has none. The +//! tests that start no server run everywhere. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use serde_json::{json, Value}; + +use super::*; +use crate::hooks::HookOutcome; +use crate::permission::OBSERVING; +use crate::routine::params::{Probe, Values}; +use crate::routine::source::{Call, MemorySource, NoSource, Record, StepSource}; +use crate::routine::store::{Routines, Store}; +use crate::routine::{Routine, Step}; + +/// A window no program has: waiting for it to be gone is true at once. +const NO_SUCH_WINDOW: &str = "window=zz no such window 7f3a9c1e"; + +fn folder(name: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("incu-routine-tools-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("temp dir"); + dir +} + +/// A picture that is not one: the bytes a test's replay keeps, so that no test +/// photographs the screen of the machine that runs it (and none slows another +/// that looks at the real screen). +fn pretend_shot(_: &Step) -> Result, String> { + Ok(b"\xff\xd8 a picture".to_vec()) +} + +fn server_with(dir: &Path, source: Box) -> Server { + Server::new(InputThread::start().expect("input thread")).with_routines( + RoutineKit::new(Store::at(dir.join("routines.json")), source) + .with_shooter(Box::new(pretend_shot)) + // No sign-in form here to ask: every control the steps name is a + // plain one. (The rule for what is not is tested in `params`.) + .with_prober(Box::new(|_| Probe::Plain)), + ) +} + +/// The person's `incu routine bind`, as a test does it. +fn bind(dir: &Path, phrase: &str, param: &str, var: &str) { + Store::at(dir.join("routines.json")) + .update(|file| file.bind(phrase, param, var, crate::routine::now_ms())) + .expect("bind"); +} + +fn server(dir: &Path) -> Server { + server_with(dir, Box::new(NoSource)) +} + +fn call(server: &mut Server, name: &str, args: Value) -> Value { + let line = json!({ + "jsonrpc": "2.0", "id": 1, "method": "tools/call", + "params": {"name": name, "arguments": args} + }) + .to_string(); + let reply = server.handle_line(&line).expect("a reply was expected"); + serde_json::from_str(&reply).expect("the reply must be valid JSON") +} + +fn said(reply: &Value) -> String { + reply["result"]["content"][0]["text"] + .as_str() + .unwrap_or_default() + .to_string() +} + +fn refused(reply: &Value) -> bool { + reply["result"]["isError"] == true +} + +/// The JSON a tool answered. +fn answer(reply: &Value) -> Value { + assert!(!refused(reply), "{reply}"); + serde_json::from_str(&said(reply)).expect("the answer is JSON") +} + +fn gone() -> Value { + json!({"condition": "window_gone", "selector": NO_SUCH_WINDOW}) +} + +fn never_appears() -> Value { + json!({"condition": "window_appears", "selector": NO_SUCH_WINDOW, "timeout_ms": 100, "poll_ms": 50}) +} + +fn step(args: Value) -> Value { + json!({"tool": "wait_for", "args": args}) +} + +fn save_steps(s: &mut Server, phrase: &str, steps: Vec) -> Value { + call(s, "routine_save", json!({"phrase": phrase, "steps": steps})) +} + +/// The tools a pre-tool hook was asked about. +type Pre = Arc>>; +/// The tools a post-tool hook was told about, and the answer it was handed. +type Post = Arc>>; + +/// What a post-tool and a pre-tool hook were told, and a veto on request. +struct Recording { + veto: Option<&'static str>, + pre: Pre, + post: Post, + /// Written to after the first `wait_for`, to stop the desktop mid-routine. + stop_after_wait: Option, + /// The arguments each pre-tool hook was shown, by tool. + inputs: Post, +} + +impl Recording { + fn new(veto: Option<&'static str>) -> (Self, Pre, Post) { + let pre = Arc::new(Mutex::new(Vec::new())); + let post = Arc::new(Mutex::new(Vec::new())); + ( + Self { + veto, + pre: pre.clone(), + post: post.clone(), + stop_after_wait: None, + inputs: Arc::new(Mutex::new(Vec::new())), + }, + pre, + post, + ) + } +} + +impl Hooks for Recording { + fn run_pre_tool_use(&self, tool_name: &str, input: &str) -> HookOutcome { + self.inputs + .lock() + .unwrap() + .push((tool_name.to_string(), input.to_string())); + self.pre.lock().unwrap().push(tool_name.to_string()); + if self.veto == Some(tool_name) { + HookOutcome::deny(vec![format!("the operator does not allow {tool_name}")]) + } else { + HookOutcome::allow(Vec::new()) + } + } + + fn run_post_tool_use( + &self, + tool_name: &str, + _input: &str, + output: &str, + _err: bool, + ) -> HookOutcome { + self.post + .lock() + .unwrap() + .push((tool_name.to_string(), output.to_string())); + if let (Some(stop), "wait_for") = (&self.stop_after_wait, tool_name) { + crate::safety::request_stop(stop).expect("stop file"); + } + HookOutcome::allow(Vec::new()) + } +} + +#[test] +fn the_six_new_tools_are_offered_in_their_tiers() { + let listed: Vec = tool_definitions(); + let names: Vec<&str> = listed.iter().filter_map(|t| t["name"].as_str()).collect(); + for tool in [ + "watch", + "routine_save", + "routine_run", + "routine_forget", + "routine_list", + "routine_match", + ] { + assert!(names.contains(&tool), "{tool} is not offered: {names:?}"); + } + for tool in ["watch", "routine_list", "routine_match"] { + assert!(OBSERVING.contains(&tool), "{tool}"); + } + for tool in ["routine_save", "routine_run", "routine_forget"] { + assert!(ACTING.contains(&tool), "{tool}"); + } + // The picture after an action is offered where it means something. + let has_shot = |name: &str| { + listed.iter().find(|t| t["name"] == name).is_some_and(|t| { + t["inputSchema"]["properties"] + .get("screenshot_after") + .is_some() + }) + }; + assert!(has_shot("routine_run")); + assert!(!has_shot("routine_save")); + assert!(!has_shot("routine_forget")); + assert!(!has_shot("watch")); + // `watch` takes `window` as wait_for does. + let watch = listed.iter().find(|t| t["name"] == "watch").unwrap(); + assert!(watch["inputSchema"]["properties"].get("window").is_some()); + assert!(watch["description"].as_str().unwrap().contains("BLOCKS")); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_routine_is_saved_from_steps_listed_matched_and_forgotten() { + let dir = folder("lifecycle"); + let mut s = server(&dir); + let saved = answer(&save_steps(&mut s, "check the window", vec![step(gone())])); + assert_eq!(saved["saved"], "check the window"); + assert_eq!(saved["replaced_an_older_routine"], false); + + let listed = answer(&call(&mut s, "routine_list", json!({}))); + assert_eq!(listed["count"], 1); + assert_eq!(listed["routines"][0]["phrase"], "check the window"); + assert_eq!(listed["routines"][0]["expires_in_days"], 30); + + // The same words in another order, with filler, are the same request. + let matched = answer(&call( + &mut s, + "routine_match", + json!({"phrase": "Window, check it please"}), + )); + assert_eq!(matched["matched"], true); + assert_eq!(matched["routine"]["steps"].as_array().unwrap().len(), 1); + let near = answer(&call( + &mut s, + "routine_match", + json!({"phrase": "check the window and the door"}), + )); + assert_eq!(near["matched"], false); + + let forgotten = call(&mut s, "routine_forget", json!({"phrase": "window check"})); + assert!(!refused(&forgotten), "{forgotten}"); + assert_eq!(answer(&call(&mut s, "routine_list", json!({})))["count"], 0); + let again = call(&mut s, "routine_forget", json!({"phrase": "window check"})); + assert!( + refused(&again) && said(&again).contains("no routine matches"), + "{again}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_run_that_cannot_be_replayed_is_refused_with_the_reason_and_nothing_is_written() { + let dir = folder("refused"); + let mut s = server(&dir); + let cases = [ + ("close this", vec![step(gone())], "this"), + ( + "click the button", + vec![json!({"tool": "click_element", "args": {"id": 16}})], + "eN id", + ), + ( + "click the spot", + vec![json!({"tool": "click", "args": {"x": 4, "y": 5}})], + "coordinates", + ), + ( + "do nothing", + vec![json!({"tool": "ui_tree", "args": {}})], + "no desktop action", + ), + ]; + for (phrase, steps, why) in cases { + let reply = save_steps(&mut s, phrase, steps); + assert!(refused(&reply), "{phrase}: {reply}"); + assert!(said(&reply).contains(why), "{phrase}: {}", said(&reply)); + } + assert!( + !dir.join("routines.json").exists(), + "a refusal wrote a file" + ); + let neither = call(&mut s, "routine_save", json!({"phrase": "open it"})); + assert!(said(&neither).contains("records"), "{}", said(&neither)); + let both = call( + &mut s, + "routine_save", + json!({"phrase": "open it", "steps": [], "records": []}), + ); + assert!(said(&both).contains("not both"), "{}", said(&both)); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_routine_is_learned_from_watchman_records_in_the_order_they_happened() { + let dir = folder("records"); + let source = MemorySource::new() + // Named second, happened first. + .with(Record::new( + "w-b", + 2_000, + vec![Call::ok("mcp__desktop__wait_for", never_gone_second())], + )) + .with(Record::new( + "w-a", + 1_000, + vec![ + Call::ok("mcp__desktop__ui_tree", json!({})), + Call::ok("mcp__desktop__wait_for", gone()), + Call::ok("Bash", json!({"command": "ls"})), + ], + )); + let mut s = server_with(&dir, Box::new(source)); + let reply = call( + &mut s, + "routine_save", + json!({"phrase": "wait for it", "records": ["w-b", "w-a"]}), + ); + let saved = answer(&reply); + assert_eq!(saved["looking_steps_dropped"], 1); + assert_eq!(saved["left_out_not_desktop_tools"], json!(["Bash"])); + assert!(saved["warning"].is_string()); + let steps = saved["steps"].as_array().unwrap(); + assert_eq!(steps.len(), 2); + assert!( + steps[0].as_str().unwrap().contains("window_gone"), + "{steps:?}" + ); + assert!(steps[1].as_str().unwrap().contains("11111111"), "{steps:?}"); + + let unknown = call( + &mut s, + "routine_save", + json!({"phrase": "wait for it", "records": ["w-zzz"]}), + ); + assert!( + refused(&unknown) && said(&unknown).contains("w-zzz"), + "{unknown}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +fn never_gone_second() -> Value { + json!({"condition": "window_gone", "selector": "window=zz 11111111"}) +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_failed_recorded_run_is_not_learned() { + let dir = folder("failed-record"); + let source = MemorySource::new().with(Record::new( + "w-1", + 1, + vec![Call::ok("wait_for", gone()).failed("selector matches nothing")], + )); + let mut s = server_with(&dir, Box::new(source)); + let reply = call( + &mut s, + "routine_save", + json!({"phrase": "wait for it", "records": ["w-1"]}), + ); + assert!( + refused(&reply) && said(&reply).contains("the run failed"), + "{reply}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_saved_routine_runs_every_step_through_the_gates_and_the_journal() { + let dir = folder("run"); + let journal = dir.join("desktop.jsonl"); + let (hooks, pre, post) = Recording::new(None); + let mut s = server(&dir) + .with_journal(journal.clone()) + .with_hook_runner(Box::new(hooks)); + answer(&save_steps( + &mut s, + "check the windows", + vec![step(gone()), step(gone())], + )); + + let ran = answer(&call( + &mut s, + "routine_run", + json!({"phrase": "windows check"}), + )); + assert_eq!(ran["steps_run"], 2, "{ran}"); + assert_eq!(ran["ran"], "check the windows"); + + // The pre-tool hook saw the routine and then EACH step, as a live call. + assert_eq!( + *pre.lock().unwrap(), + ["routine_save", "routine_run", "wait_for", "wait_for"] + ); + assert_eq!( + post.lock() + .unwrap() + .iter() + .filter(|(t, _)| t == "wait_for") + .count(), + 2 + ); + // And the journal has a line for each step and one for the routine. + let lines = std::fs::read_to_string(&journal).unwrap(); + assert_eq!(lines.matches("\"t\":\"wait_for\"").count(), 2, "{lines}"); + assert!(lines.contains("\"t\":\"routine_run\""), "{lines}"); + assert!(lines.contains("check the windows"), "{lines}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn the_first_failing_step_stops_the_replay_and_the_rest_never_run() { + let dir = folder("stop-first"); + let journal = dir.join("desktop.jsonl"); + let (hooks, pre, _post) = Recording::new(None); + let mut s = server(&dir) + .with_journal(journal.clone()) + .with_hook_runner(Box::new(hooks)); + answer(&save_steps( + &mut s, + "three waits", + vec![step(gone()), step(never_appears()), step(gone())], + )); + let reply = call(&mut s, "routine_run", json!({"phrase": "three waits"})); + assert!(refused(&reply), "{reply}"); + let said = said(&reply); + assert!(said.contains("failed at step 2 of 3"), "{said}"); + assert!(said.contains("timed out"), "{said}"); + assert!(said.contains("nothing was retried or repaired"), "{said}"); + // Step 3 was never asked of the gate, let alone the desktop. + let waits = pre + .lock() + .unwrap() + .iter() + .filter(|t| *t == "wait_for") + .count(); + assert_eq!(waits, 2); + let lines = std::fs::read_to_string(&journal).unwrap(); + assert_eq!(lines.matches("\"t\":\"wait_for\"").count(), 2, "{lines}"); + assert!( + lines.contains("refused") && lines.contains("routine_run"), + "{lines}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn the_operators_hook_can_refuse_a_step_of_a_replay() { + let dir = folder("veto"); + let (hooks, _pre, _post) = Recording::new(Some("wait_for")); + let mut s = server(&dir).with_hook_runner(Box::new(hooks)); + answer(&save_steps(&mut s, "check the window", vec![step(gone())])); + let reply = call(&mut s, "routine_run", json!({"phrase": "check the window"})); + assert!(refused(&reply), "a replay got past the hook: {reply}"); + assert!( + said(&reply).contains("failed at step 1 of 1"), + "{}", + said(&reply) + ); + assert!( + said(&reply).contains("does not allow wait_for"), + "{}", + said(&reply) + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn the_stop_file_ends_a_replay_between_steps() { + let dir = folder("stop-file"); + let journal = dir.join("desktop.jsonl"); + let (mut hooks, _pre, _post) = Recording::new(None); + hooks.stop_after_wait = Some(crate::safety::stop_file_beside(&journal)); + let mut s = server(&dir) + .with_journal(journal) + .with_hook_runner(Box::new(hooks)); + answer(&save_steps( + &mut s, + "two waits", + vec![step(gone()), step(gone())], + )); + let reply = call(&mut s, "routine_run", json!({"phrase": "two waits"})); + assert!(refused(&reply), "{reply}"); + let said = said(&reply); + assert!(said.contains("failed at step 2 of 2"), "{said}"); + assert!(said.to_lowercase().contains("stop"), "{said}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_read_only_session_may_look_at_routines_and_may_not_change_or_run_them() { + let dir = folder("tier"); + let mut s = server(&dir).with_tier(Tier::Observe); + assert!(!refused(&call(&mut s, "routine_list", json!({})))); + assert!(!refused(&call( + &mut s, + "routine_match", + json!({"phrase": "open it"}) + ))); + for tool in ["routine_save", "routine_run", "routine_forget"] { + let reply = call( + &mut s, + tool, + json!({"phrase": "open it", "steps": [step(gone())]}), + ); + assert!(refused(&reply), "{tool}: {reply}"); + assert!( + said(&reply).contains("workspace-write"), + "{tool}: {}", + said(&reply) + ); + } + assert!(!dir.join("routines.json").exists()); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_routine_unused_for_thirty_days_is_gone_unless_it_was_taught() { + let dir = folder("expiry"); + let old = |phrase: &str, taught: bool| Routine { + phrase: phrase.to_string(), + steps: vec![Step::new("wait_for", gone())], + date: "2026-01-01".to_string(), + last_used_ms: 1_000, + taught, + bindings: Default::default(), + bound_steps: String::new(), + }; + let file = Routines { + version: 1, + routines: vec![old("forgotten task", false), old("taught task", true)], + }; + let store = Store::at(dir.join("routines.json")); + store.save(&file).unwrap(); + let mut s = server(&dir); + let listed = answer(&call(&mut s, "routine_list", json!({}))); + assert_eq!(listed["count"], 1, "{listed}"); + assert_eq!(listed["routines"][0]["phrase"], "taught task"); + assert!(listed["routines"][0].get("expires_in_days").is_none()); + let run = call(&mut s, "routine_run", json!({"phrase": "forgotten task"})); + assert!( + refused(&run) && said(&run).contains("no routine matches"), + "{run}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_stored_step_the_rules_refuse_runs_nothing() { + let dir = folder("tampered"); + let (hooks, pre, _post) = Recording::new(None); + let file = Routines { + version: 1, + routines: vec![Routine { + phrase: "edited by hand".to_string(), + steps: vec![ + Step::new("wait_for", gone()), + Step::new("click", json!({"x": 1, "y": 1})), + ], + date: String::new(), + last_used_ms: crate::routine::now_ms(), + taught: true, + bindings: Default::default(), + bound_steps: String::new(), + }], + }; + Store::at(dir.join("routines.json")).save(&file).unwrap(); + let mut s = server(&dir).with_hook_runner(Box::new(hooks)); + let reply = call(&mut s, "routine_run", json!({"phrase": "edited by hand"})); + assert!( + refused(&reply) && said(&reply).contains("Nothing was done"), + "{reply}" + ); + assert_eq!( + *pre.lock().unwrap(), + ["routine_run"], + "not even the first step was tried" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_server_without_routines_refuses_them_by_name() { + let mut s = Server::new(InputThread::start().expect("input thread")); + let reply = call(&mut s, "routine_list", json!({})); + assert!( + refused(&reply) && said(&reply).contains("not configured"), + "{reply}" + ); +} + +#[test] +fn a_click_on_a_link_or_button_is_replayed_as_an_invoke_and_other_clicks_are_not() { + use super::routine_tool::invoke_for; + let click = |args: Value| Step::new("click_element", args); + let selector = "app=chrome;window~=Wikipedia;role=link;name=Honshu"; + let invoked = invoke_for(&click(json!({"selector": selector}))).expect("a link is invoked"); + assert_eq!(invoked, json!({"selector": selector, "action": "invoke"})); + assert!(invoke_for(&click( + json!({"selector": "app=a;role=button;name=OK", "button": "left"}) + )) + .is_some()); + // A right click, a text field and anything else are clicked as recorded. + assert!(invoke_for(&click(json!({"selector": selector, "button": "right"}))).is_none()); + assert!(invoke_for(&click(json!({"selector": "app=a;role=edit;name=Body"}))).is_none()); + assert!( + invoke_for(&click(json!({"selector": "app=a;name=Save"}))).is_none(), + "no role" + ); + assert!(invoke_for(&Step::new("set_value", json!({"selector": selector}))).is_none()); +} + +#[test] +fn a_keyboard_steps_window_is_looked_up_at_replay_and_a_missing_one_is_a_failed_step() { + use super::routine_tool::live_args; + // Not a keyboard step: its arguments go through as they are. + let wait = Step::new("wait_for", gone()); + assert_eq!(live_args(&wait).unwrap(), gone()); + // A keyboard step whose window does not exist fails, naming it, before any + // key is sent; the tool is never asked. + let typing = Step::new( + "type_text", + json!({"text": "hi", "in_window": NO_SUCH_WINDOW}), + ); + let why = live_args(&typing).unwrap_err(); + assert!(why.contains("in_window"), "{why}"); + assert!(why.contains("zz no such window"), "{why}"); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_replayed_typing_step_into_a_window_that_is_not_there_fails_and_types_nothing() { + let dir = folder("typing"); + let (hooks, pre, _post) = Recording::new(None); + let mut s = server(&dir).with_hook_runner(Box::new(hooks)); + // It names the field it types into (a plain one, the desktop says), so its + // text is kept and the replay focuses the field first: in a window that is + // not there, that is where it fails. + let reply = save_steps( + &mut s, + "type a note", + vec![json!({"tool": "type_text", "args": { + "text": "hi", "in_window": NO_SUCH_WINDOW, + "selector": format!("{NO_SUCH_WINDOW};role=edit;name=Body") + }})], + ); + assert!(!refused(&reply), "{reply}"); + assert!(answer(&reply)["steps"][0] + .as_str() + .unwrap() + .contains("\"hi\"")); + let run = call(&mut s, "routine_run", json!({"phrase": "type a note"})); + assert!(refused(&run), "{run}"); + assert!( + said(&run).contains("failed at step 1 of 1"), + "{}", + said(&run) + ); + assert_eq!( + *pre.lock().unwrap(), + ["routine_save", "routine_run"], + "type_text never reached the gate" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A window no program has, as an `in_window`, for steps that type. +const TYPING_WINDOW: &str = NO_SUCH_WINDOW; + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_secret_typed_into_a_password_field_is_a_parameter_and_never_stored_or_journalled() { + let dir = folder("secret"); + let journal = dir.join("desktop.jsonl"); + let (hooks, pre, _post) = Recording::new(None); + let hook_inputs = hooks.inputs.clone(); + // Where the person keeps the secret: incu's own environment. + std::env::set_var("INCU_PARAM_TEST_VAULT_PASSWORD", "hunter2-secret"); + let mut s = server(&dir) + .with_journal(journal.clone()) + .with_hook_runner(Box::new(hooks)); + let steps = || { + json!([ + {"tool": "wait_for", "args": gone()}, + // Names its own plain field: the text is kept, the replay focuses it. + {"tool": "type_text", "args": { + "text": "plain words", "in_window": TYPING_WINDOW, + "selector": format!("{TYPING_WINDOW};role=edit;name=Body") + }}, + {"tool": "set_value", "args": {"selector": "app=vault;window=Sign in;role=edit;name=Password", "value": "hunter2-secret"}}, + ]) + }; + let reply = call( + &mut s, + "routine_save", + json!({"phrase": "unlock the vault", "steps": steps()}), + ); + let saved = answer(&reply); + assert_eq!(saved["parameters"][0]["name"], "password", "{saved}"); + assert_eq!(saved["parameters"][0]["step"], 3); + assert_eq!(saved["parameters"].as_array().unwrap().len(), 1, "{saved}"); + assert!(!said(&reply).contains("hunter2"), "{}", said(&reply)); + assert!(saved["parameters_note"] + .as_str() + .unwrap() + .contains("incu routine bind")); + // On disk: the placeholder, never the value. The ordinary text is kept. + let file = std::fs::read_to_string(dir.join("routines.json")).unwrap(); + assert!(!file.contains("hunter2"), "{file}"); + assert!(file.contains("{{password}}") && file.contains("plain words")); + // The listing says what the routine needs and that nothing is bound. + let listed = answer(&call(&mut s, "routine_list", json!({}))); + assert_eq!(listed["routines"][0]["params_needed"], json!(["password"])); + assert_eq!(listed["routines"][0]["params_unbound"], json!(["password"])); + + // Unbound, the run is refused before step 1, naming the person's command: no + // step is asked of any gate. + let refusal = call(&mut s, "routine_run", json!({"phrase": "vault unlock"})); + assert!(refused(&refusal), "{refusal}"); + assert!( + said(&refusal) + .contains("incu routine bind \"unlock the vault\" password INCU_PARAM_PASSWORD"), + "{}", + said(&refusal) + ); + assert!( + said(&refusal).contains("Nothing was done"), + "{}", + said(&refusal) + ); + // `params` is not an argument any more, in any spelling, and the secret in + // one is never repeated. + for params in [ + json!({"password": "hunter2-secret"}), + json!({"password": {"env": "INCU_PARAM_TEST_VAULT_PASSWORD"}}), + json!({"password": {"env": "HOME"}}), + ] { + let attempt = call( + &mut s, + "routine_run", + json!({"phrase": "vault unlock", "params": params}), + ); + assert!(refused(&attempt), "{attempt}"); + assert!( + said(&attempt).contains("incu routine bind"), + "{}", + said(&attempt) + ); + assert!(!said(&attempt).contains("hunter2"), "{}", said(&attempt)); + } + assert!( + !pre.lock().unwrap().iter().any(|t| t == "wait_for"), + "a step was tried before the parameter was bound" + ); + // The agent cannot bind: the only way is the person's command (the Store), + // and it takes only an INCU_PARAM_ variable. + let store = Store::at(dir.join("routines.json")); + for var in ["HOME", "PATH", "ANTHROPIC_API_KEY"] { + let why = store + .update(|file| file.bind("vault unlock", "password", var, 0)) + .unwrap_err(); + assert!(why.contains("INCU_PARAM_"), "{why}"); + } + + // The person binds it. Only the variable's NAME goes in the file. + bind( + &dir, + "vault unlock", + "password", + "INCU_PARAM_TEST_VAULT_PASSWORD", + ); + let file = std::fs::read_to_string(dir.join("routines.json")).unwrap(); + assert!(file.contains("INCU_PARAM_TEST_VAULT_PASSWORD"), "{file}"); + assert!(!file.contains("hunter2"), "{file}"); + let listed = answer(&call(&mut s, "routine_list", json!({}))); + assert_eq!( + listed["routines"][0]["params_bound"], + json!({"password": "INCU_PARAM_TEST_VAULT_PASSWORD"}) + ); + assert_eq!(listed["routines"][0]["params_unbound"], json!([])); + + // Bound, the run starts: it goes as far as it can on the test desktop (the + // first step passes, the second finds no such window) and whatever it says, + // the value is nowhere. + let ran = call(&mut s, "routine_run", json!({"phrase": "vault unlock"})); + assert!(refused(&ran), "{ran}"); + assert!( + said(&ran).contains("failed at step 2 of 3"), + "{}", + said(&ran) + ); + assert!(!said(&ran).contains("hunter2"), "{}", said(&ran)); + // The hooks were shown the call with every param value hidden, and no hook + // was shown the secret. + let shown = hook_inputs.lock().unwrap().clone(); + let run_inputs: Vec<&String> = shown + .iter() + .filter(|(tool, _)| tool == "routine_run") + .map(|(_, input)| input) + .collect(); + assert!(run_inputs.len() >= 5); + for input in &run_inputs { + assert!(!input.contains("hunter2"), "{input}"); + assert!(!input.contains("INCU_PARAM_TEST"), "{input}"); + assert!(!input.contains("\"HOME\""), "{input}"); + } + // The attempts at `params` were shown as ``. + assert_eq!( + run_inputs + .iter() + .filter(|i| i.contains("\"password\":\"\"")) + .count(), + 3 + ); + // No call of the replay (the steps) was shown it either. (What the agent + // wrote into `routine_save` itself is its own, and in its own transcript.) + assert!(shown + .iter() + .filter(|(tool, _)| tool != "routine_save") + .all(|(_, input)| !input.contains("hunter2"))); + let lines = std::fs::read_to_string(&journal).unwrap(); + assert!(!lines.contains("hunter2"), "{lines}"); + let file = std::fs::read_to_string(dir.join("routines.json")).unwrap(); + assert!(!file.contains("hunter2"), "{file}"); + + // The same steps saved again keep the binding; other steps drop it, and the + // person binds again after reading them. + let again = answer(&call( + &mut s, + "routine_save", + json!({"phrase": "unlock the vault", "steps": steps()}), + )); + assert!(again.get("bindings_dropped").is_none(), "{again}"); + let listed = answer(&call(&mut s, "routine_list", json!({}))); + assert_eq!(listed["routines"][0]["params_unbound"], json!([])); + let mut other = steps(); + other.as_array_mut().unwrap().push(step(gone())); + let changed = answer(&call( + &mut s, + "routine_save", + json!({"phrase": "unlock the vault", "steps": other}), + )); + assert!( + changed["bindings_dropped"] + .as_str() + .unwrap() + .contains("incu routine bind"), + "{changed}" + ); + let listed = answer(&call(&mut s, "routine_list", json!({}))); + assert_eq!(listed["routines"][0]["params_bound"], json!({})); + assert_eq!(listed["routines"][0]["params_unbound"], json!(["password"])); + let refusal = call(&mut s, "routine_run", json!({"phrase": "vault unlock"})); + assert!( + refused(&refusal) && said(&refusal).contains("not bound"), + "{refusal}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_recordings_placeholder_is_a_parameter_and_is_not_replayed() { + let dir = folder("redacted-record"); + let source = MemorySource::new().with(Record::new( + "w-1", + 1, + vec![Call::ok( + "mcp__desktop__type_text", + json!({"text": "[REDACTED]", "in_window": TYPING_WINDOW}), + )], + )); + let mut s = server_with(&dir, Box::new(source)); + let saved = answer(&call( + &mut s, + "routine_save", + json!({"phrase": "type the code", "records": ["w-1"]}), + )); + assert_eq!(saved["parameters"][0]["name"], "text"); + assert!(saved["parameters"][0]["why"] + .as_str() + .unwrap() + .contains("placeholder")); + let file = std::fs::read_to_string(dir.join("routines.json")).unwrap(); + assert!( + !file.contains("REDACTED") && file.contains("{{text}}"), + "{file}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn the_full_text_of_every_step_is_shown_when_saving_listing_and_matching() { + let dir = folder("review"); + let mut s = server(&dir); + let saved = answer(&call( + &mut s, + "routine_save", + json!({"phrase": "greet the room", "steps": [ + {"tool": "type_text", "args": { + "text": "hello there, everyone", "in_window": TYPING_WINDOW, + "selector": format!("{TYPING_WINDOW};role=edit;name=Greeting") + }} + ]}), + )); + let listed = answer(&call(&mut s, "routine_list", json!({}))); + let matched = answer(&call( + &mut s, + "routine_match", + json!({"phrase": "room greet"}), + )); + for (what, shown) in [ + ("save", &saved["steps"][0]), + ("list", &listed["routines"][0]["steps"][0]), + ("match", &matched["routine"]["steps"][0]), + ] { + let line = shown.as_str().unwrap(); + assert!(line.contains("hello there, everyone"), "{what}: {line}"); + assert!( + line.contains(TYPING_WINDOW), + "{what}: the window in full: {line}" + ); + assert!( + !line.contains("characters"), + "{what}: not just a length: {line}" + ); + } + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn the_agent_cannot_make_a_routine_kept_and_a_save_never_sets_it() { + let dir = folder("taught"); + let mut s = server(&dir); + let reply = call( + &mut s, + "routine_save", + json!({"phrase": "check the window", "steps": [step(gone())], "taught": true}), + ); + assert!(refused(&reply), "{reply}"); + assert!( + said(&reply).contains("incu routine keep"), + "{}", + said(&reply) + ); + assert!(!dir.join("routines.json").exists()); + let tool = tool_definitions() + .into_iter() + .find(|t| t["name"] == "routine_save") + .unwrap(); + assert!(tool["inputSchema"]["properties"].get("taught").is_none()); + let plain = answer(&save_steps(&mut s, "check the window", vec![step(gone())])); + assert_eq!(plain["taught"], false); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_failed_shell_call_or_a_failed_record_is_not_learned() { + let dir = folder("any-failure"); + let source = MemorySource::new() + .with(Record::new( + "w-bash", + 1, + vec![ + Call::ok("Bash", json!({"command": "false"})).failed("exit code 1"), + Call::ok("mcp__desktop__wait_for", gone()), + ], + )) + .with( + Record::new( + "w-failed", + 2, + vec![Call::ok("mcp__desktop__wait_for", gone())], + ) + .failing("outcome is \"failed\""), + ) + .with(Record::new( + "w-good", + 3, + vec![Call::ok("mcp__desktop__wait_for", gone())], + )); + let mut s = server_with(&dir, Box::new(source)); + for (id, says) in [ + ("w-bash", "exit code 1"), + ("w-failed", "says its run failed"), + ] { + let reply = call( + &mut s, + "routine_save", + json!({"phrase": "wait for it", "records": [id]}), + ); + assert!( + refused(&reply) && said(&reply).contains(says), + "{id}: {reply}" + ); + } + // One failed record among several refuses the whole. + let mixed = call( + &mut s, + "routine_save", + json!({"phrase": "wait for it", "records": ["w-good", "w-failed"]}), + ); + assert!(refused(&mixed), "{mixed}"); + assert!(!dir.join("routines.json").exists()); + assert!(!refused(&call( + &mut s, + "routine_save", + json!({"phrase": "wait for it", "records": ["w-good"]}) + ))); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn the_invoke_falls_back_to_the_recorded_click_only_when_it_was_refused_before_acting() { + use super::routine_tool::{after_invoke, refused_before_acting, AfterInvoke}; + // The control does not offer the action, as each platform says it. + for refused in [ + "button \"Send\" does not support \"invoke\"; it supports: toggle, focus", + "button \"Send\" does not offer \"invoke\" (AXPress); it offers: AXShowMenu", + "button \"Send\" offers no actions; use click_element or set_value", + ] { + assert!(refused_before_acting(refused), "{refused}"); + assert!(matches!( + after_invoke(Err(refused.to_string())), + AfterInvoke::ClickAsRecorded + )); + } + // Everything else may have happened after the control acted: no second try. + for after in [ + "could not invoke button \"Send\": HRESULT 0x80004005", + "button \"Send\" refused AXPress (error -25200)", + "the desktop did not answer within 5000 ms", + "desktop stopped by the operator", + "the operator does not support \"invoke\" today", + "selector `app=mail;role=button` matches no control", + "button \"Send\" is disabled", + ] { + assert!(!refused_before_acting(after), "{after}"); + match after_invoke(Err(after.to_string())) { + AfterInvoke::Done(Err(why)) => assert_eq!(why, after), + _ => panic!("{after} fell back to a second click"), + } + } + // A good answer is the step's answer. + match after_invoke(Ok(vec![super::tools::text( + "invoked button (unverified: x)".to_string(), + )])) { + AfterInvoke::Done(Ok(said)) => assert!(said.contains("invoked")), + _ => panic!("a good answer was not taken"), + } +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_replay_leaves_a_folder_of_pictures_and_only_the_last_runs() { + let dir = folder("pictures"); + let mut s = server(&dir); + answer(&save_steps( + &mut s, + "check the windows", + vec![step(gone()), step(gone())], + )); + // Left by earlier runs: a finished run's folder and a loose picture of an + // older layout must be cleared; a file that is not ours must not. + let root = dir.join("last-replay"); + let old_run = root.join("run-000000000000001-1-0"); + std::fs::create_dir_all(&old_run).unwrap(); + std::fs::write(old_run.join("step-001-after.jpg"), "old").unwrap(); + // Its mark, which nobody holds: the run is over. + std::fs::write(root.join("run-000000000000001-1-0.lock"), "").unwrap(); + std::fs::write(root.join("step-099-after.jpg"), "older").unwrap(); + std::fs::write(root.join("keep-me.txt"), "mine").unwrap(); + let ran = answer(&call( + &mut s, + "routine_run", + json!({"phrase": "windows check"}), + )); + // This run's own folder, under the root. + let mine = PathBuf::from(ran["pictures"]["folder"].as_str().unwrap()); + assert_eq!(mine.parent().unwrap(), root, "{ran}"); + assert!(mine + .file_name() + .unwrap() + .to_string_lossy() + .starts_with("run-")); + assert!(!old_run.exists(), "a finished run's pictures remain"); + assert!(!root.join("step-099-after.jpg").exists()); + assert!( + root.join("keep-me.txt").exists(), + "a file that is not ours was deleted" + ); + // One before and one after each of the two steps, and no more. + assert_eq!(ran["pictures"]["kept"], 4, "{ran}"); + let named = |dir: &Path| { + let mut names: Vec = std::fs::read_dir(dir) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().to_string()) + .filter(|n| n.starts_with("step-")) + .collect(); + names.sort(); + names + }; + assert_eq!( + named(&mine), + [ + "step-001-after.jpg", + "step-001-before.jpg", + "step-002-after.jpg", + "step-002-before.jpg" + ] + ); + // A second run starts again, in a folder of its own: only it is kept. + let again = answer(&call( + &mut s, + "routine_run", + json!({"phrase": "windows check"}), + )); + let second = PathBuf::from(again["pictures"]["folder"].as_str().unwrap()); + assert_ne!(second, mine); + assert!(!mine.exists(), "the first run's folder was kept"); + assert_eq!(named(&second).len(), 4); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_screen_that_cannot_be_captured_is_said_and_does_not_fail_the_replay() { + let dir = folder("no-pictures"); + let kit = RoutineKit::new(Store::at(dir.join("routines.json")), Box::new(NoSource)) + .with_shooter(Box::new(|_| Err("no display in this session".to_string()))); + let mut s = Server::new(InputThread::start().expect("input thread")).with_routines(kit); + answer(&save_steps(&mut s, "check the window", vec![step(gone())])); + let ran = answer(&call( + &mut s, + "routine_run", + json!({"phrase": "check the window"}), + )); + assert_eq!(ran["steps_run"], 1); + assert_eq!(ran["pictures"]["kept"], 0); + assert!( + ran["pictures"]["trouble"] + .as_str() + .unwrap() + .contains("no display"), + "{ran}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_failed_replay_names_the_pictures_of_the_failed_step() { + let dir = folder("failed-pictures"); + let mut s = server(&dir); + answer(&save_steps( + &mut s, + "three waits", + vec![step(gone()), step(never_appears()), step(gone())], + )); + let reply = call(&mut s, "routine_run", json!({"phrase": "three waits"})); + assert!(refused(&reply), "{reply}"); + let said = said(&reply); + // The failed step's two pictures are named, by full path, in this run's own + // folder, and exist. + assert!(said.contains("Pictures of step 2"), "{said}"); + let start = said.find("before: ").expect("the before picture") + "before: ".len(); + let end = said[start..].find(".jpg").unwrap() + start + ".jpg".len(); + let before = PathBuf::from(&said[start..end]); + let run = before.parent().unwrap().to_path_buf(); + assert_eq!(run.parent().unwrap(), dir.join("last-replay"), "{said}"); + assert!(before.ends_with("step-002-before.jpg"), "{before:?}"); + let after = run.join("step-002-after.jpg"); + assert!(said.contains(&after.display().to_string()), "{said}"); + assert!(before.exists() && after.exists()); + assert!( + !run.join("step-003-before.jpg").exists(), + "step 3 never ran" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn two_servers_saving_at_once_each_keep_the_others_routines() { + let dir = folder("two-servers"); + let workers: Vec<_> = (0..2) + .map(|w| { + let dir = dir.clone(); + std::thread::spawn(move || { + let mut s = server(&dir); + for i in 0..4 { + let reply = save_steps( + &mut s, + &format!("task from server{w} number{i}"), + vec![step(gone())], + ); + assert!(!refused(&reply), "{reply}"); + } + }) + }) + .collect(); + for worker in workers { + worker.join().unwrap(); + } + let kept = Store::at(dir.join("routines.json")).load().unwrap(); + assert_eq!( + kept.routines.len(), + 8, + "a save overwrote another server's routine" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_routine_saved_for_copying_one_way_is_not_run_for_the_other() { + let dir = folder("direction"); + let mut s = server(&dir); + answer(&save_steps( + &mut s, + "copy backup files to desktop", + vec![step(gone())], + )); + let other = answer(&call( + &mut s, + "routine_match", + json!({"phrase": "copy desktop files to backup"}), + )); + assert_eq!(other["matched"], false); + let same = answer(&call( + &mut s, + "routine_match", + json!({"phrase": "files backup copy, to the desktop"}), + )); + assert_eq!(same["matched"], true); + let run = call( + &mut s, + "routine_run", + json!({"phrase": "copy desktop files to backup"}), + ); + assert!( + refused(&run) && said(&run).contains("no routine matches"), + "{run}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A reader that gives its lines when told to and then, when its sender is +/// dropped, ends. +struct Wire(std::sync::mpsc::Receiver>, Vec); + +impl std::io::Read for Wire { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if self.1.is_empty() { + match self.0.recv() { + Ok(more) => self.1 = more, + Err(_) => return Ok(0), + } + } + let n = self.1.len().min(buf.len()); + buf[..n].copy_from_slice(&self.1[..n]); + self.1.drain(..n); + Ok(n) + } +} + +/// What a server answers to `lines` sent whole and then end of input, in order. +fn pipelined(s: Server, lines: &[Value]) -> Vec { + let (to_server, from_client) = std::sync::mpsc::channel::>(); + for line in lines { + to_server.send(format!("{line}\n").into_bytes()).unwrap(); + } + // The client closes its end the moment it has sent them. + drop(to_server); + let mut out = Vec::new(); + super::transport::run_lines(s, Wire(from_client, Vec::new()), &mut out).unwrap(); + String::from_utf8(out) + .unwrap() + .lines() + .map(|l| serde_json::from_str(l).unwrap()) + .collect() +} + +fn ping(id: u32) -> Value { + json!({"jsonrpc": "2.0", "id": id, "method": "ping"}) +} + +/// End of input does not mean the client is gone. A `watch` that was the last +/// request sent runs to its own end and is answered, as does one with another +/// request after it. +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_pipelined_watch_completes_normally_after_end_of_input() { + let dir = folder("eof-watch"); + let watch = |id: u32| { + json!({ + "jsonrpc": "2.0", "id": id, "method": "tools/call", + "params": {"name": "watch", "arguments": { + "condition": "window_appears", "selector": NO_SUCH_WINDOW, + "timeout_ms": 600, "poll_ms": 250 + }} + }) + }; + // The last line of all. + let started = std::time::Instant::now(); + let last = pipelined(server(&dir), &[ping(1), watch(2)]); + assert_eq!(last.len(), 2, "{last:?}"); + assert_eq!(last[1]["id"], 2); + assert_eq!(last[1]["result"]["isError"], false, "{}", last[1]); + assert!( + last[1]["result"]["content"][0]["text"] + .as_str() + .unwrap() + .contains("\"result\": \"timeout\""), + "the watch did not run its time: {}", + last[1] + ); + assert!( + started.elapsed() >= std::time::Duration::from_millis(500), + "ended after {:?}, before its 600 ms", + started.elapsed() + ); + // With a request after it. + let first = pipelined(server(&dir), &[watch(1), ping(2)]); + assert_eq!(first.len(), 2, "{first:?}"); + assert_eq!(first[0]["result"]["isError"], false, "{}", first[0]); + assert_eq!(first[1]["id"], 2); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Likewise a replay that was the last request sent. +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_pipelined_routine_run_completes_normally_after_end_of_input() { + let dir = folder("eof-run"); + let mut s = server(&dir); + answer(&save_steps( + &mut s, + "three waits", + vec![step(gone()), step(gone()), step(gone())], + )); + let run = json!({ + "jsonrpc": "2.0", "id": 2, "method": "tools/call", + "params": {"name": "routine_run", "arguments": {"phrase": "three waits"}} + }); + let replies = pipelined(s, &[ping(1), run]); + assert_eq!(replies.len(), 2, "{replies:?}"); + assert_eq!(replies[1]["result"]["isError"], false, "{}", replies[1]); + let text = replies[1]["result"]["content"][0]["text"].as_str().unwrap(); + let ran: Value = serde_json::from_str(text).unwrap(); + assert_eq!(ran["steps_run"], 3, "every step ran: {ran}"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// The one thing end of input does: stop the desktop, after every line before it +/// has been answered. +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn end_of_input_with_nothing_running_stops_the_desktop_and_returns() { + let dir = folder("eof-idle"); + let s = server(&dir); + let safety = s.safety(); + assert!(!safety.is_stopped()); + let (to_server, from_client) = std::sync::mpsc::channel::>(); + drop(to_server); + let mut out = Vec::new(); + super::transport::run_lines(s, Wire(from_client, Vec::new()), &mut out).unwrap(); + assert!(out.is_empty()); + assert!(safety.is_stopped(), "the desktop was left running"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A picture is a `screenshot`: a hook that refuses it refuses the picture. +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_hook_that_refuses_screenshot_refuses_the_replays_pictures_and_they_are_asked_of_it() { + let dir = folder("screenshot-veto"); + let (hooks, pre, _post) = Recording::new(Some("screenshot")); + // No `with_shooter`: the real path, through the gates. + let mut s = Server::new(InputThread::start().expect("input thread")) + .with_routines(RoutineKit::new( + Store::at(dir.join("routines.json")), + Box::new(NoSource), + )) + .with_journal(dir.join("desktop.jsonl")) + .with_hook_runner(Box::new(hooks)); + answer(&save_steps(&mut s, "check the window", vec![step(gone())])); + let ran = answer(&call( + &mut s, + "routine_run", + json!({"phrase": "check the window"}), + )); + // The routine ran; the pictures were asked of the hook and refused. + assert_eq!(ran["steps_run"], 1, "{ran}"); + assert_eq!(ran["pictures"]["kept"], 0, "{ran}"); + let trouble = ran["pictures"]["trouble"].as_str().unwrap(); + assert!(trouble.contains("does not allow screenshot"), "{trouble}"); + let asked: Vec = pre.lock().unwrap().clone(); + assert!( + asked.iter().filter(|t| *t == "screenshot").count() >= 2, + "a picture before and after were not asked of the gate: {asked:?}" + ); + // And the journal has them: not a side door. + let lines = std::fs::read_to_string(dir.join("desktop.jsonl")).unwrap(); + assert!( + lines.contains("screenshot: refused by a hook"), + "the refused pictures are not in the journal: {lines}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +/// No picture after a step that typed a secret, nor before the one after it. +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn no_picture_is_taken_after_a_step_that_typed_a_secret_or_before_the_next() { + use super::routine_tool::Live; + use crate::routine::pictures::When; + use crate::routine::replay::Runner; + let dir = folder("secret-pictures"); + let taken: Arc>> = Arc::new(Mutex::new(Vec::new())); + let log = taken.clone(); + let kit = RoutineKit::new(Store::at(dir.join("routines.json")), Box::new(NoSource)) + .with_shooter(Box::new(move |step| { + log.lock().unwrap().push(step.args.to_string()); + Ok(b"a picture".to_vec()) + })); + let mut s = Server::new(InputThread::start().expect("input thread")).with_routines(kit); + let wait = Step::new("wait_for", gone()); + let secret = Step::new( + "type_text", + json!({"text": "{{password}}", "in_window": TYPING_WINDOW}), + ); + let mut live = Live::new(&mut s, Values::default()); + // wait (1), the secret (2), a wait (3), and a wait again (4). + for (n, step) in [(1, &wait), (2, &secret), (3, &wait), (4, &wait)] { + live.look(n, step, When::Before); + live.look(n, step, When::After); + } + let note = live.pictures_note(); + // Taken: 1 before, 1 after, 2 before, 3 after, 4 before, 4 after. + assert_eq!(note["kept"], 6, "{note}"); + let skipped: Vec<&str> = note["skipped"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_str().unwrap()) + .collect(); + assert_eq!(skipped.len(), 2, "{skipped:?}"); + assert!( + skipped[0].starts_with("step 2: no picture after"), + "{skipped:?}" + ); + assert!( + skipped[1].starts_with("step 3: no picture before"), + "{skipped:?}" + ); + assert!(skipped[0].contains("secret") && skipped[1].contains("secret")); + assert_eq!(taken.lock().unwrap().len(), 6); + + // Plain text into a field whose NAME says secret is a secret too (a routine + // edited by hand: a saved one would have made it a parameter); text into a + // plain named field is photographed. + let typing_in = |field: &str| { + Step::new( + "type_text", + json!({"text": "words", "in_window": TYPING_WINDOW, + "selector": format!("{TYPING_WINDOW};role=edit;name={field}")}), + ) + }; + let fresh = |name: &str| { + Server::new(InputThread::start().expect("input thread")).with_routines( + RoutineKit::new(Store::at(dir.join(name)), Box::new(NoSource)) + .with_shooter(Box::new(|_| Ok(b"x".to_vec()))), + ) + }; + let mut s = fresh("other.json"); + let mut live = Live::new(&mut s, Values::default()); + assert_eq!(live.look(1, &typing_in("Password"), When::After), None); + assert_eq!(live.pictures_note()["skipped"].as_array().unwrap().len(), 1); + let mut s = fresh("third.json"); + let mut live = Live::new(&mut s, Values::default()); + assert!(live.look(1, &typing_in("Username"), When::After).is_some()); + let _ = std::fs::remove_dir_all(&dir); +} + +/// What a hook is shown of a step that has a parameter is the `{{name}}`, never +/// what the name stands for; and of `routine_run`, every `params` value hidden. +#[test] +fn the_hooks_are_not_shown_a_parameters_value_or_a_params_entry() { + use super::routine_tool::{hidden_params, hook_view}; + let stored = Step::new( + "type_text", + json!({"text": "{{password}}", "in_window": "x"}), + ); + let real = json!({"text": "hunter2-secret", "window": "w1"}); + let seen = hidden_params(&real, &stored); + assert_eq!(seen, json!({"text": "{{password}}", "window": "w1"})); + // A step with none is shown as it is. + let plain = Step::new("type_text", json!({"text": "hello", "in_window": "x"})); + assert_eq!( + hidden_params(&json!({"text": "hello", "window": "w1"}), &plain), + json!({"text": "hello", "window": "w1"}) + ); + let run = json!({"phrase": "unlock the vault", "params": { + "password": {"env": "INCU_PARAM_PASSWORD"}, "pin": "1234" + }}); + assert_eq!( + hook_view("routine_run", &run), + json!({"phrase": "unlock the vault", "params": {"password": "", "pin": ""}}) + ); + assert_eq!( + hook_view("routine_run", &json!({"phrase": "x", "params": "hunter2"})), + json!({"phrase": "x", "params": ""}) + ); + // Other tools, and a routine_run with no params, are as they were. + assert_eq!(hook_view("type_text", &real), real); + assert_eq!( + hook_view("routine_run", &json!({"phrase": "x"})), + json!({"phrase": "x"}) + ); +} + +/// The person's "kept" is for what they read: other steps saved over it are not +/// kept, and the reply says so. +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn saving_other_steps_over_a_kept_routine_un_keeps_it_and_says_so() { + let dir = folder("unkept"); + let mut s = server(&dir); + answer(&save_steps(&mut s, "check the window", vec![step(gone())])); + // The person keeps it, at a terminal. + let store = Store::at(dir.join("routines.json")); + store + .update(|file| { + assert!(file.set_taught("check the window", true, crate::routine::now_ms())); + Ok(()) + }) + .unwrap(); + // The same steps: still kept, nothing to say. + let same = answer(&save_steps(&mut s, "check the window", vec![step(gone())])); + assert_eq!(same["taught"], true, "{same}"); + assert!(same.get("kept_flag_dropped").is_none()); + // Other steps: not kept, and told. + let other = answer(&save_steps( + &mut s, + "check the window", + vec![step(gone()), step(never_appears())], + )); + assert_eq!(other["taught"], false, "{other}"); + assert!( + other["kept_flag_dropped"] + .as_str() + .unwrap() + .contains("incu routine keep"), + "{other}" + ); + let listed = answer(&call(&mut s, "routine_list", json!({}))); + assert_eq!(listed["routines"][0]["taught"], false); + assert_eq!(listed["routines"][0]["expires_in_days"], 30); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn every_line_before_the_end_of_input_is_still_answered() { + let dir = folder("eof-lines"); + let (to_server, from_client) = std::sync::mpsc::channel::>(); + for id in 1..=3 { + to_server + .send(format!("{{\"jsonrpc\":\"2.0\",\"id\":{id},\"method\":\"ping\"}}\n").into_bytes()) + .unwrap(); + } + drop(to_server); + let mut out = Vec::new(); + super::transport::run_lines(server(&dir), Wire(from_client, Vec::new()), &mut out).unwrap(); + let replies = String::from_utf8(out).unwrap(); + assert_eq!(replies.lines().count(), 3, "{replies}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn watch_returns_at_once_for_a_condition_that_holds_and_tells_the_hook_what_it_saw() { + let dir = folder("watch"); + let journal = dir.join("desktop.jsonl"); + let (hooks, _pre, post) = Recording::new(None); + let mut s = server(&dir) + .with_journal(journal.clone()) + .with_hook_runner(Box::new(hooks)); + let reply = call( + &mut s, + "watch", + json!({"condition": "window_gone", "selector": NO_SUCH_WINDOW, "message": "the export finished"}), + ); + let text = said(&reply); + assert!(!refused(&reply), "{text}"); + let json_part = &text[..=text.rfind('}').unwrap()]; + let body: Value = serde_json::from_str(json_part).unwrap(); + assert_eq!(body["result"], "found"); + assert_eq!(body["message"], "the export finished"); + // The hook for `watch` was handed the reply; no other tool's hook gets one. + let told = post.lock().unwrap().clone(); + let (name, output) = told.iter().find(|(n, _)| n == "watch").expect("watch hook"); + assert_eq!(name, "watch"); + assert!( + output.contains("the export finished") && output.contains("found"), + "{output}" + ); + let lines = std::fs::read_to_string(&journal).unwrap(); + assert!( + lines.contains("started:"), + "the begin is journalled: {lines}" + ); + assert!( + !lines.contains("the export finished"), + "the message is free text: {lines}" + ); + + call(&mut s, "wait_for", gone()); + let told = post.lock().unwrap().clone(); + assert!(told + .iter() + .filter(|(n, _)| n == "wait_for") + .all(|(_, out)| out.is_empty())); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_watch_that_times_out_is_a_reply_and_still_notifies() { + let dir = folder("watch-timeout"); + let (hooks, _pre, post) = Recording::new(None); + let mut s = server(&dir).with_hook_runner(Box::new(hooks)); + let reply = call( + &mut s, + "watch", + json!({"condition": "window_appears", "selector": NO_SUCH_WINDOW, "timeout_ms": 300, "poll_ms": 250}), + ); + assert!(!refused(&reply), "{reply}"); + assert!( + said(&reply).contains("\"result\": \"timeout\""), + "{}", + said(&reply) + ); + let told = post.lock().unwrap().clone(); + assert!( + told.iter() + .any(|(n, out)| n == "watch" && out.contains("timeout")), + "{told:?}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn the_stop_file_ends_a_watch_that_has_a_day_to_run() { + let dir = folder("watch-stop"); + let journal = dir.join("desktop.jsonl"); + let stop = crate::safety::stop_file_beside(&journal); + let mut s = server(&dir).with_journal(journal); + // The stop arrives while the watch is polling, not before it starts. + let stopper = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(400)); + crate::safety::request_stop(&stop).expect("stop file"); + }); + let started = std::time::Instant::now(); + let reply = call( + &mut s, + "watch", + json!({"condition": "window_appears", "selector": NO_SUCH_WINDOW, "timeout_ms": 86_400_000}), + ); + stopper.join().expect("the stopper thread"); + assert!(refused(&reply), "{reply}"); + assert!( + started.elapsed() < std::time::Duration::from_secs(10), + "a stop took {:?}", + started.elapsed() + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +#[cfg_attr( + target_os = "linux", + ignore = "starts the input thread: needs a display" +)] +fn a_watch_message_that_is_too_long_or_not_text_is_refused() { + let dir = folder("watch-message"); + let mut s = server(&dir); + let long = "x".repeat(201); + for message in [json!(long), json!(5)] { + let reply = call( + &mut s, + "watch", + json!({"condition": "window_gone", "selector": NO_SUCH_WINDOW, "message": message}), + ); + assert!( + refused(&reply) && said(&reply).contains("message"), + "{reply}" + ); + } + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/desktop/src/mcp/routine_tool.rs b/crates/desktop/src/mcp/routine_tool.rs new file mode 100644 index 0000000..40102be --- /dev/null +++ b/crates/desktop/src/mcp/routine_tool.rs @@ -0,0 +1,817 @@ +//! The five routine tools: `routine_save`, `routine_run`, `routine_forget` +//! (acting tier) and `routine_list`, `routine_match` (observing tier). +//! +//! The rules are in [`crate::routine`]; this file is where they meet the live +//! server. Two things are decided here and nowhere else: +//! +//! - **A replay runs through the server's own gates.** [`Live`] sends every +//! step to [`Server::gated_call`], the function `tools/call` itself uses, so +//! each step gets the stop file, the permission tier, the operator's pre-tool +//! hook (which sees the real arguments), the blocked-application guards, the +//! action-rate cap, the post-tool hook and a journal line of its own. The +//! routine as a whole is one more journalled call. A replay cannot do what a +//! live call would be refused. +//! - **The engine is optional.** A server with no step source still lists, +//! matches, replays and forgets routines, and saves one from an explicit list +//! of steps; only saving from Watchman record ids needs the engine. +//! +//! # Wired to +//! +//! - **Called by** `Server::run_tool` in [`super::tools`]. +//! - **Calls** [`crate::routine`] for the rules and the file, and +//! `Server::gated_call` for every replayed step. + +use serde_json::{json, Value}; + +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::PathBuf; + +use crate::routine::learn::{check_phrase, check_record, learn_with, Cleaned, IN_WINDOW}; +use crate::routine::params::{self, types_a_secret, Probe, Values, Why}; +use crate::routine::pictures::{Pictures, When}; +use crate::routine::replay::{replay, Runner}; +use crate::routine::source::{Call, Record, StepSource}; +use crate::routine::store::Store; +use crate::routine::words::word_set; +use crate::routine::{now_ms, Routine, Step}; + +use crate::selector::Selector; + +use super::tools::{resolve_selector, text, text_of, window_id_for}; +use super::Server; + +/// The most record ids one `routine_save` reads. +const MAX_RECORDS: usize = 20; + +/// The longest phrase a routine may be named by, in characters. +const MAX_PHRASE_CHARS: usize = 200; + +/// The routines file and where records come from, held by a [`Server`]. +pub struct RoutineKit { + store: Store, + source: Box, + /// Takes the picture of a step; incu's own capture when `None`. Replaced in + /// tests, so that what is kept and named is tested without photographing the + /// screen of the machine running them. + shooter: Option, + /// Asks what the desktop says of a control (a password field or not); the + /// live desktop when `None`. Replaced in tests, which have no sign-in form to + /// ask about. + prober: Option, +} + +/// What takes the picture of the window a step is for: the image's bytes (JPEG). +pub type Shooter = Box Result, String> + Send>; + +/// What says whether the control a raw selector names is a password field. +pub type Prober = Box Probe + Send>; + +impl RoutineKit { + /// A kit over `store`, reading records from `source`. + #[must_use] + pub fn new(store: Store, source: Box) -> Self { + Self { + store, + source, + shooter: None, + prober: None, + } + } + + /// The same kit, asking `prober` what the desktop says of a control when a + /// routine is saved. + #[must_use] + pub fn with_prober(mut self, prober: Prober) -> Self { + self.prober = Some(prober); + self + } + + /// The same kit, taking its step pictures with `shooter` and not incu's own + /// capture. + #[must_use] + pub fn with_shooter(mut self, shooter: Shooter) -> Self { + self.shooter = Some(shooter); + self + } + + /// Where the routines file is. + #[must_use] + pub fn path(&self) -> &std::path::Path { + self.store.path() + } +} + +impl Server { + /// Offer the routine tools over `kit`. Without this they refuse by name: + /// a server built for a protocol test has no routines file to write. + #[must_use] + pub fn with_routines(mut self, kit: RoutineKit) -> Self { + self.routines = Some(kit); + self + } + + /// The kit, or the refusal saying there is none. + fn kit(&self) -> Result<&RoutineKit, String> { + self.routines + .as_ref() + .ok_or_else(|| "routines are not configured on this server".to_string()) + } + + /// One of the five routine tools. Called by `Server::run_tool`. + pub(super) fn routine_tool(&mut self, name: &str, args: &Value) -> Result, String> { + match name { + "routine_save" => self.routine_save(args), + "routine_run" => self.routine_run(args), + "routine_forget" => self.routine_forget(args), + "routine_list" => self.routine_list(), + "routine_match" => self.routine_match(args), + other => Err(format!("no tool named '{other}'")), + } + } + + /// `routine_save`: learn a routine from Watchman records or from steps. + fn routine_save(&mut self, args: &Value) -> Result, String> { + let phrase = phrase_arg(args)?; + check_phrase(&phrase).map_err(|r| r.to_string())?; + // Being kept (never expiring) is the person's to give, by `incu routine + // keep`. Said plainly, so an agent does not believe it did it. + if args.get("taught").is_some() { + return Err( + "`taught` is not an argument: only the person keeps a routine \ + from expiring, with `incu routine keep \"\"` at a terminal" + .to_string(), + ); + } + let kit = self.kit()?; + // What the desktop says of each control a typing step names (a password + // field, a plain one, or nothing it could check), asked once each. + let seen: RefCell> = RefCell::new(HashMap::new()); + let probe = |raw: &str| { + *seen + .borrow_mut() + .entry(raw.to_string()) + .or_insert_with(|| match &kit.prober { + Some(prober) => prober(raw), + None => probe_control(raw), + }) + }; + let calls: Vec = match (args.get("records"), args.get("steps")) { + (Some(_), Some(_)) => { + return Err("give `records` or `steps`, not both".to_string()); + } + (None, None) => { + return Err( + "give `records` (Watchman record ids) or `steps` to learn from".to_string(), + ); + } + (Some(ids), None) => { + let records = records_of(kit.source.as_ref(), ids)?; + // A run that failed is not kept, whatever calls it holds. + for record in &records { + check_record(record).map_err(|r| r.to_string())?; + } + records.into_iter().flat_map(|r| r.calls).collect() + } + (None, Some(steps)) => steps_of(steps)? + .iter() + .map(|s| Call::ok(&s.tool, s.args.clone())) + .collect(), + }; + let cleaned = learn_with(&phrase, &calls, &probe).map_err(|r| r.to_string())?; + let now = now_ms(); + let saved = kit + .store + .update(|file| file.learn(&phrase, cleaned.steps.clone(), false, now))?; + Ok(vec![text(saved_reply( + &phrase, + &cleaned, + &saved, + kit.store.path(), + ))]) + } + + /// `routine_run`: replay the routine a phrase asks for. + fn routine_run(&mut self, args: &Value) -> Result, String> { + let phrase = phrase_arg(args)?; + // The agent does not say where a secret comes from: that is bound by the + // person. Said plainly, so it does not try again with another spelling. + if args.get("params").is_some_and(|p| !p.is_null()) { + return Err( + "routine_run takes no `params`: a secret the routine does not keep is read \ + from the environment variable the PERSON bound it to, at a terminal, with \ + `incu routine bind \"\" INCU_PARAM_`; routine_list \ + shows what each routine needs (`params_needed`) and what is bound" + .to_string(), + ); + } + let routine = self.find_routine(&phrase)?; + // Every parameter must be bound, for these steps, and its variable set, + // before anything runs: refused here, not half way through. + let values = Values::from_routine(&routine)?; + let mut live = Live::new(self, values); + let outcome = replay(&routine, &mut live); + let pictures = live.pictures_note(); + let report = outcome.map_err(|failure| failure.to_string())?; + // Used, so its thirty days begin again. Not a reason to fail a routine + // that has just run to the end. + let remembered = self.remember_use(&routine.phrase); + let steps: Vec = report + .ran + .iter() + .enumerate() + .map(|(i, r)| json!(format!("{}. {} -> {}", i + 1, r.summary, r.answer))) + .collect(); + let mut reply = json!({ + "ran": routine.phrase, + "steps_run": report.ran.len(), + "steps": steps, + "pictures": pictures, + }); + if let Err(why) = remembered { + reply["note"] = json!(format!( + "it ran, but its last-used time was not saved: {why}" + )); + } + Ok(vec![text(pretty(&reply))]) + } + + /// `routine_forget`: delete the routine a phrase asks for. + fn routine_forget(&mut self, args: &Value) -> Result, String> { + let phrase = phrase_arg(args)?; + let kit = self.kit()?; + kit.store.update(|file| { + if file.forget(&phrase, now_ms()) { + Ok(()) + } else { + Err(format!( + "no routine matches \"{phrase}\"; routine_list shows what is saved" + )) + } + })?; + Ok(vec![text(format!("forgot the routine for \"{phrase}\""))]) + } + + /// `routine_list`: every routine that has not expired. + fn routine_list(&self) -> Result, String> { + let kit = self.kit()?; + let file = kit.store.load()?; + let now = now_ms(); + let routines: Vec = file.live(now).map(|r| describe(r, now)).collect(); + Ok(vec![text(pretty(&json!({ + "count": routines.len(), + "routines": routines, + "file": kit.store.path().display().to_string(), + })))]) + } + + /// `routine_match`: which routine a phrase would run, without running it. + fn routine_match(&self, args: &Value) -> Result, String> { + let phrase = phrase_arg(args)?; + let kit = self.kit()?; + let file = kit.store.load()?; + let now = now_ms(); + let words: Vec = word_set(&phrase).into_iter().collect(); + let reply = match file.matching(&phrase, now) { + Some(routine) => { + json!({"matched": true, "words": words, "routine": describe(routine, now)}) + } + None => json!({"matched": false, "words": words}), + }; + Ok(vec![text(pretty(&reply))]) + } + + /// The live routine `phrase` asks for. + fn find_routine(&self, phrase: &str) -> Result { + let file = self.kit()?.store.load()?; + file.matching(phrase, now_ms()).cloned().ok_or_else(|| { + format!("no routine matches \"{phrase}\"; routine_list shows what is saved") + }) + } + + /// Record that the routine `phrase` ran to the end, reading the file again + /// first so a routine saved meanwhile is not overwritten. + fn remember_use(&self, phrase: &str) -> Result<(), String> { + self.kit()?.store.update(|file| { + file.used(phrase, now_ms()); + Ok(()) + }) + } + + /// One tool call through every gate, as `tools/call` runs it. The content it + /// answered, or the words it was refused with. + /// + /// Called by `Server::call_tool` for the call itself, and by [`Live`] for + /// each step of a replay: one path, so a step can never be let through by a + /// door a live call does not have. + pub(super) fn gated_call(&mut self, name: &str, args: &Value) -> Result, String> { + let seen = hook_view(name, args); + self.gated_call_as(name, args, &seen) + } + + /// [`Server::gated_call`] with `seen` as the arguments the operator's hooks + /// are shown, which may differ from `args` (the call's real arguments) only + /// by having secrets taken out: see [`hook_view`] and the replay's parameters. + pub(super) fn gated_call_as( + &mut self, + name: &str, + args: &Value, + seen: &Value, + ) -> Result, String> { + self.catch_stop_file(); + let permitted = self.authorise(name); + let raw_args = seen.to_string(); + let vetoed = self.hook_veto(&permitted, name, &raw_args); + let outcome = self.outcome(&permitted, vetoed, name, args); + self.tell_post_hooks(name, &raw_args, &outcome); + self.record_call(name, args, &outcome); + outcome + } +} + +/// What the operator's hooks are shown of a call's arguments: all of them, except +/// that every value in `routine_run`'s `params` is `""`. +/// +/// A hook sees the real arguments of a call (it is the operator's own, and a +/// pre-tool hook can only judge what it can read), but a hook is also how +/// Watchman and other recorders see calls, so nothing the caller passed to name +/// a secret is handed on. A `params` entry is only a reference to where the +/// person keeps the value, never the value, and still it is not repeated. +pub(super) fn hook_view(name: &str, args: &Value) -> Value { + let mut seen = args.clone(); + if name == "routine_run" { + if let Some(params) = seen.get_mut("params").and_then(Value::as_object_mut) { + for value in params.values_mut() { + *value = json!(""); + } + } else if seen.get("params").is_some_and(|p| !p.is_null()) { + seen["params"] = json!(""); + } + } + seen +} + +/// The live desktop as a [`Runner`]: every step through the server's gates. +pub(super) struct Live<'a> { + server: &'a mut Server, + /// What the caller gave for the routine's parameters. In memory only. + values: Values, + /// The folder this run's pictures go to, or `None` when it could not be made. + pictures: Option, + /// Why there are no pictures, or fewer: the first thing that went wrong. + picture_trouble: Option, + /// The step just run typed a secret: the picture before the next is skipped. + skip_next_before: bool, + /// The pictures that were not taken because a secret was on screen. + skipped: Vec, +} + +impl<'a> Live<'a> { + /// A run over `server`, with the parameters `values` and a folder of its own + /// for its pictures (the finished runs' are cleared). + pub(super) fn new(server: &'a mut Server, values: Values) -> Self { + let (pictures, picture_trouble) = match server + .kit() + .map(|kit| kit.store.pictures_dir()) + .and_then(Pictures::begin) + { + Ok(pictures) => (Some(pictures), None), + Err(why) => (None, Some(format!("no pictures: {why}"))), + }; + Live { + server, + values, + pictures, + picture_trouble, + skip_next_before: false, + skipped: Vec::new(), + } + } + + /// What the reply says of the pictures: where they are and how many, or why + /// there are none, and which were left untaken because a secret was on screen. + pub(super) fn pictures_note(&self) -> Value { + let mut note = match &self.pictures { + Some(pictures) => json!({ + "folder": pictures.dir().display().to_string(), + "kept": pictures.count(), + "note": "one picture before and one after each step, of the last run only", + }), + None => json!({"kept": 0}), + }; + if let Some(trouble) = &self.picture_trouble { + note["trouble"] = json!(trouble); + } + if !self.skipped.is_empty() { + note["skipped"] = json!(self.skipped); + } + note + } + + /// One step, as the live desktop runs it (see [`Runner::run`]). `stored` is + /// the step as saved, `filled` the same with its parameters in. + fn step(&mut self, stored: &Step, filled: &Step) -> Result { + let mut args = live_args(filled)?; + // A typing step that names the field it types into focuses it first, then + // types (the tool takes no selector, so it is not passed on). This is what + // makes the field the step names the field its text goes to: without it + // the name would be a claim the replay never checked. A field that cannot + // be focused fails the step, and the replay stops before anything is typed. + if matches!(filled.tool.as_str(), "type_text" | "paste_text") { + if let Some(map) = args.as_object_mut() { + if let Some(Value::String(field)) = map.remove("selector") { + let focus = json!({"selector": field, "action": "focus"}); + self.server + .gated_call("element_action", &focus) + .map_err(|why| format!("could not focus the field it types into: {why}"))?; + } + } + } + // What the hooks are shown: the step's real arguments with each + // parameter's value left as its `{{name}}`. + let seen = hidden_params(&args, stored); + // A link or a button is invoked through its own pattern when it has one: + // that needs no mouse, works for a control scrolled out of view and takes + // no focus. The recorded click runs INSTEAD only when the invoke was + // refused before it did anything; see [`after_invoke`]. + if let Some(invoke) = invoke_for(filled) { + match after_invoke(self.server.gated_call("element_action", &invoke)) { + AfterInvoke::Done(answer) => return answer, + AfterInvoke::ClickAsRecorded => {} + } + } + self.server + .gated_call_as(&filled.tool, &args, &seen) + .map(|content| text_of(&content)) + } + + /// The picture of the window `step` is for (else the screen), taken THROUGH + /// the gates a `screenshot` call passes: its tier, the operator's pre-tool + /// hook (a hook that refuses `screenshot` refuses these), the + /// blocked-application guards, the stop file, the post-tool hook and the + /// journal. Not a side door to the screen. Tests replace it ([`Shooter`]). + fn take(&mut self, step: &Step) -> Result, String> { + if let Some(shooter) = self.server.kit().ok().and_then(|kit| kit.shooter.as_ref()) { + return shooter(step); + } + let mut args = json!({"format": "jpeg"}); + if let Some(window) = step_window(step).and_then(|s| window_id_for(&s).ok()) { + args["window"] = json!(window); + } + let content = self.server.gated_call("screenshot", &args)?; + image_bytes(&content) + } +} + +impl Runner for Live<'_> { + fn run(&mut self, _n: usize, step: &Step) -> Result { + // The parameters go into the arguments of this one call, in memory, and + // come out of whatever it answers. + let filled = self.values.fill(step); + self.step(step, &filled) + .map(|answer| self.values.scrub(&answer)) + .map_err(|why| self.values.scrub(&why)) + } + + fn settle(&mut self) { + std::thread::sleep(crate::limits::get().settle); + } + + fn look(&mut self, n: usize, step: &Step, when: When) -> Option { + self.pictures.as_ref()?; + // A picture is of a screen with a password on it: not when this step has + // just typed a secret (its field is filled in, and may show), nor before + // the one after it (it is still there). + match when { + When::Before if std::mem::take(&mut self.skip_next_before) => { + self.skipped.push(format!( + "step {n}: no picture before: the step before it typed a secret" + )); + return None; + } + When::After if types_a_secret(step) => { + self.skip_next_before = true; + self.skipped.push(format!( + "step {n}: no picture after: it typed a secret (a parameter, or text \ + into a field whose name says it is secret)" + )); + return None; + } + _ => {} + } + let taken = self.take(step); + let pictures = self.pictures.as_mut()?; + match taken.and_then(|bytes| pictures.keep(n, when, "jpg", &bytes)) { + Ok(path) => Some(path), + Err(why) => { + self.picture_trouble + .get_or_insert(format!("step {n}: no picture: {why}")); + None + } + } + } +} + +/// `args` (a step's real arguments for the tool) with the typed text of a step +/// that has parameters put back to the stored `{{name}}` form, for the hooks. +pub(super) fn hidden_params(args: &Value, stored: &Step) -> Value { + if params::names_of(stored).is_empty() { + return args.clone(); + } + let mut seen = args.clone(); + for key in ["text", "value"] { + if let (Some(template), Some(slot)) = (stored.args.get(key), seen.get_mut(key)) { + *slot = template.clone(); + } + } + seen +} + +/// The bytes of the image in a tool's answer. +fn image_bytes(content: &[Value]) -> Result, String> { + use base64::Engine; + let data = content + .iter() + .find(|block| block.get("type").and_then(Value::as_str) == Some("image")) + .and_then(|block| block.get("data")) + .and_then(Value::as_str) + .ok_or("the screenshot answered no image")?; + base64::engine::general_purpose::STANDARD + .decode(data) + .map_err(|e| format!("the screenshot's image is not base64: {e}")) +} + +/// What to do after the `element_action invoke` tried in place of a recorded +/// click. +pub(super) enum AfterInvoke { + /// That is the step's answer, good or bad. + Done(Result), + /// The invoke was refused before it did anything: run the recorded click. + ClickAsRecorded, +} + +/// Decide from what the invoke answered. +/// +/// The click runs after an error ONLY when the error says the control cannot be +/// invoked at all ([`refused_before_acting`]). Any other error (the pattern call +/// failed, the call timed out, the operator's hook or the stop refused it) may +/// have come after the control acted, and a second action on top would press a +/// button twice: the step fails with that error and the replay stops. +pub(super) fn after_invoke(answer: Result, String>) -> AfterInvoke { + match answer { + Ok(content) => AfterInvoke::Done(Ok(text_of(&content))), + Err(why) if refused_before_acting(&why) => AfterInvoke::ClickAsRecorded, + Err(why) => AfterInvoke::Done(Err(why)), + } +} + +/// Whether an `element_action invoke` error says the control does not offer the +/// action: nothing was done, as each platform words it (UI Automation: "does not +/// support", macOS and Linux: "does not offer", "offers no actions"). +/// +/// Narrow on purpose. It matches the whole of each platform's message (the +/// action's name in quotes AND the list of what the control does offer that +/// follows it), so a message that merely contains some of those words (an +/// operator's hook explaining a refusal) is not taken for one. +pub(super) fn refused_before_acting(why: &str) -> bool { + let said = why.to_lowercase(); + said.contains("does not support \"invoke\"; it supports:") + || (said.contains("does not offer \"invoke\"") && said.contains("; it offers:")) + || said.contains("offers no actions; use click_element or set_value") +} + +/// The window a step is for, as a selector of only its window part: the one its +/// own selector names, or its `in_window`. +fn step_window(step: &Step) -> Option { + let raw = step + .args + .get("selector") + .or_else(|| step.args.get(IN_WINDOW))? + .as_str()?; + let whole = Selector::parse(raw).ok()?; + whole + .has_window_part() + .then(|| Selector::parse(&whole.render_window_part()).ok())? +} + +/// What the desktop says of the control `raw` names, now: a password field, a +/// plain one, or nothing it could check. Asked when a routine is saved, of each +/// control a typing step names; the control may not be on screen any more, and +/// then the answer is [`Probe::Unknown`], which is not "plain": a text is kept +/// only for a control positively known to be plain. +fn probe_control(raw: &str) -> Probe { + let Ok(selector) = Selector::parse(raw) else { + return Probe::Unknown; + }; + // Only a selector that says which window: without one it would be looked for + // in whatever is in front. + if !selector.has_window_part() || !selector.has_element_part() { + return Probe::Unknown; + } + match resolve_selector(None, &selector) { + Ok(found) if found.element.secret => Probe::Secret, + Ok(_) => Probe::Plain, + Err(_) => Probe::Unknown, + } +} + +/// The arguments the tool is called with for `step`. A keyboard step's +/// `in_window` (a window-only selector) becomes the `window` of the window it +/// names NOW; the tool then refuses unless that window has the focus, so keys +/// never go to whatever happens to be in front. +pub(super) fn live_args(step: &Step) -> Result { + let mut args = step.args.clone(); + let Some(map) = args.as_object_mut() else { + return Ok(args); + }; + let Some(named) = map.remove(IN_WINDOW) else { + return Ok(args); + }; + let raw = named.as_str().ok_or("in_window is not text")?; + let selector = Selector::parse(raw).map_err(|e| format!("in_window: {e}"))?; + let window = window_id_for(&selector).map_err(|e| format!("in_window {raw}: {e}"))?; + map.insert("window".to_string(), Value::String(window)); + Ok(args) +} + +/// The `element_action invoke` that does what the left click of `step` would, +/// when the step is a click on a link or a button found by selector. +pub(super) fn invoke_for(step: &Step) -> Option { + if step.tool != "click_element" { + return None; + } + let left = step + .args + .get("button") + .and_then(Value::as_str) + .is_none_or(|b| b == "left"); + let raw = step.args.get("selector")?.as_str()?; + let role = Selector::parse(raw).ok()?.role?; + (left && matches!(role.word(), "button" | "link")) + .then(|| json!({"selector": raw, "action": "invoke"})) +} + +/// The phrase argument: required, text, not empty, not absurdly long. +fn phrase_arg(args: &Value) -> Result { + let phrase = args + .get("phrase") + .and_then(Value::as_str) + .map(str::trim) + .filter(|p| !p.is_empty()) + .ok_or("phrase is required: the words that ask for this routine")?; + if phrase.chars().count() > MAX_PHRASE_CHARS { + return Err(format!("phrase is over {MAX_PHRASE_CHARS} characters")); + } + Ok(phrase.to_string()) +} + +/// The records `ids` names, oldest first. +fn records_of(source: &dyn StepSource, ids: &Value) -> Result, String> { + let ids: Vec<&str> = ids + .as_array() + .ok_or("records must be a list of Watchman record ids")? + .iter() + .map(|id| id.as_str().ok_or("every record id must be text")) + .collect::>()?; + if ids.is_empty() || ids.len() > MAX_RECORDS { + return Err(format!("give between 1 and {MAX_RECORDS} record ids")); + } + let mut records = ids + .iter() + .map(|id| source.record(id)) + .collect::, _>>()?; + // A task spans several turns, one record each; they run in the order they + // happened, not the order they were named. Stable, so equal times keep it. + records.sort_by_key(|r| r.started_at_ms); + Ok(records) +} + +/// An explicit step list: `[{"tool": ..., "args": {...}}, ...]`. +fn steps_of(steps: &Value) -> Result, String> { + let list = steps + .as_array() + .ok_or("steps must be a list of {tool, args}")?; + list.iter() + .enumerate() + .map(|(i, s)| { + serde_json::from_value::(s.clone()) + .map_err(|e| format!("step {} is not {{tool, args}}: {e}", i + 1)) + }) + .collect() +} + +/// A routine as the agent is shown it in a list. +fn describe(routine: &Routine, now: i64) -> Value { + let mut shown = json!({ + "phrase": routine.phrase, + "step_count": routine.steps.len(), + // Everything each step types and every selector, in full: this is what + // the person reads to see what the routine will do before it runs. + "steps": routine.steps.iter().map(Step::review).collect::>(), + "learned": routine.date, + "last_used": crate::routine::day_of(routine.last_used_ms), + "taught": routine.taught, + }); + let needs = params::names(&routine.steps); + if !needs.is_empty() { + // What the person has bound each to (the variable's NAME), and what is + // still to be bound: `routine_run` refuses until none is. + let unbound: Vec<&String> = needs + .iter() + .filter(|n| !routine.bindings.contains_key(*n)) + .collect(); + let stale = + !routine.bindings.is_empty() && routine.bound_steps != params::digest(&routine.steps); + shown["params_bound"] = json!(routine.bindings); + shown["params_unbound"] = json!(unbound); + if stale { + shown["params_binding_stale"] = + json!("the steps are not the ones the parameters were bound for: bind again"); + } + shown["params_needed"] = json!(needs); + } + if let Some(days) = routine.days_left(now) { + shown["expires_in_days"] = json!(days); + } + shown +} + +/// What `routine_save` answers. +fn saved_reply( + phrase: &str, + cleaned: &Cleaned, + saved: &crate::routine::store::Saved, + file: &std::path::Path, +) -> String { + let mut reply = json!({ + "saved": phrase, + "replaced_an_older_routine": saved.replaced, + "taught": saved.taught, + // Every step with the full text it types: read it, it is what will run. + "steps": cleaned.steps.iter().map(Step::review).collect::>(), + "looking_steps_dropped": cleaned.dropped, + "file": file.display().to_string(), + }); + if saved.unkept { + reply["kept_flag_dropped"] = json!( + "the routine this replaced had been kept by the person, and these steps are not \ + the ones they kept, so it is NOT kept now and will expire after 30 days unused. \ + Only the person can keep it again, after reading it: `incu routine keep`" + ); + } + if saved.unbound { + reply["bindings_dropped"] = json!( + "the routine this replaced had parameters the person had bound, and these steps \ + are not the ones they bound them for, so every binding is dropped; the person \ + binds again with `incu routine bind`" + ); + } + if !cleaned.parameters.is_empty() { + reply["parameters"] = json!(cleaned + .parameters + .iter() + .map(|p| json!({ + "name": p.name, + "step": p.step, + "why": match p.why { + Why::SecureField => "typed into a password or other secure field", + Why::Redacted => "the recording kept only a placeholder for the text", + Why::NotKnownPlain => { + "not typed into a control the step itself names and that is known \ + to be a plain text box (focus may be anywhere, a sign-in dialog or \ + a prompt included), so it may be a password" + } + }, + })) + .collect::>()); + reply["parameters_note"] = json!( + "those steps are NOT stored with their text, and routine_run refuses to start \ + until the PERSON has bound each one to an environment variable of incu's own: \ + `incu routine bind \"\" INCU_PARAM_`, with that variable \ + set where incu is launched. routine_run takes no params, so the value never \ + passes through you or your transcript. It is used in memory for that run only \ + and is never written to the routines file, the journal or what the hooks are \ + shown. Tell the person what to bind" + ); + } + if cleaned.inferred_window > 0 { + reply["keyboard_steps_given_the_window_before_them"] = json!(cleaned.inferred_window); + reply["window_note"] = json!( + "typing and key steps name no window in a recording, so each was given the window \ + of the step before it (see `in_window` in the steps); a replay refuses to type \ + unless that window has the focus" + ); + } + if !cleaned.skipped.is_empty() { + reply["left_out_not_desktop_tools"] = json!(cleaned.skipped); + reply["warning"] = json!( + "those calls are not in the routine; if the task depended on them, a replay \ + will not do them" + ); + } + pretty(&reply) +} + +fn pretty(value: &Value) -> String { + serde_json::to_string_pretty(value).unwrap_or_default() +} diff --git a/crates/desktop/src/mcp/tools.rs b/crates/desktop/src/mcp/tools.rs index d8fd5c4..7f1f484 100644 --- a/crates/desktop/src/mcp/tools.rs +++ b/crates/desktop/src/mcp/tools.rs @@ -48,7 +48,7 @@ impl Server { pub(super) fn run_and_look(&mut self, name: &str, args: &Value) -> Result, String> { self.run_tool(name, args).map(|mut content| { let wanted = args.get("screenshot_after").and_then(Value::as_bool) == Some(true); - if wanted && ACTING.contains(&name) { + if wanted && looks_after(name) { content.extend(after_shot(args)); } content @@ -86,11 +86,45 @@ impl Server { }; super::wait_tool::wait_for(args, &stopped) } + "watch" => { + self.journal_note("watch", &Self::describe("watch", args)); + // The same stop, asked as often, for a wait of hours. Nothing + // else ends it early: end of input does not (a watch that was + // the last request sent is still wanted), so an orphaned watch + // lasts until its deadline or `incu stop`. + let stopped = || { + self.catch_stop_file(); + self.safety.is_stopped() + }; + super::wait_tool::watch(args, &stopped) + } + "routine_save" | "routine_run" | "routine_forget" | "routine_list" + | "routine_match" => self.routine_tool(name, args), other => Err(format!("no tool named '{other}'")), } } } +/// Whether a call to `name` may end with the screenshot `screenshot_after` asks +/// for: an [`ACTING`] tool that does something on the screen. Saving or +/// forgetting a routine changes a file, not the screen, so there is nothing to +/// look at afterwards. Also read by `definitions`, so the schema offers the +/// argument exactly where it is honoured. +pub(super) fn looks_after(name: &str) -> bool { + ACTING.contains(&name) && !matches!(name, "routine_save" | "routine_forget") +} + +/// The text of a tool's answer: its text blocks, joined by newlines. An image +/// block has no text and is left out. +pub(super) fn text_of(content: &[Value]) -> String { + content + .iter() + .filter(|block| block.get("type").and_then(Value::as_str) == Some("text")) + .filter_map(|block| block.get("text").and_then(Value::as_str)) + .collect::>() + .join("\n") +} + /// `screen_info`: every display. Called by [`Server::run_tool`]. fn screen_info() -> Result, String> { let info = crate::screen_info().map_err(|e| e.to_string())?; @@ -585,7 +619,10 @@ pub(super) fn element_target(args: &Value) -> Result { /// compared with a window named by id: a mismatch error describes the window, /// and for a blocked one that is its title, and a match and a mismatch would /// be told apart. -fn resolve_selector(window: Option<&str>, selector: &Selector) -> Result { +pub(super) fn resolve_selector( + window: Option<&str>, + selector: &Selector, +) -> Result { let info = uia::window_for(window, selector, &refuse_blocked).map_err(|e| e.to_string())?; uia::resolve_in(info, selector).map_err(|e| e.to_string()) } @@ -609,6 +646,18 @@ pub(super) fn disabled_error(target: &Target, element: &uia::Element) -> String } } +/// The id of the one window a window-only `selector` names, now: how a replayed +/// routine's keyboard step (`in_window`) becomes the `window` the tool checks +/// the focus against. Refused, as everything else is, when that window is a +/// blocked application's (the guard is handed to `uia::window_for`, as for every +/// other selector). +/// +/// Called by `routine_tool`'s replay. +pub(super) fn window_id_for(selector: &Selector) -> Result { + let info = uia::window_for(None, selector, &refuse_blocked).map_err(|e| e.to_string())?; + Ok(info.id) +} + /// `done`, with what only a selector adds: its `eN` numbers this call's own /// walk and not the agent's last `ui_tree`, and a cut tree cannot rule out a /// second match. @@ -664,15 +713,29 @@ pub(super) fn guard_capture(window: Option<&str>) -> Result, Ok(Some(info)) } -/// A screenshot of one window, or of the primary display — refused when a -/// blocked application's window is in what would be captured. -fn shoot(window: Option<&str>, options: CaptureOptions) -> Result, String> { +/// The capture of one window, or of the primary display, under +/// [`guard_capture`]'s rule: refused when a blocked application's window is in +/// what would be captured. The window it was of, when one was named. +/// +/// Called by [`shoot`] and by a routine replay's before and after pictures, so +/// both are held to the one guard. +pub(super) fn capture( + window: Option<&str>, + options: CaptureOptions, +) -> Result<(capture::Screenshot, Option), String> { let of = guard_capture(window)?; let shot = match &of { Some(info) => capture::capture_region(info.rect, options), None => capture::capture_primary(options), } .map_err(|e| e.to_string())?; + Ok((shot, of)) +} + +/// A screenshot of one window, or of the primary display — refused when a +/// blocked application's window is in what would be captured. +fn shoot(window: Option<&str>, options: CaptureOptions) -> Result, String> { + let (shot, of) = capture(window, options)?; let (width, height) = shot.model_size(); let t = shot.transform; Ok(vec![ diff --git a/crates/desktop/src/mcp/transport.rs b/crates/desktop/src/mcp/transport.rs index 368089c..ed4d241 100644 --- a/crates/desktop/src/mcp/transport.rs +++ b/crates/desktop/src/mcp/transport.rs @@ -146,16 +146,40 @@ pub(super) fn error_response(id: Value, code: i64, message: &str) -> String { /// When the agent process dies its end of the pipe closes, `read_line` returns /// zero bytes, and no timer can fake that. So the loop stops the desktop on EOF /// rather than relying on a heartbeat that could keep beating into the void. -pub fn run_stdio(mut server: Server) -> std::io::Result<()> { - let stdin = std::io::stdin(); - let mut stdout = std::io::stdout(); - let mut line = String::new(); +pub fn run_stdio(server: Server) -> std::io::Result<()> { + run_lines(server, std::io::stdin(), std::io::stdout()) +} +/// [`run_stdio`] over any reader and writer. +/// +/// # EOF does not end a call that is running +/// +/// The server answers one line at a time, in order, and reads the next only when +/// the last is answered. So the end of input is seen only after every line before +/// it has been answered, and **a `watch` or a `routine_run` that was the last +/// line sent completes normally**: a client that pipelined its requests and closed +/// its end is waiting for those answers, and EOF is not evidence that it has gone. +/// An earlier version read stdin on a thread and ended a waiting call at EOF; that +/// cut off the last pipelined request, and it is gone. A `watch` or a replay ends +/// by its own deadline, the stop file (`incu stop`), or an error writing its +/// answer to stdout, and an orphaned `watch` lasts until one of them. +/// +/// # Errors +/// +/// Reading or writing failed. +pub(crate) fn run_lines(mut server: Server, input: R, mut out: W) -> std::io::Result<()> +where + R: std::io::Read, + W: Write, +{ + let mut reader = std::io::BufReader::new(input); + let mut line = String::new(); loop { line.clear(); - match stdin.lock().read_line(&mut line) { - // EOF: the client is gone. Halt the desktop before returning, so a - // slow in-flight action cannot outlive the thing that asked for it. + match reader.read_line(&mut line) { + // EOF: every line before it is answered. Halt the desktop before + // returning, so a slow in-flight action cannot outlive the thing that + // asked for it. Ok(0) => { server.safety().stop(); return Ok(()); @@ -165,8 +189,8 @@ pub fn run_stdio(mut server: Server) -> std::io::Result<()> { server.safety().heartbeat(); if let Some(reply) = server.handle_line(&line) { - writeln!(stdout, "{reply}")?; - stdout.flush()?; + writeln!(out, "{reply}")?; + out.flush()?; } } Err(e) => { diff --git a/crates/desktop/src/mcp/wait_tool.rs b/crates/desktop/src/mcp/wait_tool.rs index a45acc0..0714122 100644 --- a/crates/desktop/src/mcp/wait_tool.rs +++ b/crates/desktop/src/mcp/wait_tool.rs @@ -433,12 +433,85 @@ fn condition_of(args: &Value) -> Result { pub(super) fn wait_for(args: &Value, stopped: &dyn Fn() -> bool) -> Result, String> { let condition = condition_of(args)?; let budget = Budget::new(whole(args, "timeout_ms")?, whole(args, "poll_ms")?); - let mut env = Live::new(&budget, stopped); - let outcome = wait::run(&mut env, &condition, &budget, stopped)?; + look_until(&condition, &budget, stopped, None) +} + +/// `watch`: the same look, for hours, with a note for the operator's hook. +/// +/// **Why a tool of its own and not an option of `wait_for`.** `wait_for` is held +/// to 60 seconds, polled every 50 ms to 2 s, and that bound is a safety property +/// its tests pin: a call that is supposed to return in moments. A watch is the +/// opposite promise (it holds the call open until something happens, up to a +/// day, and looks gently), so it takes its own limits ([`Budget::watch`]) and +/// its own description, which says plainly that it blocks. The look itself is +/// the same code ([`look_until`]). +/// +/// `message` is carried into the reply and so into the operator's post-tool +/// hook, which is how a watch notifies: see `Server::tell_post_hooks`. +/// +/// A watch of the SCREEN (`screen_changes`) looks every five seconds unless told +/// otherwise ([`Budget::watch_screen`]), not every second: each look is a capture +/// and a hash of the whole screen. +/// +/// **A watch holds the server for its whole duration.** The server answers one +/// call at a time on one thread, so while a watch runs no other call is served; +/// it ends when the condition holds, its time runs out, or the operator stops +/// the desktop (`incu stop`). **Nothing else ends it**: end of input on stdin does +/// not (a watch that was the last request a client sent is still wanted), so a +/// watch whose client has gone lasts until its deadline (1 hour unless asked for +/// more, 24 at most) or `incu stop`. +/// +/// Called by `Server::run_tool`. +pub(super) fn watch(args: &Value, stopped: &dyn Fn() -> bool) -> Result, String> { + let condition = condition_of(args)?; + let budget = watch_budget( + &condition, + whole(args, "timeout_ms")?, + whole(args, "poll_ms")?, + ); + let message = message_arg(args)?; + look_until(&condition, &budget, stopped, message) +} + +/// How long a `watch` of `condition` may run and how often it looks: a look at +/// the screen is the slow one, so it is the one given the slower default. +fn watch_budget(condition: &Condition, timeout: Option, poll: Option) -> Budget { + match condition { + Condition::Screen { .. } => Budget::watch_screen(timeout, poll), + _ => Budget::watch(timeout, poll), + } +} + +/// The most of a `watch` message that is kept, in characters. +const MAX_MESSAGE_CHARS: usize = 200; + +/// What the watch is to say when it fires, if the call gave one. +fn message_arg(args: &Value) -> Result, String> { + match args.get("message") { + None | Some(Value::Null) => Ok(None), + Some(Value::String(m)) if m.chars().count() <= MAX_MESSAGE_CHARS => Ok(Some(m.clone())), + Some(Value::String(_)) => Err(format!("message is over {MAX_MESSAGE_CHARS} characters")), + Some(_) => Err("message must be text".to_string()), + } +} + +/// Look for `condition` on the live desktop within `budget` and render the +/// reply. A stop is a refusal, not a reply. +fn look_until( + condition: &Condition, + budget: &Budget, + stopped: &dyn Fn() -> bool, + message: Option, +) -> Result, String> { + let mut env = Live::new(budget, stopped); + let outcome = wait::run(&mut env, condition, budget, stopped)?; if matches!(outcome, Outcome::Stopped { .. }) { return Err(crate::safety::Refusal::Stopped.message()); } - let reply = outcome.to_json(&condition, &budget); + let mut reply = outcome.to_json(condition, budget); + if let Some(message) = message { + reply["message"] = serde_json::json!(message); + } Ok(vec![text(format!( "{}\n{}", serde_json::to_string_pretty(&reply).unwrap_or_default(), @@ -451,6 +524,30 @@ mod tests { use super::*; use serde_json::json; + /// A watch of the screen looks every five seconds, not every second, and + /// anything else keeps its second; a poll the caller gives is still theirs. + #[test] + fn a_watch_of_the_screen_polls_every_five_seconds_by_default() { + let screen = condition_of(&json!({"condition": "screen_changes"})).unwrap(); + let window = + condition_of(&json!({"condition": "window_gone", "selector": "app=zz;window=none"})) + .unwrap(); + assert_eq!(watch_budget(&screen, None, None).poll_ms, 5_000); + assert_eq!(watch_budget(&window, None, None).poll_ms, 1_000); + assert_eq!(watch_budget(&screen, None, Some(2_000)).poll_ms, 2_000); + // The same bounds otherwise: an hour by default, a day at most. + assert_eq!( + watch_budget(&screen, None, None).timeout_ms, + wait::DEFAULT_WATCH_MS + ); + assert_eq!( + watch_budget(&screen, Some(u64::MAX), None).timeout_ms, + wait::MAX_WATCH_MS + ); + // And wait_for is not touched: still 250 ms. + assert_eq!(Budget::new(None, None).poll_ms, wait::DEFAULT_POLL_MS); + } + /// **A look that blocks cannot outlive its limit.** The live hang was a /// call that did not return for minutes; here the work sleeps far longer /// than the limit and `bounded` must hand back long before it returns. diff --git a/crates/desktop/src/permission.rs b/crates/desktop/src/permission.rs index b58e24e..0e7b84b 100644 --- a/crates/desktop/src/permission.rs +++ b/crates/desktop/src/permission.rs @@ -12,14 +12,14 @@ //! //! Until the desktop became a crate of its own this was incode's //! `PermissionPolicy::for_desktop(PermissionMode)`. The crate cannot depend on -//! incode, so the tier is restated here as the two rungs the 17 tools actually +//! incode, so the tier is restated here as the two rungs the 23 tools actually //! use, and [`Tier::from_mode_word`] reads incode's own mode words so the argv //! incode has always passed (`-p read-only`) keeps meaning the same thing: //! //! | mode word | tier | reaches | //! |---|---|---| -//! | `read-only` | [`Tier::Observe`] | the 6 observing tools | -//! | `workspace-write` (the default), `danger-full-access`, `allow` | [`Tier::Act`] | all 17 | +//! | `read-only` | [`Tier::Observe`] | the 9 observing tools | +//! | `workspace-write` (the default), `danger-full-access`, `allow` | [`Tier::Act`] | all 23 | //! | `prompt` | [`Tier::Deny`] | nothing | //! //! `prompt` denies everything, and that is correct rather than unfortunate: @@ -47,6 +47,12 @@ pub const OBSERVING: &[&str] = &[ // Waits and looks: it polls the tree, the window list and the screen and // changes nothing, so it is held to the same tier as the tools it is made of. "wait_for", + // The same look, held for hours, then the operator's hook is told: it + // looks and changes nothing, so it is held to the tier of what it is made of. + "watch", + // They read the routines file and touch neither the screen nor the file. + "routine_list", + "routine_match", ]; /// The tools that change the state of a machine the operator is signed into, and @@ -63,6 +69,14 @@ pub const ACTING: &[&str] = &[ "drag", "move_mouse", "click", + // `routine_run` replays steps that move the mouse and the keyboard, each + // through the same gates (and so the same tier) as a live call. + // `routine_save` and `routine_forget` write the routines file, and that file + // is what a later `routine_run` does to the machine, so changing it is + // acting, not looking. + "routine_save", + "routine_run", + "routine_forget", ]; /// How much of the desktop a server will hand over. @@ -160,14 +174,14 @@ mod tests { /// The tier table must be the server's tool list, or a tool is either /// ungoverned or unreachable. Checked against what `tools/list` offers, so a - /// eighteenth tool cannot be added without deciding its tier. + /// twenty-fourth tool cannot be added without deciding its tier. #[test] - fn every_listed_tool_has_a_tier_and_there_are_seventeen() { + fn every_listed_tool_has_a_tier_and_there_are_twenty_three() { let listed: Vec = crate::mcp::tool_definitions() .iter() .map(|t| t["name"].as_str().unwrap_or_default().to_string()) .collect(); - assert_eq!(listed.len(), 17, "{listed:?}"); + assert_eq!(listed.len(), 23, "{listed:?}"); for name in &listed { assert!( OBSERVING.contains(&name.as_str()) ^ ACTING.contains(&name.as_str()), diff --git a/crates/desktop/src/routine/fixtures/record_failed_mcp_call.json b/crates/desktop/src/routine/fixtures/record_failed_mcp_call.json new file mode 100644 index 0000000..cff1007 --- /dev/null +++ b/crates/desktop/src/routine/fixtures/record_failed_mcp_call.json @@ -0,0 +1 @@ +{"linked_memory":["00000000-0000-4000-8000-000000000002"],"producer_model":"claude-opus-5","record_id":"w-cca0d1763ffa35991f50a9b9","recorder":"watchman-capture/0.1.0","space_id":"example-project","spans":[{"attrs":{},"duration_ms":0,"error":"\"unknown scope \\\"proj-example\\\" — known scopes: general (unfiled staging)\"","input":{"query":"example query about one topic, and a second topic","scope":"proj-example","top_k":25},"kind":"ToolCall","name":"mcp__example_memory__recall_index","output":null,"parent_span_id":null,"seq":1,"span_id":1,"started_at_ms":1789511442421}],"started_at_ms":1789511442421,"task":"main","task_input":null,"task_output":null,"user_id":"user"} diff --git a/crates/desktop/src/routine/fixtures/record_shell_call.json b/crates/desktop/src/routine/fixtures/record_shell_call.json new file mode 100644 index 0000000..885c30a --- /dev/null +++ b/crates/desktop/src/routine/fixtures/record_shell_call.json @@ -0,0 +1 @@ +{"linked_memory":["00000000-0000-4000-8000-000000000001"],"producer_model":"claude-opus-5","record_id":"w-078c536885c8d141ef6c1ec7","recorder":"watchman-capture/0.1.0","space_id":"example-project","spans":[{"attrs":{},"duration_ms":0,"error":null,"input":{"command":"grep -c \"EXAMPLE_FLAG\" docker-compose.yml | sed 's/^/occurrences: /'"},"kind":"ToolCall","name":"Bash","output":"occurrences: 2","parent_span_id":null,"seq":1,"span_id":1,"started_at_ms":1789518445212}],"started_at_ms":1789518445212,"task":"main","task_input":null,"task_output":null,"user_id":"user"} diff --git a/crates/desktop/src/routine/fixtures/record_structured_output.json b/crates/desktop/src/routine/fixtures/record_structured_output.json new file mode 100644 index 0000000..897fd94 --- /dev/null +++ b/crates/desktop/src/routine/fixtures/record_structured_output.json @@ -0,0 +1 @@ +{"linked_memory":["00000000-0000-4000-8000-000000000003"],"producer_model":"claude-opus-5","record_id":"w-0503383fb5fdf726a39b6f01","recorder":"watchman-capture/0.1.0","space_id":"example-project","spans":[{"attrs":{},"duration_ms":0,"error":null,"input":{"max_results":2,"query":"select:mcp__example_memory__remember"},"kind":"ToolCall","name":"ToolSearch","output":[{"tool_name":"mcp__example_memory__remember","type":"tool_reference"}],"parent_span_id":null,"seq":1,"span_id":1,"started_at_ms":1789513566679}],"started_at_ms":1789513566679,"task":"main","task_input":null,"task_output":null,"user_id":"user"} diff --git a/crates/desktop/src/routine/learn.rs b/crates/desktop/src/routine/learn.rs new file mode 100644 index 0000000..d3ad145 --- /dev/null +++ b/crates/desktop/src/routine/learn.rs @@ -0,0 +1,1350 @@ +//! What a run must be like to be kept, and how it is cleaned. +//! +//! The rules are Atticus's learned-shortcut rules (`atticus.py` `learn`, +//! `hands_replayable`), applied to a list of recorded tool calls: +//! +//! 1. **The phrase must not point at the screen.** "this" or "that" means +//! whatever was on screen when it was said ([`super::words::is_deictic`]). +//! 2. **The run must not have failed.** ANY call that errored (a shell command, +//! a refused `ui_tree`, a file that was not there: not only a desktop call), +//! a record whose own outcome says it failed ([`check_record`]), or a +//! `wait_for` that timed out or was stopped, is a run that did not do what it +//! set out to; replaying it would repeat the failure. +//! 3. **Every kept step must be replayable on another day.** The only way to +//! name a control that survives is a selector, so a step that uses an `eN` +//! id (valid for one `ui_tree` only), a pixel coordinate (valid for one +//! layout), or a window handle (valid for one window) is refused, and so is +//! the whole routine: dropping that step would silently skip part of the +//! task. `click`, `move_mouse`, `drag` and `focus_window` have no other form, +//! so they are never replayable. +//! A step that acts must also say WHERE: its selector carries an `app` or +//! `window` part, and a keyboard step (`type_text`, `paste_text`, `key`) +//! carries [`IN_WINDOW`]. Without one the step would act in whatever window +//! is in front when it runs, which is how a live test typed into the wrong +//! program. When the recorded keyboard step has none, the window of the step +//! before it is used and the reply says so; the replay still refuses unless +//! that window has the focus. `scroll` is never replayable: it takes no +//! selector and scrolls under the pointer. +//! 4. **Reading steps are dropped.** `ui_tree`, `screenshot`, `list_windows`, +//! `screen_info` and `cursor_position` are how the agent found its way; the +//! replay needs only the actions. `wait_for` is kept, because it is what makes +//! the replay wait for the screen instead of guessing how long it takes. +//! +//! Calls that are not desktop tools at all (a shell command, a file edit) are +//! left out and REPORTED in [`Cleaned::skipped`], so the agent knows that a +//! step it may have relied on is not in the routine. +//! +//! Last, the text of a step that types into a secure field, or that a recording +//! kept only as a placeholder, is taken out and made a parameter +//! ([`super::params`]): a routine never holds a secret. +//! +//! The same per-step check, [`check_step`], runs again before a replay, so a +//! hand-edited routines file cannot smuggle in a step the learner would have +//! refused. + +use std::fmt; + +use serde_json::Value; + +use super::params::{names, parameterise, Made, Probe, MAX_PARAMS}; +use super::source::{Call, Record}; +use super::words::{is_deictic, word_set}; +use super::{Step, MAX_STEPS, MAX_STEP_BYTES}; +use crate::permission::{ACTING, OBSERVING}; +use crate::selector::Selector; + +/// The desktop tools that only look and take part in no replay. +pub const READING: &[&str] = &[ + "screen_info", + "cursor_position", + "screenshot", + "ui_tree", + "list_windows", +]; + +/// The tools a step may be, with the argument that names the control. +const REPLAYABLE: &[&str] = &[ + "wait_for", + "click_element", + "set_value", + "element_action", + "type_text", + "paste_text", + "key", +]; + +/// The tools whose control is named by a selector. +const NEEDS_SELECTOR: &[&str] = &["click_element", "set_value", "element_action"]; + +/// The tools that send keystrokes to whichever window has focus. They take no +/// selector, so a step names its window with [`IN_WINDOW`]. +pub const KEYBOARD: &[&str] = &["type_text", "paste_text", "key"]; + +/// The step argument that names the window a keyboard step is for: a selector +/// with only an `app` and/or `window` part. It is the routine's own key, not the +/// tool's: the replay turns it into the tool's `window` (a handle, looked up +/// now), so the tool refuses unless that window really has the focus. +pub const IN_WINDOW: &str = "in_window"; + +/// Why a run or a step was refused. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Refusal { + /// The phrase says "this" or "that". + Deictic, + /// The phrase has no words of its own (all filler). + NoWords, + /// A record's own outcome says the run failed, whatever its calls say. + RecordFailed { + /// The record. + id: String, + /// What it says. + why: String, + }, + /// A call in the run failed. + RunFailed { + /// Its 1-based position in the calls given. + step: usize, + /// The tool. + tool: String, + /// What went wrong. + why: String, + }, + /// A step cannot be replayed on another day. + NotReplayable { + /// Its 1-based position. + step: usize, + /// The tool. + tool: String, + /// Why not, and what to do instead. + why: String, + }, + /// Nothing is left once the reading steps are dropped. + NoSteps, + /// More steps than a routine may hold. + TooLong(usize), + /// More parameters than a routine may take. + TooManyParams(usize), +} + +impl fmt::Display for Refusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Refusal::Deictic => write!( + f, + "not learned: the phrase says \"this\" or \"that\", which means whatever was \ + on screen when it was said. Give the routine a name that stands on its own" + ), + Refusal::NoWords => write!( + f, + "not learned: the phrase has no words of its own once filler words are \ + ignored" + ), + Refusal::RecordFailed { id, why } => write!( + f, + "not learned: record {id} says its run failed ({why}). Only a run that worked \ + is kept; do the task again, cleanly, and save that" + ), + Refusal::RunFailed { step, tool, why } => write!( + f, + "not learned: the run failed at call {step} ({tool}): {why}. Only a run \ + that worked is kept; do the task again, cleanly, and save that" + ), + Refusal::NotReplayable { step, tool, why } => write!( + f, + "not learned: call {step} ({tool}) {why}. Only steps that name their control \ + by selector can be replayed" + ), + Refusal::NoSteps => write!( + f, + "not learned: no desktop action is left once looking-only calls \ + (ui_tree, screenshot, list_windows, ...) are dropped" + ), + Refusal::TooManyParams(n) => write!( + f, + "not learned: {n} parameters (secrets and redacted texts the routine would \ + have to be given) is more than a routine may take ({MAX_PARAMS}); \ + save a shorter task" + ), + Refusal::TooLong(n) => write!( + f, + "not learned: {n} steps is more than a routine may hold ({MAX_STEPS})" + ), + } + } +} + +impl std::error::Error for Refusal {} + +/// What a run comes to once it is accepted. +#[derive(Debug, Clone, PartialEq)] +pub struct Cleaned { + /// The steps to keep, in order. + pub steps: Vec, + /// How many looking-only calls were dropped. + pub dropped: usize, + /// The names of calls that are not desktop tools and were left out. + pub skipped: Vec, + /// Keyboard steps that named no window and were given the window of the step + /// before them. + pub inferred_window: usize, + /// The steps whose text was taken out and made a parameter the run must be + /// given ([`super::params`]). + pub parameters: Vec, +} + +/// The names incu's tools are recorded under as `mcp____`: the key +/// incode's client configs register it by (`desktop`), the name `incu` itself +/// answers to, and its protocol name, with the underscore a client makes of the +/// hyphen. Nothing else is taken for this server. +pub const OWN_SERVERS: &[&str] = &["desktop", "incu", "incode-desktop", "incode_desktop"]; + +/// The desktop tool a recorded name is, if it is one. +/// +/// A recorded name is the plain tool name (incode's own tools), or an MCP +/// name `mcp____`. For the second form the SERVER must be one of +/// incu's own names ([`OWN_SERVERS`]), exactly: other MCP servers have tools +/// called `key` or `scroll` too, and a server that merely has "desktop" in its +/// name (a remote-desktop tool, a screen sharer) is not this one. A routine must +/// never replay someone else's tool. +#[must_use] +pub fn desktop_tool(recorded: &str) -> Option<&str> { + let known = |name: &str| OBSERVING.contains(&name) || ACTING.contains(&name); + match recorded.strip_prefix("mcp__") { + None => known(recorded).then_some(recorded), + Some(rest) => { + let (server, tool) = rest.split_once("__")?; + (OWN_SERVERS.contains(&server) && known(tool)).then_some(tool) + } + } +} + +/// Whether `phrase` may name a routine. +/// +/// # Errors +/// +/// It points at the screen, or has no words of its own. +pub fn check_phrase(phrase: &str) -> Result<(), Refusal> { + if is_deictic(phrase) { + return Err(Refusal::Deictic); + } + if word_set(phrase).is_empty() { + return Err(Refusal::NoWords); + } + Ok(()) +} + +/// Whether `step` may be replayed, as step `n` (1-based, for the message). +/// +/// # Errors +/// +/// It is not a replayable tool, or it names a control or window in a way that +/// does not survive to another day, or its arguments are not usable. +pub fn check_step(step: &Step, n: usize) -> Result<(), Refusal> { + let refuse = |why: String| Refusal::NotReplayable { + step: n, + tool: step.tool.clone(), + why, + }; + let tool = step.tool.as_str(); + if !REPLAYABLE.contains(&tool) { + return Err(refuse(match tool { + "click" | "move_mouse" | "drag" => { + "uses raw coordinates, which are right for one layout only".to_string() + } + "focus_window" => { + "names a window by its handle, which is right for one window only".to_string() + } + "scroll" => "scrolls under the pointer, in no window of its own (it takes no \ + selector); use element_action scroll_into_view on a control, or a \ + key such as Page Down" + .to_string(), + t if t.starts_with("routine_") => { + "is a routine tool; a routine cannot run another".to_string() + } + "watch" => { + "waits for hours, which a replay must not; use wait_for with a limit".to_string() + } + other => format!("is not a tool a routine can replay ({other})"), + })); + } + let Some(args) = step.args.as_object() else { + return Err(refuse("has arguments that are not an object".to_string())); + }; + if step.args.to_string().len() > MAX_STEP_BYTES { + return Err(refuse(format!("has arguments over {MAX_STEP_BYTES} bytes"))); + } + let given = |key: &str| args.get(key).is_some_and(|v| !v.is_null()); + for key in ["id", "from_id", "to_id"] { + if given(key) { + return Err(refuse(format!( + "uses an element number (`{key}`, an eN id), which is valid for one ui_tree only" + ))); + } + } + for key in ["x", "y", "from_x", "from_y", "to_x", "to_y"] { + if given(key) { + return Err(refuse(format!("uses a raw coordinate (`{key}`)"))); + } + } + if given("window") { + return Err(refuse( + "names a window by its handle (`window`), which is valid for one window only; \ + name the window in the selector (`app=...;window=...`)" + .to_string(), + )); + } + if tool == "wait_for" && given("baseline") { + return Err(refuse( + "compares the screen with a `baseline` from one earlier moment".to_string(), + )); + } + let looks_at_screen = tool == "wait_for" + && args.get("condition").and_then(Value::as_str) == Some("screen_changes"); + // A typing step MAY name the control it types into (and only then is its + // text kept literally: see `params`); when it does, the replay focuses that + // control first, so it must say which window it is in. + let names_its_field = matches!(tool, "type_text" | "paste_text") && given("selector"); + let needs_selector = NEEDS_SELECTOR.contains(&tool) + || (tool == "wait_for" && !looks_at_screen) + || names_its_field; + check_selector(args.get("selector"), needs_selector).map_err(refuse)?; + if names_its_field { + let named = args.get("selector").and_then(Value::as_str); + if named.is_some_and(|raw| Selector::parse(raw).is_ok_and(|s| !s.has_element_part())) { + return Err(refuse( + "names a window and no control in `selector`; name the field it types into \ + (role=edit;name=...) or leave the selector out" + .to_string(), + )); + } + } + if tool == "key" { + check_key(args.get("keys")).map_err(refuse)?; + } + if KEYBOARD.contains(&tool) { + check_in_window(args.get(IN_WINDOW)).map_err(refuse)?; + } + Ok(()) +} + +/// The keys that act on the keyboard without typing a character: named keys. +const NAMED_KEYS: &[&str] = &[ + "enter", + "return", + "tab", + "escape", + "esc", + "up", + "down", + "left", + "right", + "arrowup", + "arrowdown", + "arrowleft", + "arrowright", + "up_arrow", + "down_arrow", + "left_arrow", + "right_arrow", + "home", + "end", + "pageup", + "pagedown", + "page_up", + "page_down", + "pgup", + "pgdn", + "backspace", + "delete", + "del", + "space", +]; + +/// The keys that make a chord a command and not typing: with one of these held, +/// a letter is a shortcut (Ctrl+S), not a character put in a field. +const MODIFIER_KEYS: &[&str] = &[ + "ctrl", "control", "alt", "option", "cmd", "command", "win", "windows", "super", "meta", +]; + +/// Whether `key` is an F-key (F1 to F24). +fn is_f_key(key: &str) -> bool { + key.strip_prefix('f') + .and_then(|n| n.parse::().ok()) + .is_some_and(|n| (1..=24).contains(&n)) +} + +/// A `key` step may hold only named keys and chords of them: Enter, Tab, Escape, +/// the arrows, F-keys, Home, End, Page Up and Down, Backspace, Delete, Space, and +/// any chord that has ctrl, alt, cmd, win or meta in it. A step that presses a +/// lone printable character (a letter, a digit, a symbol, with or without shift) +/// is TYPING, and a PIN typed as `{"keys": ["4"]}` presses would be stored as +/// they are, past the screen that every typed text goes through +/// ([`super::params`]); so it is refused. +fn check_key(keys: Option<&Value>) -> Result<(), String> { + let named: Vec = match keys { + Some(Value::Array(list)) => list + .iter() + .filter_map(Value::as_str) + .map(|k| k.trim().to_lowercase()) + .collect(), + Some(Value::String(chord)) => chord.split('+').map(|k| k.trim().to_lowercase()).collect(), + _ => Vec::new(), + }; + if named.is_empty() || named.iter().any(String::is_empty) { + return Err("has no keys to press".to_string()); + } + let command = named.iter().any(|k| MODIFIER_KEYS.contains(&k.as_str())); + let only_named = named + .iter() + .all(|k| k == "shift" || NAMED_KEYS.contains(&k.as_str()) || is_f_key(k)); + let some_named = named + .iter() + .any(|k| NAMED_KEYS.contains(&k.as_str()) || is_f_key(k)); + if command || (only_named && some_named) { + return Ok(()); + } + Err( + "presses a printable character as a key (a letter, digit or symbol, with or without \ + shift), which is typing and would be stored as it is; write a typing step \ + (`type_text`) on a named field instead. Only named keys (Enter, Tab, Escape, the \ + arrows, F-keys, Home, End, Page Up/Down, Backspace, Delete, Space) and chords with \ + ctrl, alt, cmd, win or meta can be a `key` step" + .to_string(), + ) +} + +/// The selector argument. When the tool needs one it must be text that parses +/// AND say which window it is in (`app` or `window`): a selector with only a +/// control part is resolved in whatever window is in front. +fn check_selector(selector: Option<&Value>, needed: bool) -> Result<(), String> { + match selector { + None | Some(Value::Null) if needed => { + Err("has no `selector` to find its control by".to_string()) + } + None | Some(Value::Null) => Ok(()), + Some(Value::String(raw)) => { + let parsed = Selector::parse(raw) + .map_err(|e| format!("has a selector that does not parse: {e}"))?; + if needed && !parsed.has_window_part() { + return Err( + "has a selector with no window part, so it would act in whatever window is \ + in front; add `app=...;window=...` (or `window~=...`) to it" + .to_string(), + ); + } + Ok(()) + } + Some(_) => Err("has a `selector` that is not text".to_string()), + } +} + +/// A keyboard step's `in_window`: a window-only selector that names a window. +fn check_in_window(value: Option<&Value>) -> Result<(), String> { + let Some(Value::String(raw)) = value.filter(|v| !v.is_null()) else { + return Err(format!( + "sends keys to whatever window has focus: it names no window (`{IN_WINDOW}`, e.g. \ + app=notepad;window~=Untitled)" + )); + }; + let parsed = Selector::parse(raw) + .map_err(|e| format!("has an `{IN_WINDOW}` that does not parse: {e}"))?; + parsed + .require_window_only() + .map_err(|e| format!("has an `{IN_WINDOW}` that is not a window: {e}"))?; + if parsed.has_window_part() { + Ok(()) + } else { + Err(format!("has an `{IN_WINDOW}` that names no app or window")) + } +} + +/// Why a `wait_for` that answered `answer` did not find what it waited for, or +/// `None` when it did. A timeout is a reply and not a tool error, so it is not +/// in a span's `error` and has to be read from the answer. +/// +/// The live answer is JSON followed by a note, which is parsed. A recorded one +/// may have been cut at 4 KiB, which is not valid JSON, so that case is read +/// from the text. +pub(super) fn wait_failure(answer: &str) -> Option { + let first = serde_json::Deserializer::from_str(answer.trim_start()) + .into_iter::() + .next() + .and_then(Result::ok); + if let Some(reply) = first { + return match reply.get("result").and_then(Value::as_str) { + Some("timeout") => Some(format!( + "the wait timed out after {} ms; last seen: {}", + reply["waited_ms"], + reply["last_seen"].as_str().unwrap_or("nothing recorded") + )), + Some("stopped") => Some("the wait was stopped".to_string()), + _ => None, + }; + } + let said = |result: &str| { + answer.contains(&format!("\"result\": \"{result}\"")) + || answer.contains(&format!("\"result\":\"{result}\"")) + }; + if said("timeout") { + Some("the wait timed out".to_string()) + } else if said("stopped") { + Some("the wait was stopped".to_string()) + } else { + None + } +} + +/// Clean a recorded run into the steps of a routine, or say why it is not +/// one. `phrase` is the routine's trigger. +/// +/// # Errors +/// +/// A [`Refusal`]: the phrase, a failed call, a step that cannot be replayed, an +/// empty result or too many steps. +pub fn learn_from_calls(phrase: &str, calls: &[Call]) -> Result { + // With nothing to ask the desktop, no control is known to be plain: every + // typed text becomes a parameter (see [`super::params::why_not_kept`]). + learn_with(phrase, calls, &|_| Probe::Unknown) +} + +/// As [`learn_from_calls`], with `probe` asked about the raw selector of a control +/// a typing step names: the desktop's own report, at save time, of whether it is +/// a password field. A text is kept only when a step names a control that +/// `probe` says is plain (see [`super::params`]). +/// +/// # Errors +/// +/// As [`learn_from_calls`]. +pub fn learn_with( + phrase: &str, + calls: &[Call], + probe: &dyn Fn(&str) -> Probe, +) -> Result { + check_phrase(phrase)?; + // ANY call that failed is a run that failed: a shell command that errored, a + // file that could not be read, a looking call that was refused. Not only the + // desktop calls: what went wrong there may be why the desktop calls that + // followed did what they did, and a routine made of them is not what the + // task needed. + for (index, call) in calls.iter().enumerate() { + if let Some(why) = &call.error { + return Err(Refusal::RunFailed { + step: index + 1, + tool: call.tool.clone(), + why: first_line(why), + }); + } + } + let mut steps = Vec::new(); + let mut dropped = 0; + let mut skipped = Vec::new(); + let mut inferred_window = 0; + // The window part of the last step that named one, for a keyboard step that + // names none of its own. + let mut last_window: Option = None; + for (index, call) in calls.iter().enumerate() { + let n = index + 1; + let Some(tool) = desktop_tool(&call.tool) else { + skipped.push(call.tool.clone()); + continue; + }; + if READING.contains(&tool) { + dropped += 1; + continue; + } + if let Some(why) = call + .output + .as_deref() + .filter(|_| tool == "wait_for") + .and_then(wait_failure) + { + return Err(Refusal::RunFailed { + step: n, + tool: tool.to_string(), + why, + }); + } + let mut step = Step::new(tool, call.args.clone()); + // The picture asked for after an action is for the agent that asked. + if let Some(map) = step.args.as_object_mut() { + map.remove("screenshot_after"); + if KEYBOARD.contains(&tool) && !map.contains_key(IN_WINDOW) { + if let Some(window) = &last_window { + map.insert(IN_WINDOW.to_string(), Value::String(window.clone())); + inferred_window += 1; + } + } + } + check_step(&step, n)?; + last_window = window_part_of(&step).or(last_window); + steps.push(step); + } + if steps.is_empty() { + return Err(Refusal::NoSteps); + } + if steps.len() > MAX_STEPS { + return Err(Refusal::TooLong(steps.len())); + } + // After every step is known replayable, and last: the text of a secure field + // or a redacted recording is taken out of the steps before they go anywhere. + let parameters = parameterise(&mut steps, probe); + let taken = names(&steps).len(); + if taken > MAX_PARAMS { + return Err(Refusal::TooManyParams(taken)); + } + Ok(Cleaned { + steps, + dropped, + skipped, + inferred_window, + parameters, + }) +} + +/// Whether a record that was read says its own run did not work. +/// +/// # Errors +/// +/// [`Refusal::RecordFailed`] when it does. +pub fn check_record(record: &Record) -> Result<(), Refusal> { + match &record.failed { + Some(why) => Err(Refusal::RecordFailed { + id: record.id.clone(), + why: first_line(why), + }), + None => Ok(()), + } +} + +/// The window part of a step's selector (or its `in_window`), as text, when it +/// names a window. +fn window_part_of(step: &Step) -> Option { + let raw = step + .args + .get("selector") + .or_else(|| step.args.get(IN_WINDOW))? + .as_str()?; + let selector = Selector::parse(raw).ok()?; + selector + .has_window_part() + .then(|| selector.render_window_part()) +} + +/// Clean an explicit step list: the same rules, with nothing to say about +/// failure because nothing was run. +/// +/// # Errors +/// +/// As [`learn_from_calls`]. +pub fn learn_from_steps(phrase: &str, steps: &[Step]) -> Result { + let calls: Vec = steps + .iter() + .map(|s| Call::ok(&s.tool, s.args.clone())) + .collect(); + learn_from_calls(phrase, &calls) +} + +/// The first line of an error, bounded: the whole of a tool refusal can be a +/// paragraph. +fn first_line(text: &str) -> String { + let line = text.lines().next().unwrap_or_default(); + let mut out: String = line.chars().take(200).collect(); + if line.chars().count() > 200 { + out.push('…'); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + /// The window every test selector names: a step that acts must say where. + const IN: &str = "app=settings;window~=Settings"; + + /// A click on the control `selector` names, in the test window. + fn sel_click(selector: &str) -> Call { + Call::ok( + "click_element", + json!({"selector": format!("{IN};{selector}")}), + ) + } + + fn learn(calls: &[Call]) -> Result { + learn_from_calls("open the settings dialog", calls) + } + + #[test] + fn a_run_of_selector_steps_is_kept_in_order() { + let cleaned = learn(&[ + sel_click("role=button;name=Settings"), + Call::ok( + "wait_for", + json!({"condition": "window_appears", "selector": "window~=Settings"}), + ), + Call::ok("type_text", json!({"text": "dark"})), + Call::ok("key", json!({"keys": ["enter"]})), + ]) + .unwrap(); + let tools: Vec<_> = cleaned.steps.iter().map(|s| s.tool.as_str()).collect(); + assert_eq!(tools, ["click_element", "wait_for", "type_text", "key"]); + assert_eq!(cleaned.dropped, 0); + // The typing names no window of its own: it is given the one the wait + // before it named, and the reply says so. + assert_eq!(cleaned.inferred_window, 2); + for typed in &cleaned.steps[2..] { + assert_eq!(typed.args[IN_WINDOW], "window~=Settings", "{typed:?}"); + } + } + + #[test] + fn a_step_that_acts_must_name_its_window() { + let bare = |args: Value| learn(&[Call::ok("click_element", args)]).unwrap_err(); + let why = bare(json!({"selector": "role=button;name=Save"})).to_string(); + assert!(why.contains("no window part"), "{why}"); + // Any of app, window or window~ is a window part. + for selector in [ + "app=notepad;role=button;name=Save", + "window=Untitled;role=button;name=Save", + "window~=Untitled;role=button;name=Save", + ] { + assert!( + learn(&[Call::ok("click_element", json!({"selector": selector}))]).is_ok(), + "{selector}" + ); + } + for tool in ["set_value", "element_action"] { + let args = json!({"selector": "role=edit;name=Body", "value": "x", "action": "press"}); + let why = learn(&[Call::ok(tool, args)]).unwrap_err().to_string(); + assert!(why.contains("no window part"), "{tool}: {why}"); + } + // An element wait is held to it too; a screen wait looks at no control. + let why = learn(&[Call::ok( + "wait_for", + json!({"condition": "element_appears", "selector": "role=edit;aid=search"}), + )]) + .unwrap_err() + .to_string(); + assert!(why.contains("no window part"), "{why}"); + } + + #[test] + fn a_keyboard_step_with_no_window_to_borrow_is_refused() { + for call in [ + Call::ok("type_text", json!({"text": "hi"})), + Call::ok("paste_text", json!({"text": "hi"})), + Call::ok("key", json!({"keys": ["enter"]})), + ] { + let why = learn(std::slice::from_ref(&call)).unwrap_err().to_string(); + assert!( + why.contains("whatever window has focus"), + "{}: {why}", + call.tool + ); + } + } + + #[test] + fn a_keyboard_step_may_name_its_window_and_that_window_must_parse_and_be_a_window() { + let typing = |in_window: Value| { + learn(&[Call::ok( + "type_text", + json!({"text": "hi", "in_window": in_window}), + )]) + }; + assert!(typing(json!("app=notepad;window~=Untitled")).is_ok()); + assert!(typing(json!("colour=red")).is_err(), "does not parse"); + assert!( + typing(json!("role=edit;name=Body")).is_err(), + "a control, not a window" + ); + assert!(typing(json!("")).is_err(), "names nothing"); + assert!(typing(json!(5)).is_err()); + } + + #[test] + fn the_window_borrowed_is_the_last_one_named_not_the_first() { + let cleaned = learn(&[ + Call::ok( + "click_element", + json!({"selector": "app=mail;window=Inbox;role=button;name=New"}), + ), + Call::ok( + "click_element", + json!({"selector": "app=mail;window=Compose;role=edit;name=Body"}), + ), + Call::ok("type_text", json!({"text": "hello"})), + ]) + .unwrap(); + assert_eq!(cleaned.steps[2].args[IN_WINDOW], "app=mail;window=Compose"); + } + + #[test] + fn looking_steps_are_dropped_and_wait_for_is_kept() { + let cleaned = learn(&[ + Call::ok("list_windows", json!({})), + Call::ok("ui_tree", json!({"depth": 4})), + Call::ok("screenshot", json!({})), + Call::ok("screen_info", json!({})), + Call::ok("cursor_position", json!({})), + sel_click("role=button;name=Settings"), + Call::ok( + "wait_for", + json!({"condition": "element_appears", "selector": "app=settings;role=edit;aid=search"}), + ), + ]) + .unwrap(); + assert_eq!(cleaned.dropped, 5); + assert_eq!(cleaned.steps.len(), 2); + assert_eq!(cleaned.steps[1].tool, "wait_for"); + } + + #[test] + fn an_element_number_is_refused_and_so_is_the_whole_routine() { + let why = learn(&[ + sel_click("role=button;name=Settings"), + Call::ok("click_element", json!({"id": 16})), + ]) + .unwrap_err(); + assert!( + matches!(&why, Refusal::NotReplayable { step: 2, tool, .. } if tool == "click_element"), + "{why:?}" + ); + assert!(why.to_string().contains("eN id"), "{why}"); + for tool in ["set_value", "element_action"] { + assert!( + learn(&[Call::ok( + tool, + json!({"id": 3, "value": "x", "action": "press"}) + )]) + .is_err(), + "{tool}" + ); + } + assert!(learn(&[Call::ok("scroll", json!({"dx": 0, "dy": -3, "id": 4}))]).is_err()); + } + + #[test] + fn raw_coordinates_are_refused_in_every_tool_that_takes_them() { + for call in [ + Call::ok("click", json!({"x": 10, "y": 20})), + Call::ok("move_mouse", json!({"x": 10, "y": 20})), + Call::ok( + "drag", + json!({"from_x": 1, "from_y": 2, "to_x": 3, "to_y": 4}), + ), + Call::ok("drag", json!({"from_id": 1, "to_id": 2})), + ] { + let why = learn(std::slice::from_ref(&call)).unwrap_err(); + assert!( + matches!(why, Refusal::NotReplayable { .. }), + "{} gave {why:?}", + call.tool + ); + } + // Scroll is refused before its arguments are read: it takes no selector. + let why = learn(&[Call::ok("scroll", json!({"dx": 0, "dy": 1}))]).unwrap_err(); + assert!(why.to_string().contains("under the pointer"), "{why}"); + } + + #[test] + fn a_window_handle_is_refused_and_focus_window_cannot_be_replayed() { + let why = learn(&[Call::ok( + "click_element", + json!({"selector": "role=button;name=OK", "window": "197330"}), + )]) + .unwrap_err(); + assert!(why.to_string().contains("window"), "{why}"); + assert!(learn(&[Call::ok("focus_window", json!({"window": "197330"}))]).is_err()); + assert!(learn(&[Call::ok("key", json!({"keys": ["a"], "window": "7"}))]).is_err()); + } + + #[test] + fn a_step_with_no_usable_selector_is_refused() { + assert!(learn(&[Call::ok("click_element", json!({}))]).is_err()); + assert!( + learn(&[sel_click("colour=red")]).is_err(), + "an unparseable selector" + ); + assert!(learn(&[Call::ok("set_value", json!({"selector": 5, "value": "x"}))]).is_err()); + // A screen wait has no control to name. + assert!(learn(&[Call::ok("wait_for", json!({"condition": "screen_changes"}))]).is_ok()); + assert!(learn(&[Call::ok( + "wait_for", + json!({"condition": "element_appears"}) + )]) + .is_err()); + } + + #[test] + fn a_wait_compared_with_one_earlier_screen_is_refused() { + let why = learn(&[Call::ok( + "wait_for", + json!({"condition": "screen_changes", "baseline": "00ff00ff00ff00ff00ff00ff00ff00ff"}), + )]) + .unwrap_err(); + assert!(why.to_string().contains("baseline"), "{why}"); + } + + #[test] + fn a_failed_call_is_a_failed_run() { + let why = learn(&[ + sel_click("role=button;name=Settings"), + sel_click("role=button;name=Missing").failed("matches no control\nsecond line"), + ]) + .unwrap_err(); + match &why { + Refusal::RunFailed { step, why, .. } => { + assert_eq!(*step, 2); + assert_eq!(why, "matches no control", "only the first line"); + } + other => panic!("{other:?}"), + } + } + + #[test] + fn a_wait_that_timed_out_or_was_stopped_is_a_failed_run() { + let wait = |answer: &str| { + Call::ok( + "wait_for", + json!({"condition": "window_appears", "selector": "window~=X"}), + ) + .answering(answer) + }; + assert!(matches!( + learn(&[wait( + "{\n \"result\": \"timeout\",\n \"waited_ms\": 5000\n}" + )]), + Err(Refusal::RunFailed { .. }) + )); + assert!(matches!( + learn(&[wait("{\"result\":\"stopped\"}")]), + Err(Refusal::RunFailed { .. }) + )); + assert!(learn(&[wait("{\n \"result\": \"found\"\n}")]).is_ok()); + } + + /// A run is failed when ANY call in it failed: a looking call that was + /// refused, a shell command that exited non-zero, a file that was not there. + #[test] + fn a_failed_call_of_any_kind_is_a_failed_run() { + let ok = sel_click("role=button;name=Settings"); + // The case that was missed: [Bash fails, incu key ok]. + let key = Call::ok( + "mcp__desktop__key", + json!({"keys": ["enter"], "in_window": IN}), + ); + let why = learn(&[ + Call::ok("Bash", json!({"command": "false"})).failed("exit code 1"), + key.clone(), + ]) + .unwrap_err(); + match &why { + Refusal::RunFailed { step, tool, why } => { + assert_eq!((*step, tool.as_str()), (1, "Bash")); + assert_eq!(why, "exit code 1"); + } + other => panic!("{other:?}"), + } + assert!(why.to_string().contains("call 1 (Bash)"), "{why}"); + // A looking call that failed, a call that is no desktop tool at all, and + // an error after the last desktop step. + for failing in [ + Call::ok("ui_tree", json!({})).failed("window has no tree"), + Call::ok("mcp__memory__recall_index", json!({})).failed("unknown scope"), + Call::ok("Read", json!({"file_path": "x"})).failed("no such file"), + ] { + assert!( + matches!( + learn(&[ok.clone(), failing.clone()]), + Err(Refusal::RunFailed { .. }) + ), + "{failing:?}" + ); + assert!(matches!( + learn(&[failing, ok.clone()]), + Err(Refusal::RunFailed { .. }) + )); + } + // And the same calls that all worked are learned. + assert!(learn(&[Call::ok("Bash", json!({"command": "true"})), key]).is_ok()); + } + + #[test] + fn a_record_whose_own_outcome_says_it_failed_is_not_learned() { + let record = Record::new("w-1", 1, vec![sel_click("role=button;name=Settings")]); + assert!(check_record(&record).is_ok()); + let failed = check_record(&record.failing("status is \"failed\"\nmore")).unwrap_err(); + assert_eq!( + failed, + Refusal::RecordFailed { + id: "w-1".to_string(), + why: "status is \"failed\"".to_string() + } + ); + assert!(failed.to_string().contains("w-1"), "{failed}"); + } + + /// Only incu's own servers: a name with "desktop" in it is not enough. + #[test] + fn another_servers_tools_are_never_taken_for_incus() { + for foreign in [ + "mcp__remote_desktop__key", + "mcp__desktop_sharing__click_element", + "mcp__my-incu-fork__key", + "mcp__Desktop__key", + "mcp__desktopx__wait_for", + "mcp__incu2__wait_for", + ] { + assert_eq!(desktop_tool(foreign), None, "{foreign}"); + } + for own in OWN_SERVERS { + let name = format!("mcp__{own}__wait_for"); + assert_eq!(desktop_tool(&name), Some("wait_for"), "{name}"); + } + // Left out of a routine and reported, as any other call that is not ours. + let cleaned = learn(&[ + Call::ok("mcp__remote_desktop__key", json!({"keys": ["enter"]})), + sel_click("role=button;name=New"), + ]) + .unwrap(); + assert_eq!(cleaned.steps.len(), 1); + assert_eq!(cleaned.skipped, ["mcp__remote_desktop__key"]); + } + + /// click username, type "alice", Tab, type "hunter2", Enter. + #[test] + fn a_password_typed_after_tab_is_never_learned_as_text() { + let cleaned = learn(&[ + sel_click("role=edit;name=Username"), + Call::ok("type_text", json!({"text": "alice", "in_window": IN})), + Call::ok("key", json!({"keys": ["Tab"], "in_window": IN})), + Call::ok("type_text", json!({"text": "hunter2", "in_window": IN})), + Call::ok("key", json!({"keys": ["Enter"], "in_window": IN})), + ]) + .unwrap(); + // No step names a field, so no typed text is kept: not the password, and + // not the user name either. + assert_eq!(cleaned.parameters.len(), 2); + assert_eq!(cleaned.steps[3].args["text"], "{{text_2}}"); + assert_eq!(cleaned.steps[1].args["text"], "{{text}}"); + let kept = format!("{cleaned:?}"); + assert!( + !kept.contains("hunter2") && !kept.contains("alice"), + "{kept}" + ); + } + + /// Text is kept when the step itself names a plain text box that the desktop + /// confirms is not a password field, and not otherwise. + #[test] + fn a_step_that_names_a_plain_field_keeps_its_text_when_the_desktop_confirms_it() { + let named = |field: &str| { + Call::ok( + "type_text", + json!({"text": "rust", "in_window": IN, "selector": format!("{IN};{field}")}), + ) + }; + let plain = |_: &str| Probe::Plain; + let kept = + learn_with("search for rust", &[named("role=edit;name=Search")], &plain).unwrap(); + assert!(kept.parameters.is_empty()); + assert_eq!(kept.steps[0].args["text"], "rust"); + // The desktop could not check it: a parameter. + let unchecked = + learn_from_calls("search for rust", &[named("role=edit;name=Search")]).unwrap(); + assert_eq!(unchecked.steps[0].args["text"], "{{search}}"); + // Marked as a password by the desktop, whatever it is called. + let secret = learn_with( + "search for rust", + &[named("role=edit;name=Search")], + &|_: &str| Probe::Secret, + ) + .unwrap(); + assert_eq!(secret.steps[0].args["text"], "{{search}}"); + assert_eq!( + secret.parameters[0].why, + crate::routine::params::Why::SecureField + ); + // A selector that names a window and no control is refused outright: the + // replay would have nothing to focus. + let no_control = Call::ok( + "type_text", + json!({"text": "x", "in_window": IN, "selector": IN}), + ); + assert!(matches!( + learn_with("search for rust", &[no_control], &plain), + Err(Refusal::NotReplayable { .. }) + )); + // One with no window part would be focused in whatever is in front. + let no_window = Call::ok( + "type_text", + json!({"text": "x", "in_window": IN, "selector": "role=edit;name=Search"}), + ); + assert!(matches!( + learn_with("search for rust", &[no_window], &plain), + Err(Refusal::NotReplayable { .. }) + )); + } + + /// A PIN typed as key presses is typing, past every screen on typed text. + #[test] + fn a_pin_typed_as_key_presses_refuses_the_whole_learn() { + let press = |key: &str| Call::ok("key", json!({"keys": [key], "in_window": IN})); + let calls = [ + sel_click("role=edit;name=Code"), + press("4"), + press("2"), + press("7"), + press("1"), + press("enter"), + ]; + let why = learn(&calls).unwrap_err(); + match &why { + Refusal::NotReplayable { step, tool, why } => { + assert_eq!((*step, tool.as_str()), (2, "key")); + assert!(why.contains("printable character"), "{why}"); + assert!(why.contains("type_text"), "{why}"); + } + other => panic!("{other:?}"), + } + // Whatever the character: a letter, a symbol, with shift or without. + for keys in [ + json!(["a"]), + json!(["A"]), + json!(["shift", "a"]), + json!(["shift", "4"]), + json!(["!"]), + json!(["-"]), + json!("shift+9"), + json!(["numpad4"]), + json!([]), + ] { + let step = Call::ok("key", json!({"keys": keys, "in_window": IN})); + assert!( + matches!(learn(&[step]), Err(Refusal::NotReplayable { .. })), + "{keys}" + ); + } + } + + #[test] + fn named_keys_and_command_chords_are_kept_as_key_steps() { + for keys in [ + json!(["enter"]), + json!(["Return"]), + json!(["tab"]), + json!(["shift", "tab"]), + json!(["escape"]), + json!(["Esc"]), + json!(["up"]), + json!(["Down"]), + json!(["left"]), + json!(["right"]), + json!(["f1"]), + json!(["F5"]), + json!(["f24"]), + json!(["home"]), + json!(["end"]), + json!(["pageup"]), + json!(["PageDown"]), + json!(["page_down"]), + json!(["backspace"]), + json!(["delete"]), + json!(["space"]), + json!(["ctrl", "a"]), + json!(["control", "shift", "t"]), + json!(["alt", "f4"]), + json!(["cmd", "c"]), + json!(["win", "e"]), + json!(["meta", "l"]), + json!("ctrl+s"), + json!("shift+tab"), + ] { + let step = Call::ok("key", json!({"keys": keys, "in_window": IN})); + assert!(learn(&[step]).is_ok(), "{keys}"); + } + // F25 is no F-key. + let f25 = Call::ok("key", json!({"keys": ["f25"], "in_window": IN})); + assert!(learn(&[f25]).is_err()); + } + + #[test] + fn a_routine_that_would_take_too_many_parameters_is_refused() { + let mut calls = vec![sel_click("role=edit;name=Username")]; + for i in 0..=MAX_PARAMS { + calls.push(Call::ok( + "type_text", + json!({"text": format!("{{{{mine{i}}}}}"), "in_window": IN}), + )); + } + let why = learn(&calls).unwrap_err(); + assert_eq!(why, Refusal::TooManyParams(MAX_PARAMS + 1)); + assert!( + why.to_string().contains("more than a routine may take"), + "{why}" + ); + } + + #[test] + fn typing_into_a_password_field_is_learned_as_a_parameter() { + let cleaned = learn(&[ + sel_click("role=edit;name=Password"), + Call::ok( + "type_text", + json!({"text": "hunter2-secret", "in_window": IN}), + ), + ]) + .unwrap(); + assert_eq!(cleaned.parameters.len(), 1); + assert_eq!(cleaned.steps[1].args["text"], "{{text}}"); + assert!(!format!("{cleaned:?}").contains("hunter2")); + // A step that names the field itself, with the name saying it is secret. + let named = learn_with( + "sign in", + &[Call::ok( + "type_text", + json!({"text": "hunter2-secret", "in_window": IN, "selector": format!("{IN};role=edit;name=Password")}), + )], + &|_: &str| Probe::Plain, + ) + .unwrap(); + assert_eq!(named.steps[0].args["text"], "{{password}}"); + // The desktop's own word that a control is a password field counts too. + let by_desktop = learn_with( + "sign in", + &[Call::ok( + "set_value", + json!({"selector": format!("{IN};role=edit;name=Code"), "value": "482913"}), + )], + &|raw: &str| { + if raw.contains("name=Code") { + Probe::Secret + } else { + Probe::Plain + } + }, + ) + .unwrap(); + assert_eq!(by_desktop.steps[0].args["value"], "{{code}}"); + } + + #[test] + fn this_and_that_are_refused_before_anything_is_looked_at() { + let calls = [sel_click("role=button;name=Settings")]; + assert_eq!( + learn_from_calls("close this", &calls).unwrap_err(), + Refusal::Deictic + ); + assert_eq!( + learn_from_calls("click that button", &calls).unwrap_err(), + Refusal::Deictic + ); + assert_eq!( + learn_from_calls("please do the", &calls).unwrap_err(), + Refusal::NoWords + ); + } + + #[test] + fn nothing_but_looking_is_not_a_routine() { + assert_eq!( + learn(&[ + Call::ok("ui_tree", json!({})), + Call::ok("screenshot", json!({})) + ]) + .unwrap_err(), + Refusal::NoSteps + ); + assert_eq!(learn(&[]).unwrap_err(), Refusal::NoSteps); + } + + #[test] + fn too_many_steps_are_refused() { + let calls: Vec = (0..=MAX_STEPS) + .map(|_| sel_click("role=button;name=Next")) + .collect(); + assert_eq!(learn(&calls).unwrap_err(), Refusal::TooLong(MAX_STEPS + 1)); + } + + #[test] + fn calls_that_are_not_desktop_tools_are_left_out_and_reported() { + let cleaned = learn(&[ + Call::ok("Bash", json!({"command": "open -a Notes"})), + sel_click("role=button;name=New"), + Call::ok("mcp__memory__recall_index", json!({})), + ]) + .unwrap(); + assert_eq!(cleaned.steps.len(), 1); + assert_eq!(cleaned.skipped, ["Bash", "mcp__memory__recall_index"]); + } + + #[test] + fn an_mcp_name_counts_only_from_a_server_that_looks_like_this_one() { + assert_eq!(desktop_tool("click_element"), Some("click_element")); + assert_eq!( + desktop_tool("mcp__desktop__click_element"), + Some("click_element") + ); + assert_eq!(desktop_tool("mcp__incode-desktop__key"), Some("key")); + assert_eq!(desktop_tool("mcp__incu__wait_for"), Some("wait_for")); + // Someone else's `key`, and a name that is not a desktop tool at all. + assert_eq!(desktop_tool("mcp__computer-use__key"), None); + assert_eq!(desktop_tool("mcp__desktop__format_disk"), None); + assert_eq!(desktop_tool("Bash"), None); + } + + #[test] + fn the_picture_asked_for_after_an_action_is_not_kept() { + let cleaned = learn(&[Call::ok( + "click_element", + json!({"selector": "app=dialog;role=button;name=OK", "screenshot_after": true}), + )]) + .unwrap(); + assert_eq!( + cleaned.steps[0].args, + json!({"selector": "app=dialog;role=button;name=OK"}) + ); + } + + #[test] + fn routine_tools_and_watch_are_not_steps() { + for tool in ["routine_run", "routine_save", "watch"] { + assert!( + matches!( + learn(&[Call::ok(tool, json!({"phrase": "x"}))]), + Err(Refusal::NotReplayable { .. }) + ), + "{tool}" + ); + } + } + + #[test] + fn an_explicit_list_is_cleaned_by_the_same_rules() { + let ok = learn_from_steps( + "sign in to the portal", + &[Step::new( + "click_element", + json!({"selector": "app=portal;role=button;name=Sign in"}), + )], + ); + assert!(ok.is_ok()); + let bad = learn_from_steps( + "sign in to the portal", + &[Step::new("click", json!({"x": 1, "y": 2}))], + ); + assert!(bad.is_err()); + } + + #[test] + fn check_step_is_what_a_replay_asks_of_a_stored_step() { + let keys = json!({"keys": ["ctrl", "s"], "in_window": "app=notepad"}); + assert!(check_step(&Step::new("key", keys), 1).is_ok()); + let bare = Step::new("key", json!({"keys": ["ctrl", "s"]})); + assert!(check_step(&bare, 1).is_err(), "no window named"); + assert!(check_step(&Step::new("click_element", json!({"id": 1})), 1).is_err()); + assert!(check_step(&Step::new("key", json!("ctrl+s")), 1).is_err()); + } +} diff --git a/crates/desktop/src/routine/mod.rs b/crates/desktop/src/routine/mod.rs new file mode 100644 index 0000000..6840f56 --- /dev/null +++ b/crates/desktop/src/routine/mod.rs @@ -0,0 +1,321 @@ +//! Routines: what an agent did once at the desktop, kept so it can be done +//! again with no model. +//! +//! # Where this sits +//! +//! A routine is a trigger phrase and the list of desktop tool calls that +//! carried it out. [`learn`] decides whether a run is fit to keep and cleans +//! it, [`store`] keeps the routines in one JSON file, [`words`] finds the +//! routine a phrase asks for, and [`replay`] runs one again, step by step, to +//! the first failure. The MCP tools in `mcp::routine_tool` are the only +//! callers; they are the seam where the live desktop and its gates enter. +//! +//! # What is reused and what is not built +//! +//! There is **no trace format and no trace store**. Watchman already records +//! every tool call, so a routine is made from Watchman's records +//! ([`source::StepSource`], with [`watchman::WatchmanSource`] reading the local +//! memory engine's read-only `/v1/memory/watchman/*` endpoints) or from an +//! explicit step list. The engine is optional: [`source::MemorySource`] is the +//! in-memory source tests use, and a crate with no engine at all still saves, +//! lists, matches and replays routines from an explicit list. +//! +//! The design is the learned-shortcut design of the Atticus project (MIT; +//! `atticus.py`, "learned shortcuts"), kept as simple as it is there: +//! +//! - per routine: the cleaned steps, the trigger phrase, the date, when it was +//! last used, and whether the person kept it (`incu routine keep`: no tool can +//! set it); +//! - a phrase matches when its WORD SET is the routine's, in any order, with +//! the filler words ignored ([`words`]); +//! - nothing is learned when the phrase says "this" or "that", when the run +//! failed, or when any step could not be replayed on another day: an `eN` +//! id, a pixel coordinate or a window handle ([`learn`]); +//! - reading steps (`ui_tree`, `screenshot`, ...) are dropped, `wait_for` is +//! kept; +//! - a replay that fails reports the step and stops. There is NO step repair: +//! the caller redoes the whole task and saves the new run, which replaces +//! the routine; +//! - a routine unused for [`EXPIRY_DAYS`] days is forgotten, unless the user +//! taught it. +//! +//! # Not here +//! +//! Voice, scheduling, chats and the browser belong to Watchman. This module +//! never calls a model and never schedules anything. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +pub mod learn; +pub mod oslock; +pub mod params; +pub mod pictures; +pub mod replay; +pub mod source; +pub mod store; +pub mod watchman; +pub mod words; + +/// A routine unused this long is forgotten, unless the user taught it. +pub const EXPIRY_DAYS: i64 = 30; + +/// Milliseconds in a day, for expiry. +pub const DAY_MS: i64 = 24 * 60 * 60 * 1000; + +/// The most steps one routine may hold. A run longer than this is not a +/// shortcut, and a bound keeps a hand-edited file from being replayed forever. +pub const MAX_STEPS: usize = 100; + +/// The most routines the file holds; saving a new one past this is refused +/// rather than silently dropping an old one. +pub const MAX_ROUTINES: usize = 100; + +/// The most one step's arguments may take as JSON, in bytes. Typed text is the +/// bulk of it; a routine that types a page is not a shortcut. +pub const MAX_STEP_BYTES: usize = 4 * 1024; + +/// The most one routine may take, by [`Routine::size_bytes`]. With +/// [`MAX_ROUTINES`] of them the file stays under what it may be +/// ([`store::MAX_FILE_BYTES`]), with room for the indentation the file is +/// written with; a test writes the largest file the two allow and reads it back. +pub const MAX_ROUTINE_BYTES: usize = 24 * 1024; + +/// What a step costs beyond its own bytes: the keys, quotes and indentation of +/// the readable file. Counted so that a routine of many tiny steps, which is +/// mostly structure, is not mistaken for a small one. +const STEP_OVERHEAD_BYTES: usize = 160; + +/// One desktop tool call to run again: the tool and the arguments it was +/// given. The same names and arguments the live tools take, so one dispatcher +/// runs both. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Step { + /// A desktop tool's name, as `tools/list` offers it. + pub tool: String, + /// Its arguments. + #[serde(default)] + pub args: Value, +} + +impl Step { + /// A step. + #[must_use] + pub fn new(tool: &str, args: Value) -> Self { + Self { + tool: tool.to_string(), + args, + } + } + + /// The step in one line, for the agent and the operator. Typed text and + /// values are shown by length only, and a selector without a text field's + /// name, as the journal shows them (`mcp::gate`, the redaction boundary): a + /// line about a routine must not carry what a password field was given. + #[must_use] + pub fn summary(&self) -> String { + let mut shown = Vec::new(); + if let Some(map) = self.args.as_object() { + for (key, value) in map { + let text = match (key.as_str(), value.as_str()) { + ("text" | "value", Some(s)) => format!("{} characters", s.chars().count()), + // As the journal has a selector: the name of a text field + // is what was typed in it, so it is left out + // ([`crate::selector::Selector::render_redacted`]). This line + // is in the failure of a replay, which is journalled whole. + ("selector", Some(s)) => crate::selector::Selector::parse(s).map_or_else( + |_| "(unreadable)".to_string(), + |sel| format!("\"{}\"", sel.render_redacted()), + ), + _ => value.to_string(), + }; + shown.push(format!("{key}={text}")); + } + } + if shown.is_empty() { + self.tool.clone() + } else { + format!("{} {}", self.tool, shown.join(" ")) + } + } + + /// The step in one line with EVERYTHING it types and every selector in + /// full: what `routine_save`, `routine_list` and `routine_match` show, so the + /// person can read exactly what a routine will type and where before it runs. + /// + /// A parameter (`{{password}}`) is shown as that name, which is all the file + /// holds of it. Not for the journal or a failure message: those take + /// [`Step::summary`], which keeps typed text out. + #[must_use] + pub fn review(&self) -> String { + let mut shown = Vec::new(); + if let Some(map) = self.args.as_object() { + for (key, value) in map { + shown.push(format!("{key}={value}")); + } + } + if shown.is_empty() { + self.tool.clone() + } else { + format!("{} {}", self.tool, shown.join(" ")) + } + } +} + +/// A learned routine. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Routine { + /// The phrase it was learned under, as the user said it. + pub phrase: String, + /// What to run, in order, already cleaned. + pub steps: Vec, + /// The day it was learned, `YYYY-MM-DD`. For people; expiry uses + /// [`Routine::last_used_ms`]. + pub date: String, + /// When it last ran to the end (or was learned), in Unix milliseconds. + pub last_used_ms: i64, + /// The person asked to keep it (`incu routine keep`). Such a routine never + /// expires. Only the person sets this: no tool takes it as an argument, so an + /// agent cannot exempt what it saved from expiry. A save that replaces a kept + /// routine with other steps un-keeps it. + #[serde(default)] + pub taught: bool, + /// Which environment variable each of the routine's parameters (`{{name}}` in + /// its steps) is read from, bound by the person (`incu routine bind`). Only + /// the variable's NAME is stored, never a value. Empty for a routine with no + /// parameters, or none bound yet. + #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")] + pub bindings: std::collections::BTreeMap, + /// [`params::digest`] of the steps when the bindings were made. A routine + /// whose steps no longer have this digest is not run: the bindings are for + /// other steps. Empty when nothing is bound. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub bound_steps: String, +} + +impl Routine { + /// What this routine takes of the file, in bytes, as [`MAX_ROUTINE_BYTES`] + /// counts it: its phrase, each step's tool and arguments, and + /// [`STEP_OVERHEAD_BYTES`] for each step. + #[must_use] + pub fn size_bytes(&self) -> usize { + self.phrase.len() + + 64 + + self + .steps + .iter() + .map(|s| s.tool.len() + s.args.to_string().len() + STEP_OVERHEAD_BYTES) + .sum::() + // The bindings (a name and a variable each) and the digest of the steps. + + self + .bindings + .iter() + .map(|(name, var)| name.len() + var.len() + 32) + .sum::() + + self.bound_steps.len() + } + + /// Whether this routine has gone unused long enough to be forgotten at + /// `now_ms`. A routine the user taught never is. A clock that went + /// backwards (a `last_used_ms` after `now_ms`) does not expire anything. + #[must_use] + pub fn is_expired(&self, now_ms: i64) -> bool { + !self.taught && now_ms.saturating_sub(self.last_used_ms) > EXPIRY_DAYS * DAY_MS + } + + /// Whole days until it expires, or `None` for a routine that never does. + #[must_use] + pub fn days_left(&self, now_ms: i64) -> Option { + if self.taught { + return None; + } + let left = EXPIRY_DAYS * DAY_MS - now_ms.saturating_sub(self.last_used_ms); + // Rounded up: a routine saved a moment ago has 30 days, not 29. + Some(((left + DAY_MS - 1) / DAY_MS).max(0)) + } +} + +/// `YYYY-MM-DD` of a Unix time in milliseconds (UTC). +#[must_use] +pub fn day_of(ms: i64) -> String { + chrono::DateTime::from_timestamp_millis(ms) + .map(|t| t.format("%Y-%m-%d").to_string()) + .unwrap_or_default() +} + +/// Now, in Unix milliseconds. +#[must_use] +pub fn now_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX)) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn routine(taught: bool, last_used_ms: i64) -> Routine { + Routine { + phrase: "open notes".to_string(), + steps: vec![Step::new("key", json!({"keys": ["ctrl", "n"]}))], + date: "2026-01-01".to_string(), + last_used_ms, + taught, + bindings: Default::default(), + bound_steps: String::new(), + } + } + + #[test] + fn a_routine_expires_after_thirty_unused_days_and_not_before() { + let r = routine(false, 0); + assert!( + !r.is_expired(30 * DAY_MS), + "exactly thirty days is not over" + ); + assert!(r.is_expired(30 * DAY_MS + 1)); + assert!(!r.is_expired(1)); + } + + #[test] + fn a_routine_the_user_taught_never_expires() { + let r = routine(true, 0); + assert!(!r.is_expired(10_000 * DAY_MS)); + assert_eq!(r.days_left(10_000 * DAY_MS), None); + } + + #[test] + fn a_clock_that_went_backwards_expires_nothing() { + assert!(!routine(false, 100 * DAY_MS).is_expired(0)); + } + + #[test] + fn days_left_counts_down_and_stops_at_zero() { + let r = routine(false, 0); + assert_eq!(r.days_left(0), Some(30)); + assert_eq!(r.days_left(10 * DAY_MS), Some(20)); + assert_eq!(r.days_left(99 * DAY_MS), Some(0)); + } + + #[test] + fn a_step_summary_never_carries_typed_text() { + let s = Step::new("type_text", json!({"text": "hunter2 and more"})); + let line = s.summary(); + assert!(line.contains("16 characters"), "{line}"); + assert!(!line.contains("hunter2"), "{line}"); + let s = Step::new( + "set_value", + json!({"selector": "role=edit", "value": "secret"}), + ); + assert!(!s.summary().contains("secret"), "{}", s.summary()); + assert!(s.summary().contains("role=edit")); + } + + #[test] + fn the_day_is_a_utc_date() { + assert_eq!(day_of(0), "1970-01-01"); + assert_eq!(day_of(1_790_107_969_427), "2026-09-22"); + } +} diff --git a/crates/desktop/src/routine/oslock.rs b/crates/desktop/src/routine/oslock.rs new file mode 100644 index 0000000..df8835e --- /dev/null +++ b/crates/desktop/src/routine/oslock.rs @@ -0,0 +1,151 @@ +//! An exclusive OS lock on a file, held for as long as the value lives. +//! +//! # Why this and not `std::fs::File::lock` +//! +//! The standard library has had `File::try_lock` since Rust 1.89, and it is the +//! natural thing to use. This crate's declared floor is 1.88 (`rust-version`, +//! measured against the whole dependency tree and written into `Cargo.toml`, +//! `README.md` and `CONTRIBUTING.md`), so using it would make that floor false, +//! and moving the floor is a decision for the whole project, not for one module. +//! `fd-lock` 4.0.4 is the same thing (`flock` on Unix, `LockFileEx` on Windows), +//! it is already in the lock file (rustyline uses it), and it adds no package. +//! Swapping this module for `File::try_lock` when the floor moves is a change +//! to this file alone. +//! +//! # What the lock is +//! +//! The OS holds it exactly as long as the handle that took it is open and drops +//! it the instant the holder exits or crashes. That is the property that makes it +//! right for a lock file: no file "left behind" can be stale, so nothing has to +//! guess at a dead holder from the age of a file (which is how a holder merely +//! slower than the guess got robbed). +//! +//! `fd-lock` hands out a guard that unlocks when dropped and borrows the lock. +//! This module does not need either: it takes the lock, forgets the guard, and +//! lets the OS release it when the file is closed, which happens when the +//! [`OsLock`] is dropped. So a held lock is just a value that can be stored. + +use std::fs::File; +use std::io; +use std::time::{Duration, Instant}; + +/// A file that may be locked, and holds the lock once it has been taken. +pub struct OsLock { + inner: fd_lock::RwLock, +} + +impl std::fmt::Debug for OsLock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("OsLock") + } +} + +impl OsLock { + /// A lock on `file`. Nothing is locked until [`OsLock::try_acquire`]. + #[must_use] + pub fn new(file: File) -> Self { + Self { + inner: fd_lock::RwLock::new(file), + } + } + + /// Try once to take the lock for this process: `Ok(true)` when it is held + /// now, `Ok(false)` when another holder has it. + /// + /// # Errors + /// + /// The OS refused for another reason. + pub fn try_acquire(&mut self) -> io::Result { + match self.inner.try_write() { + Ok(guard) => { + // Kept: the OS releases it when the file is closed, with `self`. + std::mem::forget(guard); + Ok(true) + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => Ok(false), + Err(e) => Err(e), + } + } + + /// Take the lock, trying again every 10 ms for up to `wait`. `Err(None)` is + /// "still held when the time ran out"; `Err(Some(e))` is the OS refusing for + /// another reason. + /// + /// # Errors + /// + /// As above. + pub fn acquire_within(&mut self, wait: Duration) -> Result<(), Option> { + let started = Instant::now(); + loop { + match self.try_acquire() { + Ok(true) => return Ok(()), + Ok(false) if started.elapsed() >= wait => return Err(None), + Ok(false) => std::thread::sleep(Duration::from_millis(10)), + Err(e) => return Err(Some(e)), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(name: &str) -> (std::path::PathBuf, File) { + let dir = std::env::temp_dir().join(format!("incu-oslock-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("lock"); + let handle = open(&path); + (dir, handle) + } + + fn open(path: &std::path::Path) -> File { + File::options() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(path) + .unwrap() + } + + #[test] + fn a_lock_is_held_until_its_handle_goes_and_a_second_handle_cannot_have_it() { + let (dir, first) = file("held"); + let mut first = OsLock::new(first); + assert!(first.try_acquire().unwrap()); + let mut second = OsLock::new(open(&dir.join("lock"))); + assert!(!second.try_acquire().unwrap(), "two holders"); + // The holder goes without unlocking, as a crash does: the OS lets go. + drop(first); + assert!( + second.try_acquire().unwrap(), + "the lock outlived its holder" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn waiting_ends_when_the_holder_lets_go_or_when_the_time_is_up() { + let (dir, first) = file("wait"); + let mut first = OsLock::new(first); + assert!(first.try_acquire().unwrap()); + let mut waiter = OsLock::new(open(&dir.join("lock"))); + // Held throughout: gives up, saying so (not an OS error). + let started = Instant::now(); + assert!(matches!( + waiter.acquire_within(Duration::from_millis(150)), + Err(None) + )); + assert!(started.elapsed() >= Duration::from_millis(150)); + // Let go partway: the waiter gets it. + let releaser = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(200)); + drop(first); + }); + waiter.acquire_within(Duration::from_secs(5)).unwrap(); + releaser.join().unwrap(); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/desktop/src/routine/params.rs b/crates/desktop/src/routine/params.rs new file mode 100644 index 0000000..ce53426 --- /dev/null +++ b/crates/desktop/src/routine/params.rs @@ -0,0 +1,1348 @@ +//! Parameters: text a routine is GIVEN each time it runs instead of keeping. +//! +//! # Why +//! +//! The routines file holds what a routine types, so that the person can read it +//! and so that a replay needs no model. Text typed into a password or other +//! secure field must not be on disk at all, and neither must a recording's +//! stand-in for text it redacted (replaying `[REDACTED]` into a login box is a +//! wrong password, not a shortcut). Such a step is made a PARAMETER when the +//! routine is learned: its text is stored as `{{name}}`, and `routine_run` +//! refuses before step 1 unless every one is bound. The value is substituted into +//! the arguments of the one call that types it, in memory. It is not written to +//! the routines file or the journal ([`crate::mcp`]'s `describe` shows the length +//! only), it is hidden from the operator's hooks, and [`Values::scrub`] takes it +//! back out of any message a replay returns. +//! +//! # The PERSON binds parameters, not the agent +//! +//! `routine_run` takes no `params`. Each `{{param}}` is read from the environment +//! variable the person bound it to with `incu routine bind +//! ` at a terminal. Only the variable's NAME is stored (in the routine, +//! with a [`digest`] of its steps at the time), never a value. Had the agent +//! named the variable, it could name any `INCU_PARAM_*` variable the person set +//! for some other routine and have it typed into a window of its choosing; and a +//! literal value would land in the agent's own transcript. Saving different steps +//! drops every binding (as it drops the kept flag), and a routine whose steps are +//! not the ones that were bound is refused. Variable names must begin +//! [`ENV_PREFIX`], so a parameter can never be bound to `PATH` or an API key. +//! +//! # What makes a step a parameter +//! +//! **The rule is inverted: typed text is stored literally only when the step +//! itself names a control that is positively not secret** ([`why_not_kept`]): a +//! selector of its own with a window part, role `edit` or `combobox`, no +//! secret-looking name or automation id ([`is_secret_name`]), and not marked as a +//! password field by the desktop when the routine is saved. Everything else is a +//! parameter: +//! +//! - a typing step with no selector of its own (it types where focus is; focus +//! moves without a key, to a sign-in dialog a click opened, a first-step sudo +//! or KeePass prompt, the address bar by Ctrl+L, Alt+D or F6, and no list of +//! such moves is complete); +//! - one that names a button or other control that is not a text box, or a secure +//! one, or one that could not be checked; +//! - text a recording replaced with a placeholder or a digest +//! ([`looks_redacted`]), or a value that is not text at all; +//! - text that already has `{{name}}` in it: the agent made it one. +//! +//! This is a screen on what the learner can see, not a promise (a `key` that +//! types a password a letter at a time is not looked at), which is why +//! `routine_save`, `routine_list` and `routine_match` show the full text of every +//! step: the person reads what is kept. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; + +use super::{Routine, Step}; +use crate::selector::{Selector, Text}; +use crate::uia::Role; + +/// The arguments that carry typed text, by tool. +fn text_key(tool: &str) -> Option<&'static str> { + match tool { + "set_value" => Some("value"), + "type_text" | "paste_text" => Some("text"), + _ => None, + } +} + +/// The longest a parameter's value may be, in characters. +pub const MAX_PARAM_CHARS: usize = 1024; + +/// What the name of an environment variable a parameter reads must begin with. +pub const ENV_PREFIX: &str = "INCU_PARAM_"; + +/// The most parameters one routine may take. +pub const MAX_PARAMS: usize = 16; + +/// Whether `name` can be a parameter's: a lowercase letter, then lowercase +/// letters, digits and `_`, at most 32. +fn is_name(name: &str) -> bool { + let mut chars = name.chars(); + chars.next().is_some_and(|c| c.is_ascii_lowercase()) + && name.len() <= 32 + && chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') +} + +/// The parameter names in `text`, in order of appearance (`{{name}}`). +#[must_use] +pub fn tokens(text: &str) -> Vec { + let mut found = Vec::new(); + let mut rest = text; + while let Some(start) = rest.find("{{") { + let after = &rest[start + 2..]; + match after.find("}}") { + Some(end) if is_name(&after[..end]) => { + found.push(after[..end].to_string()); + rest = &after[end + 2..]; + } + _ => rest = after, + } + } + found +} + +/// The parameter names a step takes. +#[must_use] +pub fn names_of(step: &Step) -> Vec { + text_key(&step.tool) + .and_then(|key| step.args.get(key)) + .and_then(Value::as_str) + .map(tokens) + .unwrap_or_default() +} + +/// The parameter names a routine takes, each once, in order of first use. +#[must_use] +pub fn names(steps: &[Step]) -> Vec { + let mut seen = BTreeSet::new(); + steps + .iter() + .flat_map(names_of) + .filter(|n| seen.insert(n.clone())) + .collect() +} + +/// The values a run was given, by name. +#[derive(Debug, Default, Clone)] +pub struct Values(BTreeMap); + +impl Values { + /// The values of the parameters `routine` takes, read from the environment + /// variables the PERSON bound them to (`incu routine bind`), in this + /// process's own environment. + /// + /// # Errors + /// + /// See [`Values::from_routine_with`]. + pub fn from_routine(routine: &Routine) -> Result { + Self::from_routine_with(routine, &|var| std::env::var(var).ok()) + } + + /// [`Values::from_routine`] with `env` as the environment, so a test need not + /// set the process's. + /// + /// **The agent does not choose where a value comes from.** `routine_run` takes + /// no `params`: each `{{param}}` is read from the variable the person bound it + /// to, so an agent cannot name some other variable of incu's environment and + /// have it typed into a window of its choosing. The binding is stored with a + /// digest of the routine's steps at the time ([`digest`]), and a routine whose + /// steps are not the ones that were bound is refused. + /// + /// # Errors + /// + /// A parameter is not bound (the error names the command that binds it); the + /// steps are not the ones that were bound; a bound variable name is not one + /// this program accepts ([`check_env_name`], as the file may have been edited); + /// or the variable is not set, is empty, or is too long. All before any step + /// runs. No error contains a value. + pub fn from_routine_with( + routine: &Routine, + env: &dyn Fn(&str) -> Option, + ) -> Result { + let needed = names(&routine.steps); + if needed.len() > MAX_PARAMS { + return Err(format!( + "this routine takes {} parameters, more than the {MAX_PARAMS} one may; it was \ + not made by this program", + needed.len() + )); + } + if needed.is_empty() { + return Ok(Values::default()); + } + let unbound: Vec<&String> = needed + .iter() + .filter(|n| !routine.bindings.contains_key(*n)) + .collect(); + if !unbound.is_empty() { + return Err(format!( + "routine_run cannot start: {} {} not bound to a value. Nothing was done. These \ + are the secrets and redacted texts the routine does not keep, and only the \ + PERSON binds them, at a terminal, to an environment variable of incu's own: \ + `incu routine bind \"{}\" {} {ENV_PREFIX}{}`", + unbound + .iter() + .map(|n| n.as_str()) + .collect::>() + .join(", "), + if unbound.len() == 1 { "is" } else { "are" }, + routine.phrase, + unbound[0], + unbound[0].to_uppercase() + )); + } + if routine.bound_steps != digest(&routine.steps) { + return Err(format!( + "routine_run cannot start: the steps of \"{}\" are not the ones its parameters \ + were bound for (the file was changed since). Nothing was done. The person \ + reads the routine again (`incu routine list`) and binds again with \ + `incu routine bind`", + routine.phrase + )); + } + let mut values = BTreeMap::new(); + for name in &needed { + let var = &routine.bindings[name]; + check_env_name(var).map_err(|e| format!("the binding of {name} is not usable: {e}"))?; + let value = env(var).filter(|v| !v.is_empty()).ok_or_else(|| { + format!( + "routine_run cannot start: the environment variable {var} that {name} is \ + bound to is not set (or is empty) in incu's environment. The person sets \ + it where incu is launched. Nothing was done" + ) + })?; + if value.chars().count() > MAX_PARAM_CHARS { + return Err(format!( + "the value of {var} is over {MAX_PARAM_CHARS} characters" + )); + } + values.insert(name.clone(), value); + } + Ok(Values(values)) + } + + /// Whether there are no values. + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// `step` with its `{{name}}` parts replaced by the values. In one pass: a + /// value that itself contains `{{x}}` is typed as it is. + #[must_use] + pub fn fill(&self, step: &Step) -> Step { + let Some(key) = text_key(&step.tool) else { + return step.clone(); + }; + let mut filled = step.clone(); + if let Some(text) = step.args.get(key).and_then(Value::as_str) { + filled.args[key] = Value::String(self.render(text)); + } + filled + } + + fn render(&self, template: &str) -> String { + let mut out = String::new(); + let mut rest = template; + while let Some(start) = rest.find("{{") { + out.push_str(&rest[..start]); + let after = &rest[start + 2..]; + match after.find("}}") { + Some(end) if is_name(&after[..end]) => { + let name = &after[..end]; + match self.0.get(name) { + Some(value) => out.push_str(value), + None => out.push_str(&rest[start..start + 2 + end + 2]), + } + rest = &after[end + 2..]; + } + _ => { + out.push_str("{{"); + rest = after; + } + } + } + out.push_str(rest); + out + } + + /// `text` with every value put back to its `{{name}}`, for a message that is + /// returned or journalled. A value of any length is scrubbed (one character + /// is a value too). + /// + /// A value of four characters or more is replaced wherever it stands. A + /// shorter one only where it stands WHOLE, not inside a longer word or number + /// (`a` in `cabin`, `12` in `2012`): taken out of the middle of a word it + /// would eat the message, and a secret that short is not hidden by being a + /// piece of one anyway. + #[must_use] + pub fn scrub(&self, text: &str) -> String { + let mut out = text.to_string(); + let mut by_length: Vec<(&String, &String)> = + self.0.iter().filter(|(_, v)| !v.is_empty()).collect(); + by_length.sort_by_key(|(_, v)| std::cmp::Reverse(v.len())); + for (name, value) in by_length { + let placeholder = format!("{{{{{name}}}}}"); + out = if value.chars().count() >= WHOLE_ONLY_BELOW { + out.replace(value.as_str(), &placeholder) + } else { + replace_whole(&out, value, &placeholder) + }; + } + out + } +} + +/// Values shorter than this many characters are scrubbed only as whole words. +const WHOLE_ONLY_BELOW: usize = 4; + +/// `text` with each occurrence of `value` that is not part of a longer run of +/// letters and digits replaced by `with`. +fn replace_whole(text: &str, value: &str, with: &str) -> String { + let word = |c: char| c.is_alphanumeric(); + let mut out = String::new(); + let mut rest = text; + while let Some(at) = rest.find(value) { + let (before, after) = (&rest[..at], &rest[at + value.len()..]); + // What stands beside it, across what is already out. + let left_word = before + .chars() + .next_back() + .or_else(|| out.chars().next_back()) + .is_some_and(word); + let right_word = after.chars().next().is_some_and(word); + out.push_str(before); + if left_word && value.chars().next().is_some_and(word) + || right_word && value.chars().next_back().is_some_and(word) + { + out.push_str(value); + } else { + out.push_str(with); + } + rest = after; + } + out.push_str(rest); + out +} + +/// Whether `var` is a name this program will read a parameter from: capitals, +/// digits and `_`, beginning [`ENV_PREFIX`] with something after it, at most 128. +/// +/// The prefix is what keeps a binding to what the person set aside for routines: +/// a parameter can never be bound to `PATH`, `HOME` or an API key of the person's +/// that incu inherited, however the file came to say so. +/// +/// # Errors +/// +/// It is not. +pub fn check_env_name(var: &str) -> Result<(), String> { + let well_formed = var.len() <= 128 + && var + .chars() + .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_'); + if well_formed && var.starts_with(ENV_PREFIX) && var.len() > ENV_PREFIX.len() { + Ok(()) + } else { + Err(format!( + "the variable must be named {ENV_PREFIX} (capitals, digits and _), so that a \ + routine can only read what the person set aside for it, never the rest of incu's \ + environment" + )) + } +} + +/// A digest of a routine's steps: SHA-256 of their JSON, in hex. Stored with a +/// routine's bindings, so a binding is only honoured for the steps it was made +/// for. +/// +/// **Not a boundary.** It is unkeyed and lives in the same file as the steps, so +/// whatever can write the file can write the steps and their digest together. It +/// catches a stale binding and an accidental edit, and nothing more: an agent that +/// can write the user's app-data folder, or run `incu`, is not stopped by it. The +/// boundary is running agents sandboxed, where that folder and the binary are out +/// of reach. No cryptography is added to look like one. +#[must_use] +pub fn digest(steps: &[Step]) -> String { + use sha2::{Digest, Sha256}; + let json = serde_json::to_string(steps).unwrap_or_default(); + Sha256::digest(json.as_bytes()) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +/// Whether a control's name or automation id says it holds a secret. +/// +/// The long words match anywhere (`PasswordBox`, `confirm_password`); the short +/// ones only as whole words, so a PIN field is found and a `Pinterest` button is +/// not. +#[must_use] +pub fn is_secret_name(text: &str) -> bool { + const ANYWHERE: &[&str] = &[ + "password", + "passwd", + "passcode", + "passphrase", + "secret", + "credential", + "apikey", + "api key", + "api_key", + "private key", + "privatekey", + "security code", + "securitycode", + "verification code", + "one-time", + "one time", + "card number", + "cardnumber", + "social security", + ]; + const AS_WORDS: &[&str] = &[ + "pin", "otp", "pwd", "cvv", "cvc", "token", "ssn", "2fa", "mfa", + ]; + let lowered = text.to_lowercase(); + if ANYWHERE.iter().any(|w| lowered.contains(w)) { + return true; + } + lowered + .split(|c: char| !c.is_alphanumeric()) + .any(|w| AS_WORDS.contains(&w)) +} + +/// Whether the control a selector names is a secret field by what the selector +/// says of it: its name or automation id. +#[must_use] +pub fn selector_is_secret(raw: &str) -> bool { + let Ok(selector) = Selector::parse(raw) else { + return false; + }; + let named = match &selector.name { + Some(Text::Exact(n) | Text::Contains(n)) => is_secret_name(n), + None => false, + }; + named + || selector + .automation_id + .as_deref() + .is_some_and(is_secret_name) +} + +/// Whether `text` is a stand-in for text that was removed from a recording: +/// a placeholder (`[REDACTED]`, ``, `****`, `[12 chars]`), or a digest +/// of the value (`sha256:...`, a bare 64-digit hex). +#[must_use] +pub fn looks_redacted(text: &str) -> bool { + let trimmed = text.trim(); + let lowered = trimmed.to_lowercase(); + if lowered.is_empty() { + return false; + } + if lowered.contains("redact") { + return true; + } + if trimmed.chars().count() >= 3 && trimmed.chars().all(|c| "*•●∗·".contains(c)) { + return true; + } + for prefix in ["sha256:", "sha-256:", "sha512:", "sha1:", "md5:", "blake3:"] { + if lowered.starts_with(prefix) { + return true; + } + } + if lowered.len() == 64 && lowered.chars().all(|c| c.is_ascii_hexdigit()) { + return true; + } + let bracketed = [('[', ']'), ('<', '>'), ('(', ')')] + .iter() + .any(|(open, close)| lowered.starts_with(*open) && lowered.ends_with(*close)); + if bracketed { + let inner = lowered[1..lowered.len() - 1].trim(); + const MARKS: &[&str] = &[ + "secret", + "hidden", + "masked", + "removed", + "omitted", + "filtered", + "sensitive", + "private", + "scrubbed", + "censored", + "withheld", + "elided", + "stripped", + "digest", + "sha", + "hash", + "value", + "password", + "token", + ]; + if inner.chars().count() <= 48 && MARKS.iter().any(|m| inner.contains(m)) { + return true; + } + let words: Vec<&str> = inner.split_whitespace().collect(); + if words.len() == 2 + && words[0].chars().all(|c| c.is_ascii_digit()) + && matches!( + words[1], + "char" | "chars" | "character" | "characters" | "bytes" + ) + { + return true; + } + } + false +} + +/// Why a step was made a parameter. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Why { + /// It types into a secure field: the name or automation id of the control the + /// step names says so, or the desktop reports it as a password field. + SecureField, + /// The recording kept a placeholder or a digest in place of the text. + Redacted, + /// Nothing positively says the control is a plain text field: the step names + /// none of its own (it types where focus happens to be: the focus may have + /// moved without a key, to a sign-in dialog, a prompt or the address bar), or + /// names one that is not an edit or combo box, or one that could not be + /// checked on the desktop when the routine was saved. + NotKnownPlain, +} + +/// What the desktop says of the control a selector names, at the moment a routine +/// is saved. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Probe { + /// It is there and is marked as a password or other secure field. + Secret, + /// It is there and is not so marked. + Plain, + /// It could not be checked: the window or control is not on screen (or no + /// desktop is there to ask). Not the same as `Plain`. + Unknown, +} + +/// The roles a control may have for its text to be kept: a text box or a combo +/// box. (The selector vocabulary has no document role; a rich-text area is +/// `edit` or `pane`, and only `edit` is taken.) +fn is_plain_role(role: Option) -> bool { + matches!(role, Some(Role::Edit | Role::ComboBox)) +} + +/// Why a typing step's text must not be kept, if it must not. +/// +/// **The rule is inverted: typed text is kept only when the step itself names a +/// control that is positively not secret.** A step's text is stored literally +/// only when the step carries a selector of its own that +/// +/// 1. has a window part (so it says which window), +/// 2. names a control of role `edit` or `combobox`, +/// 3. has no secret-looking name or automation id ([`is_secret_name`]), and +/// 4. is not marked as a password or secure field by the desktop at save time +/// (`probe`; a control that could not be checked does not pass). +/// +/// Everything else becomes a parameter: no selector of its own, an inherited +/// target (the last click, which may have opened a dialog), a button, a secure +/// control, text a recording redacted. The earlier rule followed focus through +/// the steps and kept the rest; focus moves without a key (a click that opens a +/// sign-in dialog, a first-step prompt, Ctrl+L, Alt+D, F6), and no list of such +/// moves is complete. `None` for a step that is not typing, or already a +/// parameter. +fn why_not_kept(step: &Step, probe: &dyn Fn(&str) -> Probe) -> Option { + let key = text_key(&step.tool)?; + let value = step.args.get(key).cloned().unwrap_or(Value::Null); + match &value { + Value::String(text) if !tokens(text).is_empty() => return None, + Value::String(text) if looks_redacted(text) => return Some(Why::Redacted), + Value::String(_) => {} + Value::Null => return None, + // A recording's `{"redacted": true}` or the like: not text. + _ => return Some(Why::Redacted), + } + let Some(raw) = control_of_any(step) else { + return Some(Why::NotKnownPlain); + }; + let Ok(selector) = Selector::parse(&raw) else { + return Some(Why::NotKnownPlain); + }; + if !selector.has_window_part() { + return Some(Why::NotKnownPlain); + } + if selector_is_secret(&raw) { + return Some(Why::SecureField); + } + if !is_plain_role(selector.role) { + return Some(Why::NotKnownPlain); + } + match probe(&raw) { + Probe::Plain => None, + Probe::Secret => Some(Why::SecureField), + Probe::Unknown => Some(Why::NotKnownPlain), + } +} + +/// The selector a step carries for its own control, whatever the tool. +fn control_of_any(step: &Step) -> Option { + step.args + .get("selector") + .and_then(Value::as_str) + .filter(|raw| Selector::parse(raw).is_ok_and(|s| s.has_element_part())) + .map(str::to_string) +} + +/// Whether `step` puts a secret on screen, as far as the replay can tell without +/// asking the desktop: it types a parameter, or text into a control whose NAME or +/// automation id says it is secret. Used to keep the replay from photographing it. +#[must_use] +pub fn types_a_secret(step: &Step) -> bool { + !names_of(step).is_empty() + || (text_key(&step.tool).is_some() + && control_of_any(step).is_some_and(|raw| selector_is_secret(&raw))) +} + +/// One step made a parameter. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Made { + /// The parameter's name. + pub name: String, + /// The step, 1-based. + pub step: usize, + /// Why. + pub why: Why, +} + +/// Make every typing step whose text is not positively safe to keep ([`why_not_kept`]) +/// a parameter, in place, and say which. `probe` is asked about the raw selector +/// of a control a step names: the desktop's own report, at save time, of whether +/// it is a password field. +pub fn parameterise(steps: &mut [Step], probe: &dyn Fn(&str) -> Probe) -> Vec { + let mut made = Vec::new(); + let mut taken: BTreeSet = names(steps).into_iter().collect(); + for (index, step) in steps.iter_mut().enumerate() { + let (Some(why), Some(key)) = (why_not_kept(step, probe), text_key(&step.tool)) else { + continue; + }; + // Named after the control the step names, when it names one. + let control = control_of_any(step); + let name = fresh_name( + match why { + Why::SecureField => "password", + Why::Redacted | Why::NotKnownPlain => "text", + }, + control.as_deref(), + &mut taken, + ); + if let Some(map) = step.args.as_object_mut() { + map.insert(key.to_string(), Value::String(format!("{{{{{name}}}}}"))); + } + made.push(Made { + name, + step: index + 1, + why, + }); + } + made +} + +/// A parameter name no other has: the control's own name when it makes a good +/// one, else `fallback`, with a number to tell it from the others. +fn fresh_name(fallback: &str, control: Option<&str>, taken: &mut BTreeSet) -> String { + let from_control = control + .and_then(|raw| Selector::parse(raw).ok()) + .and_then(|s| match s.name { + Some(Text::Exact(n) | Text::Contains(n)) => Some(n), + None => None, + }) + .map(|n| { + let mut slug: String = n + .to_lowercase() + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect(); + slug.truncate(24); + slug.trim_matches('_').to_string() + }) + .filter(|s| is_name(s)); + let base = from_control.unwrap_or_else(|| fallback.to_string()); + let mut name = base.clone(); + let mut n = 2; + while !taken.insert(name.clone()) { + name = format!("{base}_{n}"); + n += 1; + } + name +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + /// A desktop with nothing on it: no control can be checked. + fn no_probe(_: &str) -> Probe { + Probe::Unknown + } + + /// A desktop that says every control it is asked about is a plain text box. + fn plain_probe(_: &str) -> Probe { + Probe::Plain + } + + #[test] + fn placeholders_are_found_only_when_they_are_well_formed_names() { + assert_eq!(tokens("a {{password}} b {{pin_2}}"), ["password", "pin_2"]); + assert!(tokens("{{Password}} {{1x}} {{}} {{a b}} {{ x }} {{x").is_empty()); + assert_eq!(tokens("{{{{x}}"), ["x"]); + } + + #[test] + fn a_routines_parameters_are_its_placeholders_once_each_in_order() { + let steps = [ + Step::new("type_text", json!({"text": "{{user}} / {{password}}"})), + Step::new( + "set_value", + json!({"value": "{{password}}", "selector": "x"}), + ), + Step::new("key", json!({"keys": ["enter"]})), + ]; + assert_eq!(names(&steps), ["user", "password"]); + } + + /// A routine that types `{{password}}`, with the bindings the person made. + fn routine_with(steps: Vec, bind: &[(&str, &str)]) -> Routine { + let mut routine = Routine { + phrase: "unlock the vault".to_string(), + steps, + date: String::new(), + last_used_ms: 0, + taught: false, + bindings: BTreeMap::new(), + bound_steps: String::new(), + }; + for (param, var) in bind { + routine + .bindings + .insert((*param).to_string(), (*var).to_string()); + } + if !bind.is_empty() { + routine.bound_steps = digest(&routine.steps); + } + routine + } + + fn typing_password() -> Vec { + vec![Step::new("type_text", json!({"text": "{{password}}"}))] + } + + /// The environment of a test: this variable and no other. + fn secret_env(var: &str) -> Option { + (var == "INCU_PARAM_PASSWORD").then(|| "hunter2-secret".to_string()) + } + + fn no_env(_: &str) -> Option { + None + } + + /// A value, for the tests of what is done with one. + fn values(pairs: &[(&str, &str)]) -> Values { + Values( + pairs + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect(), + ) + } + + #[test] + fn a_run_needs_every_parameter_bound_by_the_person_for_these_steps() { + let bound = routine_with(typing_password(), &[("password", "INCU_PARAM_PASSWORD")]); + // Bound, for these steps, and the variable is set: read from it. + let got = Values::from_routine_with(&bound, &secret_env).unwrap(); + assert!(!got.is_empty()); + assert_eq!(got.fill(&bound.steps[0]).args["text"], "hunter2-secret"); + // Not bound: refused, naming the command and never a value. + let unbound = routine_with(typing_password(), &[]); + let why = Values::from_routine_with(&unbound, &secret_env).unwrap_err(); + assert!( + why.contains("not bound") && why.contains("Nothing was done"), + "{why}" + ); + assert!( + why.contains("incu routine bind \"unlock the vault\" password INCU_PARAM_PASSWORD"), + "{why}" + ); + assert!(!why.contains("hunter2"), "{why}"); + // Some bound, some not: the unbound ones are named. + let two = vec![Step::new( + "type_text", + json!({"text": "{{user}} {{password}}"}), + )]; + let half = routine_with(two, &[("password", "INCU_PARAM_PASSWORD")]); + let why = Values::from_routine_with(&half, &secret_env).unwrap_err(); + assert!(why.contains("user") && !why.contains("password,"), "{why}"); + // The variable is not set, or empty, or too long: said by name. + let why = Values::from_routine_with(&bound, &no_env).unwrap_err(); + assert!( + why.contains("INCU_PARAM_PASSWORD") && why.contains("not set"), + "{why}" + ); + let empty = |_: &str| Some(String::new()); + assert!(Values::from_routine_with(&bound, &empty).is_err()); + let long = |_: &str| Some("x".repeat(MAX_PARAM_CHARS + 1)); + assert!(Values::from_routine_with(&bound, &long).is_err()); + // A routine with none needs none. + let plain = routine_with(vec![Step::new("key", json!({"keys": ["enter"]}))], &[]); + assert!(Values::from_routine_with(&plain, &no_env) + .unwrap() + .is_empty()); + } + + /// The steps changed since the person bound: a binding is for steps it was + /// made for, and is not honoured for others (a hand-edited file). + #[test] + fn a_routine_whose_steps_changed_since_they_were_bound_is_refused() { + let mut routine = routine_with(typing_password(), &[("password", "INCU_PARAM_PASSWORD")]); + assert!(Values::from_routine_with(&routine, &secret_env).is_ok()); + routine + .steps + .push(Step::new("key", json!({"keys": ["enter"]}))); + let why = Values::from_routine_with(&routine, &secret_env).unwrap_err(); + assert!( + why.contains("not the ones its parameters were bound for"), + "{why}" + ); + assert!(why.contains("incu routine bind"), "{why}"); + // The digest is of the steps and nothing else: same steps, same digest. + assert_eq!(digest(&typing_password()), digest(&typing_password())); + assert_ne!(digest(&typing_password()), digest(&routine.steps)); + assert_eq!(digest(&typing_password()).len(), 64); + } + + /// Only a variable the person set aside for routines can be bound: the prefix + /// is the line, in the command and again when the file is read. + #[test] + fn a_binding_can_name_only_an_incu_param_variable() { + for good in [ + "INCU_PARAM_PASSWORD", + "INCU_PARAM_A", + "INCU_PARAM_VAULT_PIN_2", + ] { + assert!(check_env_name(good).is_ok(), "{good}"); + } + for bad in [ + "HOME", + "PATH", + "ANTHROPIC_API_KEY", + "INCU_PARAM_", + "incu_param_password", + "INCU_PARAM_A B", + "INCU_PARAM_A=B", + "XINCU_PARAM_A", + "", + ] { + assert!(check_env_name(bad).is_err(), "{bad:?}"); + } + // A file edited to bind a parameter to something else is refused at run + // time, before the variable is read. + let mut edited = routine_with(typing_password(), &[("password", "HOME")]); + edited.bound_steps = digest(&edited.steps); + let read = std::cell::Cell::new(false); + let spy = |_: &str| { + read.set(true); + Some("leaked".to_string()) + }; + let why = Values::from_routine_with(&edited, &spy).unwrap_err(); + assert!( + why.contains("INCU_PARAM_") && !why.contains("leaked"), + "{why}" + ); + assert!(!read.get(), "the variable was read"); + } + + #[test] + fn a_routine_may_take_no_more_than_the_most_parameters() { + let steps: Vec = (0..=MAX_PARAMS) + .map(|i| Step::new("type_text", json!({"text": format!("{{{{p{i}}}}}")}))) + .collect(); + assert_eq!(names(&steps).len(), MAX_PARAMS + 1); + let bound: Vec<(String, String)> = (0..=MAX_PARAMS) + .map(|i| (format!("p{i}"), format!("INCU_PARAM_P{i}"))) + .collect(); + let pairs: Vec<(&str, &str)> = bound + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + .collect(); + let routine = routine_with(steps, &pairs); + let why = Values::from_routine_with(&routine, &|_| Some("x".to_string())).unwrap_err(); + assert!(why.contains("more than the 16"), "{why}"); + } + + #[test] + fn a_value_is_substituted_in_one_pass_and_scrubbed_from_what_comes_back() { + let step = Step::new("type_text", json!({"text": "pw={{password}}!"})); + let tricky = values(&[("password", "a{{password}}b")]); + let filled = tricky.fill(&step); + assert_eq!(filled.args["text"], "pw=a{{password}}b!"); + // The stored step is untouched. + assert_eq!(step.args["text"], "pw={{password}}!"); + let message = "could not type hunter2-secret into the window"; + assert_eq!( + values(&[("password", "hunter2-secret")]).scrub(message), + "could not type {{password}} into the window" + ); + } + + /// A value of any length is scrubbed; a short one only where it stands whole. + #[test] + fn a_short_value_is_scrubbed_whole_and_never_out_of_the_middle_of_a_word() { + let with = |value: &'static str| values(&[("password", value)]); + // One character. + assert_eq!( + with("x").scrub("typed x into box"), + "typed {{password}} into box" + ); + assert_eq!( + with("x").scrub("the box"), + "the box", + "x inside a word stays" + ); + assert_eq!(with("x").scrub("x"), "{{password}}"); + assert_eq!( + with("x").scrub("x,x;(x)"), + "{{password}},{{password}};({{password}})" + ); + // Two and three. + assert_eq!(with("ab").scrub("a cab ab"), "a cab {{password}}"); + assert_eq!(with("12").scrub("in 2012: 12"), "in 2012: {{password}}"); + assert_eq!( + with("pin").scrub("the pin, not a spinner"), + "the {{password}}, not a spinner" + ); + // Four and more: anywhere, because the whole secret is the evidence. + assert_eq!( + with("hunt").scrub("it was hunter2 and hunt"), + "it was {{password}}er2 and {{password}}" + ); + // Longer values go first, so a short one inside a long one cannot split it. + let both = values(&[("a", "secret"), ("b", "secret-longer")]); + assert_eq!(both.scrub("secret-longer and secret"), "{{b}} and {{a}}"); + } + + #[test] + fn a_field_whose_name_says_secret_is_secret_and_a_lookalike_is_not() { + for yes in [ + "Password", + "Confirm password", + "PasswordBox", + "new_passcode", + "PIN", + "Enter your PIN", + "OTP code", + "API key", + "Security code", + "Token", + "CVV", + ] { + assert!(is_secret_name(yes), "{yes}"); + } + for no in [ + "Username", + "Search", + "Pinterest", + "Spinner", + "Subject", + "Tokyo", + "Body", + ] { + assert!(!is_secret_name(no), "{no}"); + } + assert!(selector_is_secret( + "app=login;window=Sign in;role=edit;name=Password" + )); + assert!(selector_is_secret( + "app=login;window=Sign in;role=edit;aid=txtPassword" + )); + assert!(!selector_is_secret( + "app=login;window=Sign in;role=edit;name=Email" + )); + assert!(!selector_is_secret("not a selector")); + } + + #[test] + fn what_a_recording_keeps_in_place_of_text_is_recognised() { + for yes in [ + "[REDACTED]", + "", + "***", + "••••••••", + "[secret]", + "", + "[12 chars]", + "<8 characters>", + "sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + "[value removed]", + "(redacted by watchman)", + ] { + assert!(looks_redacted(yes), "{yes}"); + } + for no in [ + "hello", + "", + "ab", + "two words", + "[ ] todo", + "bold", + "(optional)", + "12 characters long", + "deadbeef", + ] { + assert!(!looks_redacted(no), "{no}"); + } + } + + /// A typing step that names a plain text box of its own, in a window, that + /// the desktop confirms is not a password field: the only text that is kept. + fn typing_in(selector: &str, text: &str) -> Step { + Step::new( + "type_text", + json!({"text": text, "in_window": "app=browser;window~=Home", "selector": selector}), + ) + } + + const SEARCH_BOX: &str = "app=browser;window~=Home;role=edit;name=Search"; + + /// The inverted rule: a plain named search box keeps its literal text. + #[test] + fn a_plain_named_search_box_keeps_its_literal_text() { + let mut steps = vec![ + typing_in(SEARCH_BOX, "rust lifetimes"), + Step::new( + "set_value", + json!({"selector": "app=browser;window~=Home;role=combobox;name=Language", "value": "English"}), + ), + ]; + let made = parameterise(&mut steps, &plain_probe); + assert!(made.is_empty(), "{made:?}"); + assert_eq!(steps[0].args["text"], "rust lifetimes"); + assert_eq!(steps[1].args["value"], "English"); + assert!(names(&steps).is_empty()); + } + + /// Everything that does not positively say "plain text box" is a parameter. + #[test] + fn a_text_is_kept_only_for_a_control_positively_known_to_be_a_plain_text_box() { + let in_window = "app=browser;window~=Home"; + type Case<'a> = (&'a str, Value, &'a dyn Fn(&str) -> Probe, Why); + let cases: &[Case] = &[ + // The desktop could not check it (the window is gone, or no desktop). + ( + "unchecked", + typing_in(SEARCH_BOX, "x").args, + &no_probe, + Why::NotKnownPlain, + ), + // The desktop says it is a password field, whatever its name. + ( + "marked secure", + typing_in(SEARCH_BOX, "x").args, + &|_| Probe::Secret, + Why::SecureField, + ), + // A name that says secret. + ( + "secret name", + typing_in(&format!("{in_window};role=edit;name=Password"), "x").args, + &plain_probe, + Why::SecureField, + ), + ( + "secret automation id", + typing_in(&format!("{in_window};role=edit;aid=txtPassword"), "x").args, + &plain_probe, + Why::SecureField, + ), + // Not a text box. + ( + "a button", + typing_in(&format!("{in_window};role=button;name=Sign in"), "x").args, + &plain_probe, + Why::NotKnownPlain, + ), + ( + "a text label", + typing_in(&format!("{in_window};role=text;name=Hello"), "x").args, + &plain_probe, + Why::NotKnownPlain, + ), + ( + "no role", + typing_in(&format!("{in_window};name=Search"), "x").args, + &plain_probe, + Why::NotKnownPlain, + ), + // Does not say which window. + ( + "no window part", + typing_in("role=edit;name=Search", "x").args, + &plain_probe, + Why::NotKnownPlain, + ), + // No selector of its own at all: it types where focus is. + ( + "no selector", + json!({"text": "x", "in_window": in_window}), + &plain_probe, + Why::NotKnownPlain, + ), + ]; + for (what, args, probe, why) in cases { + let mut steps = vec![Step::new("type_text", args.clone())]; + let made = parameterise(&mut steps, probe); + assert_eq!(made.len(), 1, "{what}: kept as text"); + assert_eq!(made[0].why, *why, "{what}"); + assert!( + steps[0].args["text"].as_str().unwrap().starts_with("{{"), + "{what}" + ); + } + } + + /// A click that opens a sign-in dialog: the password goes into a field no + /// step named, and the focus moved without any key. + #[test] + fn a_password_typed_after_a_click_that_opened_a_sign_in_dialog_is_a_parameter() { + let mut steps = vec![ + Step::new( + "click_element", + json!({"selector": "app=editor;window=Doc;role=button;name=Sign in"}), + ), + Step::new( + "type_text", + json!({"text": "hunter2", "in_window": "app=editor;window~=Sign in"}), + ), + ]; + // Even a desktop that calls every control plain does not save it: the + // step does not name the field. + let made = parameterise(&mut steps, &plain_probe); + assert_eq!(made.len(), 1); + assert!(!serde_json::to_string(&steps).unwrap().contains("hunter2")); + } + + /// A first-step prompt: nothing has named a control yet (sudo, KeePass). + #[test] + fn a_secret_typed_as_the_first_step_is_a_parameter() { + let mut steps = vec![ + Step::new( + "type_text", + json!({"text": "correct horse", "in_window": "app=terminal;window~=sudo"}), + ), + Step::new( + "key", + json!({"keys": ["enter"], "in_window": "app=terminal"}), + ), + ]; + let made = parameterise(&mut steps, &plain_probe); + assert_eq!(made.len(), 1); + assert_eq!(made[0].step, 1); + assert!(!serde_json::to_string(&steps).unwrap().contains("horse")); + } + + /// Focus moves with no key at all, or with one no list of keys would hold: + /// Ctrl+L, Alt+D, F6 put it in the address bar. + #[test] + fn text_typed_after_ctrl_l_or_alt_d_or_f6_is_a_parameter() { + let window = "app=browser;window~=Home"; + for keys in [json!(["ctrl", "l"]), json!(["alt", "d"]), json!(["f6"])] { + let mut steps = vec![ + Step::new( + "click_element", + json!({"selector": format!("{window};role=edit;name=Search")}), + ), + Step::new("key", json!({"keys": keys, "in_window": window})), + Step::new( + "type_text", + json!({"text": "secret-url", "in_window": window}), + ), + ]; + let made = parameterise(&mut steps, &plain_probe); + assert_eq!(made.len(), 1, "{keys}"); + assert_eq!(made[0].step, 3); + } + } + + /// The login form, now: every text is a parameter, the user name too, because + /// no step names a field. + #[test] + fn the_login_form_with_a_tab_keeps_no_text() { + let window = "app=login;window=Sign in"; + let mut steps = vec![ + Step::new( + "click_element", + json!({"selector": format!("{window};role=edit;name=Username")}), + ), + Step::new("type_text", json!({"text": "alice", "in_window": window})), + Step::new("key", json!({"keys": ["Tab"], "in_window": window})), + Step::new("type_text", json!({"text": "hunter2", "in_window": window})), + Step::new("key", json!({"keys": ["Enter"], "in_window": window})), + ]; + let made = parameterise(&mut steps, &plain_probe); + assert_eq!(made.len(), 2, "{made:?}"); + let kept = serde_json::to_string(&steps).unwrap(); + assert!( + !kept.contains("hunter2") && !kept.contains("alice"), + "{kept}" + ); + // The two get two names. + assert_eq!(names(&steps), ["text", "text_2"]); + } + + #[test] + fn typing_into_a_password_field_becomes_a_parameter_named_after_it() { + let mut steps = vec![ + typing_in( + "app=login;window~=Home;role=edit;name=Email", + "me@example.com", + ), + typing_in( + "app=login;window~=Home;role=edit;name=Password", + "hunter2-secret", + ), + ]; + let made = parameterise(&mut steps, &plain_probe); + assert_eq!( + made, + [Made { + name: "password".to_string(), + step: 2, + why: Why::SecureField + }] + ); + assert_eq!(steps[1].args["text"], "{{password}}"); + // The email goes into a named plain field: it stays. + assert_eq!(steps[0].args["text"], "me@example.com"); + assert!(!serde_json::to_string(&steps).unwrap().contains("hunter2")); + } + + #[test] + fn a_set_value_follows_the_same_rule_through_its_own_selector() { + let mut steps = vec![ + Step::new( + "set_value", + json!({"selector": "app=vault;window=V;role=edit;name=Token", "value": "hunter2-secret"}), + ), + // No window part: not kept. + Step::new( + "set_value", + json!({"selector": "role=edit;name=Title", "value": "a title"}), + ), + Step::new( + "set_value", + json!({"selector": "app=vault;window=V;role=edit;name=Title", "value": "a title"}), + ), + ]; + let made = parameterise(&mut steps, &plain_probe); + assert_eq!(made[0].name, "token"); + assert_eq!(steps[0].args["value"], "{{token}}"); + assert_eq!(made.len(), 2, "{made:?}"); + assert_eq!(made[1].step, 2); + assert_eq!(steps[2].args["value"], "a title"); + } + + #[test] + fn the_desktops_own_report_of_a_password_field_counts_when_the_name_does_not_say() { + let mut steps = vec![Step::new( + "set_value", + json!({"selector": "app=bank;window=Login;role=edit;name=Enter it here", "value": "s3cret"}), + )]; + let seen_as_password = |raw: &str| { + if raw.contains("Enter it here") { + Probe::Secret + } else { + Probe::Plain + } + }; + let made = parameterise(&mut steps, &seen_as_password); + assert_eq!(made.len(), 1); + assert_eq!(made[0].why, Why::SecureField); + assert!(!steps[0].args.to_string().contains("s3cret")); + } + + #[test] + fn the_replay_can_tell_a_step_that_puts_a_secret_on_screen() { + let window = "app=login;window=Sign in"; + // A parameter is a secret, whatever is around it. + let param = Step::new( + "type_text", + json!({"text": "{{text}}", "in_window": window}), + ); + assert!(types_a_secret(¶m)); + // Text into a field whose name says secret. + assert!(types_a_secret(&typing_in( + &format!("{window};role=edit;name=Password"), + "plain" + ))); + // Plain text into a plain field, and steps that type nothing. + assert!(!types_a_secret(&typing_in(SEARCH_BOX, "plain"))); + assert!(!types_a_secret(&Step::new("key", json!({"keys": ["tab"]})))); + assert!(!types_a_secret(&Step::new( + "click_element", + json!({"selector": format!("{window};role=edit;name=Password")}) + ))); + } + + #[test] + fn a_redacted_recording_text_is_a_parameter_not_a_replayed_placeholder() { + let mut steps = vec![ + Step::new( + "set_value", + json!({"selector": "app=mail;window~=Compose;role=edit;name=Body", "value": "[REDACTED]"}), + ), + Step::new( + "type_text", + json!({"text": {"redacted": true, "sha256": "ab"}, "in_window": "app=mail"}), + ), + typing_in("app=mail;window~=Compose;role=edit;name=Subject", "hello"), + ]; + let made = parameterise(&mut steps, &plain_probe); + assert_eq!(made.len(), 2); + assert!(made.iter().all(|m| m.why == Why::Redacted)); + assert_eq!(steps[0].args["value"], "{{body}}"); + // No control of its own: named plainly, numbered to stay apart. + assert_eq!(steps[1].args["text"], "{{text}}"); + assert_eq!( + steps[2].args["text"], "hello", + "ordinary text in a named plain field stays" + ); + assert_eq!(names(&steps), ["body", "text"]); + } + + #[test] + fn two_secrets_get_two_names_and_an_agents_own_placeholder_is_left_alone() { + let mut steps = vec![ + Step::new( + "set_value", + json!({"selector": "app=a;window=W;role=edit;name=Password", "value": "one"}), + ), + Step::new( + "set_value", + json!({"selector": "app=a;window=W;role=edit;name=Password", "value": "two"}), + ), + Step::new( + "type_text", + json!({"text": "{{mine}}", "in_window": "app=a"}), + ), + ]; + let made = parameterise(&mut steps, &plain_probe); + assert_eq!(made[0].name, "password"); + assert_eq!(made[1].name, "password_2"); + assert_eq!(made.len(), 2, "{{mine}} is the agent's, not made"); + assert_eq!(names(&steps), ["password", "password_2", "mine"]); + } +} diff --git a/crates/desktop/src/routine/pictures.rs b/crates/desktop/src/routine/pictures.rs new file mode 100644 index 0000000..3fab779 --- /dev/null +++ b/crates/desktop/src/routine/pictures.rs @@ -0,0 +1,626 @@ +//! The pictures of the last replay: one before and one after each step. +//! +//! # What and why +//! +//! A replay that fails is only useful if the agent that redoes the task, and the +//! person reading over its shoulder, can see what the screen looked like. So +//! each step of `routine_run` gets one picture before it and one after it, taken +//! through the same gate as `screenshot` (the window the step is for, else the +//! screen), and a failure names the failed step's two pictures by path. There is +//! no video and no ffmpeg: a step is a still. +//! +//! # One folder for each run +//! +//! Two replays can be going at once (two servers share the routines file). Each +//! run keeps its pictures in a folder of its own beside the routines file, +//! `last-replay/run-