Keep ordered capture paste attached to the original rich editor

docs/capture-tool.md

@@ -50,11 +50,15 @@ automatically. Known protected fields are excluded. Destination accessibility
 checks inspect identity and state, not input text. The receiving app controls
 whether it accepts images and where attachments appear.
 
-On Windows, attaching an image can briefly move focus to the containing page.
-Capture waits up to 1.5 seconds for the original input to regain focus, without
-moving the caret or sending the image again. A different input, window or replaced
-clipboard still stops the sequence. Presses while pasting do not queue another
-batch; briefly held shortcut keys pause the current sequence until released.
+On Windows, attaching an image can briefly move focus to the page or an upload
+control. Capture waits up to 1.5 seconds for the original editor to regain focus,
+using fresh accessibility state after each paste. A rich editor rebuilt in place
+can continue only when it uniquely replaces the original editor in the same
+container. Capture never moves the caret or sends an accepted image again.
+A different input, container, window or replaced clipboard still stops the
+sequence. Typing, navigating, clicking or scrolling during paste also stops it.
+Presses while pasting do not queue another batch; briefly held shortcut
+keys pause the current sequence until released.
 
 ## Menu and clipboard
 

src/Focalet.CaptureTool/CaptureInputIdentity.cs

@@ -0,0 +1,89 @@
+using FlaUI.Core.AutomationElements;
+using FlaUI.Core.Definitions;
+using FlaUI.UIA3;
+
+namespace Focalet.CaptureTool;
+
+/// <summary>Per-paste editor identity, without retaining a stale UIA provider or reading its text.</summary>
+internal sealed class CaptureInputIdentity
+{
+    private int[] runtimeId;
+    private readonly Replacement? replacement;
+
+    private CaptureInputIdentity(int[] runtimeId, Replacement? replacement)
+        => (this.runtimeId, this.replacement) = (runtimeId, replacement);
+
+    internal static AutomationElement Editor(UIA3Automation automation, AutomationElement focused)
+    {
+        var walker = automation.TreeWalkerFactory.GetRawViewWalker();
+        var depth = 0;
+        for (var node = focused; node is not null; node = walker.GetParent(node))
+        {
+            if (node.Properties.ControlType.ValueOrDefault == ControlType.Edit) return node;
+            if (node.Properties.ControlType.ValueOrDefault is ControlType.Document or ControlType.Window) break;
+            // Do not walk outside the focused control's bounded local ancestry.
+            if (++depth >= 8) break;
+        }
+        return focused;
+    }
+
+    internal static CaptureInputIdentity Remember(UIA3Automation automation, AutomationElement editor)
+    {
+        var id = Id(editor);
+        Replacement? replacement = null;
+        var shape = Shape.Read(editor);
+        if (shape.Type == ControlType.Edit)
+        {
+            var parent = automation.TreeWalkerFactory.GetRawViewWalker().GetParent(editor);
+            if (parent is not null && parent.Properties.ControlType.ValueOrDefault is not (ControlType.Document or ControlType.Window))
+            {
+                var children = Children(automation, parent);
+                if (children is not null && children.Count(child => Shape.Read(child) == shape) == 1)
+                    replacement = new(Id(parent), shape, children.Select(Id).ToArray());
+            }
+        }
+        return new(id, replacement);
+    }
+
+    internal bool Matches(UIA3Automation automation, AutomationElement editor)
+    {
+        var id = Id(editor);
+        if (Same(runtimeId, id)) return true;
+        if (replacement is not { } previous || previous.Shape != Shape.Read(editor) ||
+            previous.Siblings.Any(sibling => Same(sibling, id))) return false;
+        var parent = automation.TreeWalkerFactory.GetRawViewWalker().GetParent(editor);
+        if (parent is null || !Same(previous.ParentId, Id(parent))) return false;
+        var children = Children(automation, parent);
+        // A new provider may represent a remounted editor. Accept only the unique
+        // replacement of our removed editor in its original, still-live container.
+        // A pre-existing sibling, a second matching editor or a new container is
+        // never treated as the selected input, even if its labels look identical.
+        if (children is null || children.Any(child => Same(runtimeId, Id(child))) ||
+            children.Count(child => previous.Shape == Shape.Read(child)) != 1 ||
+            !children.Any(child => Same(id, Id(child)))) return false;
+        runtimeId = id;
+        return true;
+    }
+
+    private static List<AutomationElement>? Children(UIA3Automation automation, AutomationElement parent)
+    {
+        var result = new List<AutomationElement>();
+        var walker = automation.TreeWalkerFactory.GetRawViewWalker();
+        for (var child = walker.GetFirstChild(parent); child is not null; child = walker.GetNextSibling(child))
+        {
+            if (result.Count == 64) return null;
+            result.Add(child);
+        }
+        return result;
+    }
+
+    private static int[] Id(AutomationElement element) => element.Properties.RuntimeId.Value;
+    private static bool Same(int[] first, int[] second) => first.Length > 0 && first.SequenceEqual(second);
+    private sealed record Replacement(int[] ParentId, Shape Shape, int[][] Siblings);
+    private sealed record Shape(ControlType Type, string AutomationId, string ClassName, string Framework)
+    {
+        internal static Shape Read(AutomationElement element) => new(element.Properties.ControlType.ValueOrDefault,
+            element.Properties.AutomationId.ValueOrDefault ?? "", element.Properties.ClassName.ValueOrDefault ?? "",
+            element.Properties.FrameworkId.ValueOrDefault ?? "");
+    }
+}

src/Focalet.CaptureTool/CapturePasteActivity.cs

@@ -0,0 +1,64 @@
+using System.ComponentModel;
+using System.Runtime.InteropServices;
+
+namespace Focalet.CaptureTool;
+
+/// <summary>Stop an active paste when the user types or navigates, even if an editor is rebuilt in place.</summary>
+internal sealed class CapturePasteActivity : IDisposable
+{
+    internal const nuint PasteTag = 0x46434C54;
+    private readonly Hook keyboardCallback;
+    private readonly Hook mouseCallback;
+    private nint keyboard;
+    private nint mouse;
+    internal bool Changed { get; private set; }
+
+    internal CapturePasteActivity()
+    {
+        keyboardCallback = Keyboard;
+        mouseCallback = Mouse;
+        var module = GetModuleHandle(null);
+        keyboard = SetWindowsHookEx(13, keyboardCallback, module, 0);
+        mouse = SetWindowsHookEx(14, mouseCallback, module, 0);
+        if (keyboard == 0 || mouse == 0)
+        {
+            var error = Marshal.GetLastWin32Error();
+            Dispose();
+            throw new Win32Exception(error, "Could not watch destination input changes.");
+        }
+    }
+
+    private nint Keyboard(int code, nint message, nint data)
+    {
+        if (code >= 0 && message is 0x0100 or 0x0104)
+        {
+            var key = Marshal.PtrToStructure<KeyboardEvent>(data);
+            // Ignore only our own injected paste, modifier holds and busy Alt+A.
+            // Other typing/navigation cancels, including input from another tool.
+            if (key.Extra != PasteTag && key.Key is not (0x10 or 0x11 or 0x12 or >= 0xA0 and <= 0xA5) &&
+                !(key.Key == 0x41 && (GetAsyncKeyState(0x12) & 0x8000) != 0)) Changed = true;
+        }
+        return CallNextHookEx(0, code, message, data);
+    }
+
+    private nint Mouse(int code, nint message, nint data)
+    {
+        // Moving the pointer is harmless; clicks and scrolling express new intent.
+        if (code >= 0 && message is 0x0201 or 0x0204 or 0x0207 or 0x020A or 0x020B or 0x020E) Changed = true;
+        return CallNextHookEx(0, code, message, data);
+    }
+
+    public void Dispose()
+    {
+        if (keyboard != 0) { UnhookWindowsHookEx(keyboard); keyboard = 0; }
+        if (mouse != 0) { UnhookWindowsHookEx(mouse); mouse = 0; }
+    }
+
+    private delegate nint Hook(int code, nint message, nint data);
+    [StructLayout(LayoutKind.Sequential)] private struct KeyboardEvent { public uint Key, Scan, Flags, Time; public nuint Extra; }
+    [DllImport("user32.dll", SetLastError = true)] private static extern nint SetWindowsHookEx(int kind, Hook callback, nint module, uint thread);
+    [DllImport("user32.dll")] private static extern bool UnhookWindowsHookEx(nint hook);
+    [DllImport("user32.dll")] private static extern nint CallNextHookEx(nint hook, int code, nint message, nint data);
+    [DllImport("user32.dll")] private static extern short GetAsyncKeyState(int key);
+    [DllImport("kernel32.dll", CharSet = CharSet.Unicode)] private static extern nint GetModuleHandle(string? name);
+}

src/Focalet.CaptureTool/CapturePasteSequence.cs

@@ -15,18 +15,19 @@ internal static class CapturePasteSequence
     public static async Task<CapturePasteResult> PasteAsync(CaptureClipboardBatch batch, CapturePasteTarget target, bool textOnly, bool slowerImages = false)
     {
         using var clipboard = new PasteClipboard();
+        using var activity = new CapturePasteActivity();
         var sent = 0;
         var unreadImages = 0;
         for (var index = 0; index < batch.Items.Count; index++)
         {
             foreach (var image in textOnly ? new[] { false } : new[] { true, false })
             {
-                if (await WaitForInputAsync(target, clipboard, sent > 0) is { } notReady)
+                if (await WaitForInputAsync(target, clipboard, activity, sent > 0) is { } notReady)
                     return new(sent, unreadImages, notReady);
                 if (image) clipboard.SetImage(batch.Items[index]);
                 else clipboard.SetText(batch.TextParts[index]);
                 clipboard.Arm();
-                if (await WaitForInputAsync(target, clipboard, true) is { } interrupted)
+                if (await WaitForInputAsync(target, clipboard, activity, true) is { } interrupted)
                     return new(sent, unreadImages, interrupted);
                 if (!target.Paste(CapturePasteTarget.GetClipboardSequenceNumber(), clipboard.Handle))
                     return new(sent, unreadImages, "Paste could not be dispatched. It will not be retried.");
@@ -44,6 +45,7 @@ public static async Task<CapturePasteResult> PasteAsync(CaptureClipboardBatch ba
                 while (wait.ElapsedMilliseconds < (image ? 5000 : 3000))
                 {
                     await Task.Delay(50);
+                    if (activity.Changed) return new(sent, unreadImages, "Keyboard or mouse input interrupted paste.");
                     if (!clipboard.OwnsClipboard)
                         return new(sent, unreadImages, "Another application replaced the clipboard.");
                     if (!target.IsCurrent()) return new(sent, unreadImages, "The destination window changed during paste.");
@@ -69,14 +71,16 @@ public static async Task<CapturePasteResult> PasteAsync(CaptureClipboardBatch ba
         return new(sent, unreadImages);
     }
 
-    private static async Task<string?> WaitForInputAsync(CapturePasteTarget target, PasteClipboard clipboard, bool checkClipboard)
+    private static async Task<string?> WaitForInputAsync(CapturePasteTarget target, PasteClipboard clipboard, CapturePasteActivity activity, bool checkClipboard)
     {
         var wait = Stopwatch.StartNew();
         while (true)
         {
+            if (activity.Changed) return "Keyboard or mouse input interrupted paste.";
             if (!target.IsCurrent()) return "The destination window changed during paste.";
             if (checkClipboard && !clipboard.OwnsClipboard) return "Another application replaced the clipboard.";
             var state = await target.InputStateAsync();
+            if (activity.Changed) return "Keyboard or mouse input interrupted paste.";
             if (!target.IsCurrent()) return "The destination window changed during paste.";
             if (checkClipboard && !clipboard.OwnsClipboard) return "Another application replaced the clipboard.";
             if (state == CaptureInputState.Changed) return "The focused input changed during paste.";

src/Focalet.CaptureTool/CapturePasteTarget.cs

@@ -16,7 +16,7 @@ internal sealed record CaptureInputObservation(CapturePasteTarget Target, Captur
 internal sealed record CapturePasteTarget(nint Window, nint Focus, uint ProcessId, long ProcessStarted)
     : WindowFocus(Window, Focus, ProcessId, ProcessStarted)
 {
-    private AutomationElement? InputElement { get; init; }
+    private CaptureInputIdentity? InputIdentity { get; init; }
     private UIA3Automation? InputAutomation { get; init; }
     public string Description => NativeCaptureWindow.Title(Window);
     public new static CapturePasteTarget? Remember()
@@ -57,7 +57,11 @@ internal static CaptureInputObservation ObserveInput(UIA3Automation automation,
         if (!target.IsCurrent() || !element.Properties.HasKeyboardFocus.ValueOrDefault) return Unknown();
         // The user already chose the caret. Keep identity for interruption checks,
         // without selecting UIA output ranges or moving focus.
-        return new(target with { InputElement = element, InputAutomation = automation }, CaptureInputKind.Input);
+        var editor = CaptureInputIdentity.Editor(automation, element);
+        if (editor.Properties.IsPassword.ValueOrDefault || !editor.Properties.IsEnabled.ValueOrDefault)
+            return new(target, CaptureInputKind.Protected);
+        var identity = CaptureInputIdentity.Remember(automation, editor);
+        return new(target with { InputIdentity = identity, InputAutomation = automation }, CaptureInputKind.Input);
     }
 
     internal static bool IsTerminalControl(string windowClass, string? inputClass) =>
@@ -78,32 +82,37 @@ public async Task<bool> IsInputCurrentAsync()
     internal async Task<CaptureInputState> InputStateAsync()
     {
         if (!IsCurrent()) return CaptureInputState.Changed;
-        if (InputElement is null) return CaptureInputState.Current;
+        if (InputIdentity is null) return CaptureInputState.Current;
         return await Task.Run(() =>
         {
             try
             {
-                if (InputElement.Properties.HasKeyboardFocus.ValueOrDefault)
-                    return IsCurrent() ? CaptureInputState.Current : CaptureInputState.Changed;
                 var focused = InputAutomation?.FocusedElement();
                 if (!IsCurrent()) return CaptureInputState.Changed;
                 if (focused is null) return CaptureInputState.Settling;
                 var process = focused.Properties.ProcessId.ValueOrDefault;
                 if (process != ProcessId && (Focus == 0 || process != WindowProcess(Focus))) return CaptureInputState.Changed;
                 var window = focused.Properties.NativeWindowHandle.ValueOrDefault;
                 if (window != 0 && !ContainsWindow(window)) return CaptureInputState.Changed;
-                if (InputAutomation?.Compare(InputElement, focused) == true) return CaptureInputState.Settling;
-                if (focused.Properties.IsPassword.ValueOrDefault ||
-                    focused.Patterns.Value.PatternOrDefault?.IsReadOnly.ValueOrDefault == false)
+                if (!focused.Properties.HasKeyboardFocus.ValueOrDefault) return CaptureInputState.Settling;
+                var editor = CaptureInputIdentity.Editor(InputAutomation!, focused);
+                if (editor.Properties.IsPassword.ValueOrDefault || !editor.Properties.IsEnabled.ValueOrDefault)
+                    return CaptureInputState.Changed;
+                if (editor.Properties.ControlType.ValueOrDefault == ControlType.Edit &&
+                    editor.Patterns.Value.PatternOrDefault?.IsReadOnly.ValueOrDefault == true)
+                    return CaptureInputState.Settling;
+                if (InputIdentity.Matches(InputAutomation!, editor))
+                    return IsCurrent() ? CaptureInputState.Current : CaptureInputState.Changed;
+                if (editor.Properties.ControlType.ValueOrDefault == ControlType.Edit ||
+                    editor.Patterns.Value.PatternOrDefault?.IsReadOnly.ValueOrDefault == false)
                     return CaptureInputState.Changed;
-                // An attachment can briefly move accessibility focus to the
-                // containing page while the SAME editor is being updated.
-                // Never bind to another editor or move the user's caret.
-                var type = focused.Properties.ControlType.ValueOrDefault;
-                return type == ControlType.Document || type == ControlType.Pane || type == ControlType.Window
-                    ? CaptureInputState.Settling : CaptureInputState.Changed;
+                // Upload controls can temporarily take focus. Wait for the
+                // original editor; never paste into the temporary control.
+                return CaptureInputState.Settling;
             }
-            catch (Exception error) when (error is not OutOfMemoryException) { return CaptureInputState.Changed; }
+            // UIA providers can be temporarily unavailable while a rich editor
+            // updates. An unreadable provider is not evidence of another input.
+            catch (Exception error) when (error is not OutOfMemoryException) { return CaptureInputState.Settling; }
         });
     }
 
@@ -131,7 +140,7 @@ public bool Paste(uint clipboardSequence, nint clipboardOwner = 0)
 
     private static Input Key(ushort value, bool up = false) => new()
     {
-        Type = 1, Data = new InputData { Keyboard = new KeyboardInput { Key = value, Flags = up ? 2u : 0u } },
+        Type = 1, Data = new InputData { Keyboard = new KeyboardInput { Key = value, Flags = up ? 2u : 0u, Extra = CapturePasteActivity.PasteTag } },
     };
 
     [StructLayout(LayoutKind.Sequential)]

src/Focalet.CaptureTool/CapturePasteTool.cs

@@ -148,7 +148,6 @@ private async void Paste()
                 if (observation.Kind == CaptureInputKind.Protected)
                 { Notify("Paste cancelled", "The selected input is protected or read-only."); return; }
                 target = observation.Target;
-                if (!await target.IsInputCurrentAsync()) return;
                 // Retain the capture for another deliberate Alt+A at its current
                 // destination. Busy invocations are ignored, never queued or retried.
                 var result = await CapturePasteSequence.PasteAsync(batch, target, textOnly, slowerImages);

src/Focalet.Flutter/pubspec.yaml

@@ -2,7 +2,7 @@ name: focalet_flutter
 description: Focalet's cross-platform floating UI.
 repository: https://github.com/timctho/focalet
 publish_to: 'none'
-version: 0.3.1+15
+version: 0.3.2+16
 
 environment:
   sdk: ^3.13.2

tests/Focalet.Windows.Tests/CaptureHotkeyAcceptance.cs

@@ -117,6 +117,16 @@ public static async Task RunAsync(CaptureClipboardBatch batch)
         await Task.Delay(700);
         if (!context.HasBatch || previous.Rtf != interrupted || Clipboard.GetText() != "interrupted hotkey fixture")
             throw new InvalidOperationException("Interrupted paste resumed automatically or discarded the batch.");
+        notices.Clear();
+        picturesBeforeInterruption = Count(previous.Rtf!, @"\pict");
+        SendHotkey(capture: false);
+        await ForegroundRoutingAcceptance.WaitFor(() => Count(previous.Rtf!, @"\pict") > picturesBeforeInterruption,
+            "Navigation fixture did not receive its first image.");
+        SendInput(2, [Key(0x27), Key(0x27, true)], Marshal.SizeOf<Input>()); // Move the caret in the SAME input.
+        await ForegroundRoutingAcceptance.WaitFor(() => notices.Contains("Paste stopped") && !context.Busy,
+            "User navigation within the same input did not stop the batch.");
+        if (Count(previous.Rtf!, @"\pict") != picturesBeforeInterruption + 1)
+            throw new InvalidOperationException("Paste continued after user navigation.");
         if (enters != 0 || captured != 2) throw new InvalidOperationException("Repeated paste sent Enter or captured again.");
         Console.WriteLine("PASS Shift+Alt+A captures without clipboard changes; deliberate Alt+A repeats silently at the current input; busy presses do not queue; cancellation and interruption retain the batch without automatic retries.");
     }