Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/macos-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ jobs:
run: |
node scripts/docs-lint.mjs
node --test tests/docs-site-toc.test.mjs
node --test tests/background-capability-guidance.test.mjs

- name: Select Xcode 26.6 (if present) or fallback to default
run: |
Expand Down
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@

- Clarify observation evidence failures and runtime refusal guidance so same-build verification errors do not imply that an update will fix them. #710.

- Strengthen selected-CLI guidance checks, repair published guide links, restore the read-only clipboard example, run guidance checks in regular macOS CI, and make noncooperative detection timeout proof independent of scheduler timing.

## 4.3.4 - 2026-09-11

**Highlights:** Restore provider-compatible MCP tools and frontmost daemon captures.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -292,24 +292,42 @@ struct ElementDetectionTimeoutRunnerTests {
}
}

@Test
func `Detection timeout wins over noncooperative work`() async throws {
let startedAt = Date()

do {
_ = try await ElementDetectionTimeoutRunner.run(seconds: 0.02) {
let stopAt = Date().addingTimeInterval(0.5)
while Date() < stopAt {
try? await Task.sleep(nanoseconds: 50_000_000)
}
@Test(.timeLimit(.minutes(1)))
func `Detection timeout wins over noncooperative work`() async {
let gate = ElementDetectionTimeoutWorkGate()
let task = Task {
try await ElementDetectionTimeoutRunner.run(seconds: 1.0) {
await gate.wait()
await gate.finish()
return [DetectedElement]()
}
}
let watchdog = Task {
do {
try await Task.sleep(for: .seconds(10))
await gate.release()
} catch {}
}
await gate.waitUntilBlocked()
#expect(await gate.hasEntered, "The timed operation must enter before its deadline")

do {
_ = try await task.value
Issue.record("Expected detection timeout")
} catch let CaptureError.detectionTimedOut(duration) {
#expect(duration == 0.02)
#expect(duration == 1.0)
} catch {
Issue.record("Expected detection timeout, got \(error)")
}

#expect(Date().timeIntervalSince(startedAt) < 0.25)
// Work remains blocked regardless of cancellation until the caller has returned.
#expect(await !gate.isReleased, "Detection timeout must not join noncooperative work")
watchdog.cancel()
await gate.release()
if await gate.hasEntered {
await gate.waitUntilFinished()
}
await watchdog.value
}

@Test
Expand Down Expand Up @@ -338,6 +356,49 @@ struct ElementDetectionTimeoutRunnerTests {
}
}

private actor ElementDetectionTimeoutWorkGate {
private var waiting: CheckedContinuation<Void, Never>?
private var entered: CheckedContinuation<Void, Never>?
private var finished: CheckedContinuation<Void, Never>?
private var hasFinished = false
private(set) var hasEntered = false
private(set) var isReleased = false

func wait() async {
guard !self.isReleased else { return }
await withCheckedContinuation { continuation in
self.hasEntered = true
self.waiting = continuation
self.entered?.resume()
self.entered = nil
}
}

func waitUntilBlocked() async {
guard self.waiting == nil, !self.isReleased else { return }
await withCheckedContinuation { self.entered = $0 }
}

func release() {
self.isReleased = true
self.waiting?.resume()
self.waiting = nil
self.entered?.resume()
self.entered = nil
}

func finish() {
self.hasFinished = true
self.finished?.resume()
self.finished = nil
}

func waitUntilFinished() async {
guard !self.hasFinished else { return }
await withCheckedContinuation { self.finished = $0 }
}
}

@Suite(.tags(.fast))
struct ElementDetectionCacheTests {
@Test
Expand Down
5 changes: 3 additions & 2 deletions docs/agent-skill.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ read_when:

# Agent Skill for Peekaboo

The [Peekaboo skill](../skills/peekaboo/SKILL.md) teaches agents to observe macOS UI, select an exact target, act in the background where supported, and verify the result. It covers native apps and browser chrome, routes page content to browser tooling, and explains execution-host permissions, snapshot ownership, coordinates, and unverified input outcomes. Ordinary automation uses the installed CLI; source builds belong to Peekaboo development.
The [Peekaboo skill](https://github.com/openclaw/Peekaboo/blob/main/skills/peekaboo/SKILL.md) teaches agents to observe macOS UI, select an exact target, act in the background where supported, and verify the result. It covers native apps and browser chrome, routes page content to browser tooling, and explains execution-host permissions, snapshot ownership, coordinates, and unverified input outcomes. Ordinary automation uses the installed CLI; source builds belong to Peekaboo development.

## Prerequisites

Expand Down Expand Up @@ -58,11 +58,12 @@ For skill/documentation edits, run from the repository root:

```bash
node scripts/docs-lint.mjs
node --test tests/background-capability-guidance.test.mjs
ruby -e 'h=File.read("skills/peekaboo/SKILL.md").split(/^---\s*$/,3)[1]; keys=h.lines.grep(/^[A-Za-z0-9_-]+:/).map { |line| line.split(":",2).first }; abort("unexpected skill frontmatter") unless keys.sort == ["description","name"]'
git diff --check
```

Source-code changes follow [AGENTS.md](../AGENTS.md) and the [building guide](building.md). Live checks should use a controlled target, inspect the actual screen or state readback, and preserve retry-unsafe outcomes. Image dimensions and command success alone are insufficient proof that the intended UI changed.
Source-code changes follow [AGENTS.md](https://github.com/openclaw/Peekaboo/blob/main/AGENTS.md) and the [building guide](building.md). Live checks should use a controlled target, inspect the actual screen or state readback, and preserve retry-unsafe outcomes. Image dimensions and command success alone are insufficient proof that the intended UI changed.

## Canonical references

Expand Down
3 changes: 3 additions & 0 deletions skills/peekaboo/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ GUI_SOCKET="$HOME/Library/Application Support/Peekaboo/bridge.sock"
"$PB" app list --include-hidden --include-background --json
"$PB" window list --app Safari --json

# Read clipboard contents only when relevant to the authorized task.
"$PB" clipboard get --json

# Pixels only; an exact window also publishes a coordinate receipt.
"$PB" see --window-id "$WINDOW_ID" --no-elements --path /tmp/peekaboo-window.png --json

Expand Down
13 changes: 9 additions & 4 deletions tests/background-capability-guidance.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,14 @@ test('bundled skill never advertises app/PID-only background press', () => {
);
}

assert.equal(isTargetedRawPress('peekaboo press return --pid 1234'), true);
assert.equal(isTargetedRawPress('"$PB" press return --pid 1234'), true);
assert.equal(hasSafeRawPressRoute('peekaboo press return --pid 1234'), false);
assert.equal(hasSafeRawPressRoute('peekaboo press return --pid 1234 --window-id 42'), true);
for (const binary of ['peekaboo', '"$PB"']) {
for (const target of ['--app TextEdit', '--pid 1234']) {
const unsafe = `${binary} press return ${target}`;
assert.equal(isTargetedRawPress(unsafe), true);
assert.equal(hasSafeRawPressRoute(unsafe), false);
assert.equal(hasSafeRawPressRoute(`${unsafe} --window-id 42`), true);
}
}
});

test('bundled skill keeps routine management examples read-only', () => {
Expand All @@ -79,6 +83,7 @@ test('bundled skill keeps routine management examples read-only', () => {
assert.doesNotMatch(skill, /^(?:peekaboo|"\$PB") clipboard (?:set|clear|restore)\b/m);
assert.doesNotMatch(skill, /^(?:peekaboo|"\$PB") permissions request\b/m);
assert.doesNotMatch(skill, /^(?:peekaboo|"\$PB") app focus\b/m);
assert.match(skill, /^(?:peekaboo|"\$PB") clipboard get --json$/m);
assert.match(skill, /^(?:peekaboo|"\$PB") permissions status --all-sources --json$/m);
assert.match(skill, /^(?:peekaboo|"\$PB") app list --include-hidden --include-background --json$/m);
});
Expand Down