From 5950dd4ca09b54838a36e7a9f32a788b5323a3ef Mon Sep 17 00:00:00 2001 From: Clawdia Date: Sat, 18 Jul 2026 02:16:57 -0700 Subject: [PATCH] Overlay local skill improvements --- skills/claude-foreman/.github/CODEOWNERS | 1 + .../claude-foreman/.github/workflows/test.yml | 17 + skills/claude-foreman/.gitignore | 4 + .../scripts/smoke-claude-profile.sh | 221 ++++++++ .../scripts/smoke-openclaw-model.sh | 168 ++++++ skills/claude-foreman/scripts/test.sh | 505 ++++++++++++++++++ skills/moltmaster/SKILL.md | 7 +- skills/moltmaster/scripts/moltmaster.mjs | 27 +- skills/moltmaster/scripts/moltmaster.test.mjs | 2 +- skills/shell-swap/.github/CODEOWNERS | 1 + skills/shell-swap/BASELINE_TEST_AUDIT.md | 32 ++ skills/shell-swap/RESEARCH-harness-pin-gap.md | 100 ++++ skills/shell-swap/scripts/test.sh | 324 +++++++++++ skills/web-use/BASELINE_TEST_AUDIT.md | 24 + skills/web-use/ROADMAP.md | 15 + skills/web-use/SKILL.md | 203 +++++-- skills/web-use/references/backends.md.example | 17 +- .../web-use/references/extraction-backends.md | 3 +- skills/web-use/scripts/browserless_extract.py | 233 +++++--- .../scripts/browserless_media_requests.py | 169 ++++++ skills/web-use/scripts/browserless_session.py | 87 ++- skills/web-use/scripts/test.sh | 43 ++ .../scripts/tinyfish_browser_extract.py | 52 +- 23 files changed, 2110 insertions(+), 145 deletions(-) create mode 100644 skills/claude-foreman/.github/CODEOWNERS create mode 100644 skills/claude-foreman/.github/workflows/test.yml create mode 100644 skills/claude-foreman/.gitignore create mode 100755 skills/claude-foreman/scripts/smoke-claude-profile.sh create mode 100755 skills/claude-foreman/scripts/smoke-openclaw-model.sh create mode 100755 skills/claude-foreman/scripts/test.sh create mode 100644 skills/shell-swap/.github/CODEOWNERS create mode 100644 skills/shell-swap/BASELINE_TEST_AUDIT.md create mode 100644 skills/shell-swap/RESEARCH-harness-pin-gap.md create mode 100755 skills/shell-swap/scripts/test.sh create mode 100644 skills/web-use/BASELINE_TEST_AUDIT.md create mode 100644 skills/web-use/ROADMAP.md create mode 100755 skills/web-use/scripts/browserless_media_requests.py create mode 100755 skills/web-use/scripts/test.sh diff --git a/skills/claude-foreman/.github/CODEOWNERS b/skills/claude-foreman/.github/CODEOWNERS new file mode 100644 index 0000000..37028cf --- /dev/null +++ b/skills/claude-foreman/.github/CODEOWNERS @@ -0,0 +1 @@ +* @clawSean diff --git a/skills/claude-foreman/.github/workflows/test.yml b/skills/claude-foreman/.github/workflows/test.yml new file mode 100644 index 0000000..e11d784 --- /dev/null +++ b/skills/claude-foreman/.github/workflows/test.yml @@ -0,0 +1,17 @@ +name: Tests + +on: + pull_request: + push: + branches: + - main + +jobs: + offline: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install shellcheck + run: sudo apt-get update && sudo apt-get install -y shellcheck + - name: Run offline tests + run: scripts/test.sh diff --git a/skills/claude-foreman/.gitignore b/skills/claude-foreman/.gitignore new file mode 100644 index 0000000..ce1055b --- /dev/null +++ b/skills/claude-foreman/.gitignore @@ -0,0 +1,4 @@ +cost-log.json +cost-log.json.lock +artifacts/ +rollback/ diff --git a/skills/claude-foreman/scripts/smoke-claude-profile.sh b/skills/claude-foreman/scripts/smoke-claude-profile.sh new file mode 100755 index 0000000..a9c6851 --- /dev/null +++ b/skills/claude-foreman/scripts/smoke-claude-profile.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# Live Claude CLI profile smoke. +# Proves a named profile can authenticate from the env var listed in +# claude-profiles.json and returns actual result text from Claude. + +set -euo pipefail + +SCRIPT_SRC="$0" +if command -v readlink >/dev/null 2>&1; then + SCRIPT_SRC="$(readlink -f "$0" 2>/dev/null || echo "$0")" +elif command -v realpath >/dev/null 2>&1; then + SCRIPT_SRC="$(realpath "$0" 2>/dev/null || echo "$0")" +fi +SKILL_DIR="$(cd "$(dirname "$SCRIPT_SRC")/.." && pwd)" + +PROFILES_FILE="${FOREMAN_CLAUDE_PROFILES_FILE:-${CLAUDE_PROFILES_FILE:-/root/.openclaw/claude-profiles.json}}" +PROFILE="" +MODEL="sonnet" +PROMPT="" +TIMEOUT_SECONDS=180 + +usage() { + cat <<'USAGE' +Usage: smoke-claude-profile.sh --profile [options] + +Options: + --profile Profile key from claude-profiles.json. + --profiles-file Profile JSON path. Default: /root/.openclaw/claude-profiles.json + --model Claude model alias/id. Default: sonnet + --prompt Prompt to send. Default: unique exact-reply sentinel + --timeout Live command timeout. Default: 180 + +The script prints the selected profile, env var name, stream path, Claude init +model, exit code, and parsed result text. It never prints token values. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --profile) + PROFILE="${2:?missing --profile value}" + shift 2 + ;; + --profiles-file) + PROFILES_FILE="${2:?missing --profiles-file value}" + shift 2 + ;; + --model) + MODEL="${2:?missing --model value}" + shift 2 + ;; + --prompt) + PROMPT="${2:?missing --prompt value}" + shift 2 + ;; + --timeout) + TIMEOUT_SECONDS="${2:?missing --timeout value}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "[smoke] Unknown flag: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +for _bin in claude python3; do + if ! command -v "$_bin" >/dev/null 2>&1; then + echo "[smoke] Required command not found in PATH: $_bin" >&2 + exit 1 + fi +done + +if [[ -z "$PROFILE" ]]; then + PROFILE="$(python3 -c ' +import json, sys +try: + print((json.load(open(sys.argv[1])).get("active") or "").strip()) +except Exception: + print("") +' "$PROFILES_FILE")" +fi +if [[ -z "$PROFILE" ]]; then + echo "[smoke] No profile selected. Pass --profile ." >&2 + exit 1 +fi +if [[ ! -f "$PROFILES_FILE" ]]; then + echo "[smoke] Profiles file not found: $PROFILES_FILE" >&2 + exit 1 +fi + +PROFILE_META=$(python3 -c ' +import json, sys +profiles_file, profile = sys.argv[1], sys.argv[2] +data = json.load(open(profiles_file)) +entry = (data.get("profiles") or {}).get(profile) +if not isinstance(entry, dict): + raise SystemExit("unknown profile: " + profile) +env_var = str(entry.get("env_var") or "").strip() +label = str(entry.get("label") or profile).strip() +if not env_var: + raise SystemExit("profile has no env_var: " + profile) +print(env_var + "\x1f" + label) +' "$PROFILES_FILE" "$PROFILE") +IFS=$'\x1f' read -r TOKEN_ENV_VAR PROFILE_LABEL <<< "$PROFILE_META" + +if [[ ! "$TOKEN_ENV_VAR" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + echo "[smoke] Profile '$PROFILE' uses invalid env_var '$TOKEN_ENV_VAR'." >&2 + echo "[smoke] Env var names must match [A-Za-z_][A-Za-z0-9_]*." >&2 + exit 1 +fi + +TOKEN="${!TOKEN_ENV_VAR:-}" +if [[ -z "$TOKEN" ]]; then + echo "[smoke] Profile '$PROFILE' expects token env var \$$TOKEN_ENV_VAR, but it is empty." >&2 + exit 1 +fi +export CLAUDE_CODE_OAUTH_TOKEN="$TOKEN" + +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$-${PROFILE}" +ARTIFACT_DIR="$SKILL_DIR/artifacts/smokes" +mkdir -p "$ARTIFACT_DIR" +STREAM_FILE="$ARTIFACT_DIR/${RUN_ID}.stream.jsonl" +ERR_FILE="$ARTIFACT_DIR/${RUN_ID}.stderr" + +if [[ -z "$PROMPT" ]]; then + PROMPT="Reply exactly: FOREMAN_PROFILE_SMOKE_${PROFILE}_${RUN_ID}" +fi + +echo "[smoke] profile=$PROFILE label=$PROFILE_LABEL env=\$$TOKEN_ENV_VAR model=$MODEL" +echo "[smoke] stream=$STREAM_FILE" +echo "[smoke] prompt=$PROMPT" + +CMD=( + claude + -p "$PROMPT" + --model "$MODEL" + --output-format stream-json + --verbose + --no-session-persistence +) + +set +e +if command -v timeout >/dev/null 2>&1; then + timeout "$TIMEOUT_SECONDS" "${CMD[@]}" >"$STREAM_FILE" 2>"$ERR_FILE" +else + "${CMD[@]}" >"$STREAM_FILE" 2>"$ERR_FILE" +fi +EXIT_CODE=$? +set -e + +SUMMARY=$(python3 -c ' +import json, sys +stream = sys.argv[1] +init_model = "" +session = "" +result = "" +stop = "" +cost = 0 +turns = 0 +found = False +try: + for line in open(stream): + line = line.strip() + if not line: + continue + try: + ev = json.loads(line) + except Exception: + continue + if ev.get("type") == "system" and ev.get("subtype") == "init": + init_model = str(ev.get("model") or "") + session = str(ev.get("session_id") or session) + elif ev.get("type") == "result": + found = True + result = str(ev.get("result") or "") + stop = str(ev.get("stop_reason") or ev.get("subtype") or "") + cost = ev.get("total_cost_usd") or 0 + turns = ev.get("num_turns") or 0 + session = str(ev.get("session_id") or session) +except FileNotFoundError: + pass +print(json.dumps({ + "found": found, + "init_model": init_model, + "session_id": session, + "stop_reason": stop, + "turns": turns, + "cost_usd": cost, + "result": result, +})) +' "$STREAM_FILE") + +python3 -c ' +import json, sys +d = json.loads(sys.argv[1]) +print("[smoke] init_model=" + str(d.get("init_model") or "")) +print("[smoke] session=" + str(d.get("session_id") or "")) +print("[smoke] stop_reason=" + str(d.get("stop_reason") or "")) +print("[smoke] turns=" + str(d.get("turns") or 0) + " cost=$" + str(d.get("cost_usd") or 0)) +print("[smoke] result=" + str(d.get("result") or "")) +' "$SUMMARY" + +FOUND=$(python3 -c 'import json,sys; print("1" if json.loads(sys.argv[1]).get("found") else "0")' "$SUMMARY") +RESULT_TEXT=$(python3 -c 'import json,sys; print(json.loads(sys.argv[1]).get("result") or "")' "$SUMMARY") + +if [[ "$EXIT_CODE" -ne 0 ]]; then + echo "[smoke] FAIL: claude exited $EXIT_CODE; stderr=$ERR_FILE" >&2 + exit "$EXIT_CODE" +fi +if [[ "$FOUND" != "1" || -z "${RESULT_TEXT//[[:space:]]/}" ]]; then + echo "[smoke] FAIL: no non-empty result text parsed from stream." >&2 + exit 1 +fi + +echo "[smoke] PASS" diff --git a/skills/claude-foreman/scripts/smoke-openclaw-model.sh b/skills/claude-foreman/scripts/smoke-openclaw-model.sh new file mode 100755 index 0000000..1add5a6 --- /dev/null +++ b/skills/claude-foreman/scripts/smoke-openclaw-model.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Live OpenClaw model-provider smoke. +# Proves a selectable provider/model ref can return actual response text through +# the OpenClaw agent pipeline. This matters for CLI backends because direct +# `openclaw infer model run` exercises provider transports, not agent runtimes. + +set -euo pipefail + +SCRIPT_SRC="$0" +if command -v readlink >/dev/null 2>&1; then + SCRIPT_SRC="$(readlink -f "$0" 2>/dev/null || echo "$0")" +elif command -v realpath >/dev/null 2>&1; then + SCRIPT_SRC="$(realpath "$0" 2>/dev/null || echo "$0")" +fi +SKILL_DIR="$(cd "$(dirname "$SCRIPT_SRC")/.." && pwd)" + +MODEL="" +PROMPT="" +TIMEOUT_SECONDS=240 +SESSION_KEY="" + +usage() { + cat <<'USAGE' +Usage: smoke-openclaw-model.sh --model [options] + +Options: + --model Model ref exactly as shown by `openclaw models list`. + --prompt Prompt to send. Default: unique exact-reply sentinel + --timeout Live command timeout. Default: 240 + --session-key OpenClaw local session key. Default: generated smoke key + +The script saves raw JSON/text output under artifacts/smokes and prints the +parsed response text when available. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --model) + MODEL="${2:?missing --model value}" + shift 2 + ;; + --prompt) + PROMPT="${2:?missing --prompt value}" + shift 2 + ;; + --timeout) + TIMEOUT_SECONDS="${2:?missing --timeout value}" + shift 2 + ;; + --session-key) + SESSION_KEY="${2:?missing --session-key value}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "[smoke] Unknown flag: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +for _bin in openclaw python3; do + if ! command -v "$_bin" >/dev/null 2>&1; then + echo "[smoke] Required command not found in PATH: $_bin" >&2 + exit 1 + fi +done + +if [[ -z "$MODEL" ]]; then + echo "[smoke] Missing --model ." >&2 + exit 1 +fi + +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" +ARTIFACT_DIR="$SKILL_DIR/artifacts/smokes" +mkdir -p "$ARTIFACT_DIR" +OUT_FILE="$ARTIFACT_DIR/${RUN_ID}.openclaw-model.json" +ERR_FILE="$ARTIFACT_DIR/${RUN_ID}.openclaw-model.stderr" +SESSION_KEY="${SESSION_KEY:-agent:main:foreman-openclaw-smoke-${RUN_ID}}" + +if [[ -z "$PROMPT" ]]; then + SAFE_MODEL="${MODEL//[^A-Za-z0-9_]/_}" + PROMPT="Reply exactly: OPENCLAW_MODEL_SMOKE_${SAFE_MODEL}_${RUN_ID}" +fi + +echo "[smoke] model=$MODEL" +echo "[smoke] output=$OUT_FILE" +echo "[smoke] session_key=$SESSION_KEY" +echo "[smoke] prompt=$PROMPT" + +CMD=( + openclaw agent + --local + --json + --session-key "$SESSION_KEY" + --model "$MODEL" + --message "$PROMPT" + --timeout "$TIMEOUT_SECONDS" +) + +set +e +if command -v timeout >/dev/null 2>&1; then + timeout "$TIMEOUT_SECONDS" "${CMD[@]}" >"$OUT_FILE" 2>"$ERR_FILE" +else + "${CMD[@]}" >"$OUT_FILE" 2>"$ERR_FILE" +fi +EXIT_CODE=$? +set -e + +RESULT_TEXT=$(python3 -c ' +import json, sys +path, err_path = sys.argv[1], sys.argv[2] +try: + raw = open(path).read() +except FileNotFoundError: + raw = "" +text = "" +try: + data = json.loads(raw) + def walk(o): + if isinstance(o, dict): + for key in ("text", "content", "message", "reply", "output", "result"): + v = o.get(key) + if isinstance(v, str) and v.strip(): + return v + for v in o.values(): + got = walk(v) + if got: + return got + elif isinstance(o, list): + for v in o: + got = walk(v) + if got: + return got + return "" + text = walk(data) +except Exception: + text = raw.strip() +if not text.strip(): + try: + err = open(err_path).read() + except FileNotFoundError: + err = "" + for marker in ("FailoverError:", "Error:"): + if marker in err: + text = err.rsplit(marker, 1)[-1].strip().splitlines()[0].strip() + break +print(text) +' "$OUT_FILE" "$ERR_FILE") + +echo "[smoke] exit_code=$EXIT_CODE" +echo "[smoke] result=$RESULT_TEXT" + +if [[ "$EXIT_CODE" -ne 0 ]]; then + echo "[smoke] FAIL: openclaw exited $EXIT_CODE; stderr=$ERR_FILE" >&2 + exit "$EXIT_CODE" +fi +if [[ -z "${RESULT_TEXT//[[:space:]]/}" ]]; then + echo "[smoke] FAIL: no non-empty result text parsed from output." >&2 + exit 1 +fi + +echo "[smoke] PASS" diff --git a/skills/claude-foreman/scripts/test.sh b/skills/claude-foreman/scripts/test.sh new file mode 100755 index 0000000..dc71137 --- /dev/null +++ b/skills/claude-foreman/scripts/test.sh @@ -0,0 +1,505 @@ +#!/usr/bin/env bash +# Offline regression tests for claude-foreman. +# No live Claude/API calls: the claude binary is replaced with a fake stream. + +set -euo pipefail + +SKILL_DIR="$(cd "$(dirname "$0")/.." && pwd)" +PASS=0 +FAIL=0 +TMPDIR="" + +pass() { + PASS=$((PASS + 1)) + echo " PASS: $1" +} + +fail() { + FAIL=$((FAIL + 1)) + echo " FAIL: $1" >&2 +} + +assert_file() { + local path="$1" + local label="$2" + if [[ -f "$path" ]]; then + pass "$label" + else + fail "$label" + fi +} + +assert_executable() { + local path="$1" + local label="$2" + if [[ -x "$path" ]]; then + pass "$label" + else + fail "$label" + fi +} + +assert_contains() { + local haystack="$1" + local needle="$2" + local label="$3" + if grep -Fq -- "$needle" <<<"$haystack"; then + pass "$label" + else + fail "$label" + fi +} + +assert_not_contains() { + local haystack="$1" + local needle="$2" + local label="$3" + if grep -Fq -- "$needle" <<<"$haystack"; then + fail "$label" + else + pass "$label" + fi +} + +run_expect_success() { + local label="$1" + shift + if "$@"; then + pass "$label" + else + fail "$label" + fi +} + +run_expect_failure() { + local label="$1" + shift + if "$@" >/dev/null 2>&1; then + fail "$label" + else + pass "$label" + fi +} + +make_fake_claude() { + local fake_dir="$1" + cat > "$fake_dir/claude" <<'SH' +#!/usr/bin/env bash +set -euo pipefail + +: "${FAKE_CLAUDE_LOG:?missing FAKE_CLAUDE_LOG}" +: "${FAKE_CLAUDE_MODE:=success}" + +{ + printf 'ARGS\n' + printf '%s\n' "$@" + printf 'TOKEN=%s\n' "${CLAUDE_CODE_OAUTH_TOKEN:-}" +} >> "$FAKE_CLAUDE_LOG" + +case "$FAKE_CLAUDE_MODE" in + success) + cat <<'JSON' +{"type":"system","subtype":"init","session_id":"fake-success-session","model":"opus"} +{"type":"assistant","message":{"content":[{"type":"text","text":"done"}],"stop_reason":"end_turn"}} +{"type":"result","subtype":"success","result":"FOREMAN_STREAM_OK","total_cost_usd":0.0123,"num_turns":1,"session_id":"fake-success-session"} +JSON + ;; + max_turns) + cat <<'JSON' +{"type":"system","subtype":"init","session_id":"fake-max-session","model":"opus"} +{"type":"result","subtype":"error_max_turns","result":"FOREMAN_PARTIAL","total_cost_usd":0.5,"num_turns":15,"session_id":"fake-max-session"} +JSON + ;; + no_result) + cat <<'JSON' +{"type":"system","subtype":"init","session_id":"fake-no-result-session","model":"opus"} +JSON + exit 42 + ;; + permission_denial) + cat <<'JSON' +{"type":"system","subtype":"init","session_id":"fake-denial-session","model":"opus"} +{"type":"result","subtype":"success","result":"FOREMAN_DENIAL_OK","total_cost_usd":0.25,"num_turns":2,"session_id":"fake-denial-session","permission_denials":[{"tool_name":"Bash","tool_input":{"command":"cat /root/.secrets/example"}}]} +JSON + ;; + redaction) + cat <<'JSON' +{"type":"system","subtype":"init","session_id":"fake-redaction-session","model":"opus"} +{"type":"assistant","message":{"content":[{"type":"tool_use","name":"WebFetch","input":{"url":"https://example.com/path?token=SECRET_TOKEN#frag"}}],"stop_reason":"tool_use"}} +{"type":"assistant","message":{"content":[{"type":"text","text":"done"}],"stop_reason":"end_turn"}} +{"type":"result","subtype":"success","result":"FOREMAN_REDACTION_OK","total_cost_usd":0,"num_turns":2,"session_id":"fake-redaction-session"} +JSON + ;; + rate_limit_personal) + if [[ "${CLAUDE_CODE_OAUTH_TOKEN:-}" == "personal-token" ]]; then + cat <<'JSON' +{"type":"assistant","message":{"content":[{"type":"text","text":"You've hit your session limit · resets 2:20am (UTC)"}],"stop_reason":"end_turn"},"error":"rate_limit"} +{"type":"result","subtype":"success","is_error":true,"api_error_status":429,"result":"You've hit your session limit · resets 2:20am (UTC)","total_cost_usd":0,"num_turns":1,"session_id":"fake-rate-limit-session","usage":{"input_tokens":0,"output_tokens":0}} +JSON + exit 1 + fi + cat <<'JSON' +{"type":"system","subtype":"init","session_id":"fake-fallback-session","model":"opus"} +{"type":"assistant","message":{"content":[{"type":"text","text":"fallback ok"}],"stop_reason":"end_turn"}} +{"type":"result","subtype":"success","result":"FOREMAN_FALLBACK_OK","total_cost_usd":0.034,"num_turns":1,"session_id":"fake-fallback-session","usage":{"input_tokens":10,"output_tokens":3}} +JSON + ;; + *) + echo "unknown fake mode: $FAKE_CLAUDE_MODE" >&2 + exit 99 + ;; +esac +SH + chmod +x "$fake_dir/claude" +} + +run_dispatch() { + local mode="$1" + local target="$2" + local prompt="$3" + shift 3 + + FAKE_CLAUDE_MODE="$mode" \ + FAKE_CLAUDE_LOG="$TMPDIR/claude-$mode.log" \ + FOREMAN_CLAUDE_PROFILES_FILE="${FOREMAN_CLAUDE_PROFILES_FILE:-$TMPDIR/no-default-profiles.json}" \ + PATH="$TMPDIR/fake-bin:$PATH" \ + "$SKILL_DIR/scripts/dispatch.sh" plan "$target" "$prompt" "$@" 2>&1 +} + +echo "=== claude-foreman offline tests ===" +echo "" + +TMPDIR=$(mktemp -d) +trap 'rm -rf "$TMPDIR"' EXIT +mkdir -p "$TMPDIR/fake-bin" "$TMPDIR/target" +make_fake_claude "$TMPDIR/fake-bin" + +echo "[1] Structure" +assert_file "$SKILL_DIR/SKILL.md" "SKILL.md exists" +assert_file "$SKILL_DIR/README.md" "README.md exists" +assert_file "$SKILL_DIR/scripts/dispatch.sh" "dispatch.sh exists" +assert_executable "$SKILL_DIR/scripts/dispatch.sh" "dispatch.sh is executable" +assert_file "$SKILL_DIR/scripts/smoke-claude-profile.sh" "smoke-claude-profile.sh exists" +assert_executable "$SKILL_DIR/scripts/smoke-claude-profile.sh" "smoke-claude-profile.sh is executable" +assert_file "$SKILL_DIR/scripts/smoke-openclaw-model.sh" "smoke-openclaw-model.sh exists" +assert_executable "$SKILL_DIR/scripts/smoke-openclaw-model.sh" "smoke-openclaw-model.sh is executable" +assert_file "$SKILL_DIR/scripts/test-claude-auth-router.sh" "test-claude-auth-router.sh exists" +assert_executable "$SKILL_DIR/scripts/test-claude-auth-router.sh" "test-claude-auth-router.sh is executable" +for profile in plan implement review wide-open; do + assert_file "$SKILL_DIR/profiles/${profile}.md" "profiles/${profile}.md exists" +done + +echo "" +echo "[2] Syntax and lint" +run_expect_success "dispatch.sh passes bash -n" bash -n "$SKILL_DIR/scripts/dispatch.sh" +run_expect_success "smoke-claude-profile.sh passes bash -n" bash -n "$SKILL_DIR/scripts/smoke-claude-profile.sh" +run_expect_success "smoke-openclaw-model.sh passes bash -n" bash -n "$SKILL_DIR/scripts/smoke-openclaw-model.sh" +if command -v shellcheck >/dev/null 2>&1; then + run_expect_success "dispatch.sh passes shellcheck" shellcheck "$SKILL_DIR/scripts/dispatch.sh" + run_expect_success "test.sh passes shellcheck" shellcheck "$SKILL_DIR/scripts/test.sh" + run_expect_success "smoke-claude-profile.sh passes shellcheck" shellcheck "$SKILL_DIR/scripts/smoke-claude-profile.sh" + run_expect_success "smoke-openclaw-model.sh passes shellcheck" shellcheck "$SKILL_DIR/scripts/smoke-openclaw-model.sh" + run_expect_success "test-claude-auth-router.sh passes shellcheck" shellcheck "$SKILL_DIR/scripts/test-claude-auth-router.sh" +else + echo " SKIP: shellcheck not installed" +fi + +echo "" +echo "[3] Argument validation" +run_expect_failure "dispatch.sh with no args exits non-zero" "$SKILL_DIR/scripts/dispatch.sh" +run_expect_failure "dispatch.sh with 1 arg exits non-zero" "$SKILL_DIR/scripts/dispatch.sh" plan +run_expect_failure "dispatch.sh with 2 args exits non-zero" "$SKILL_DIR/scripts/dispatch.sh" plan "$TMPDIR/target" +run_expect_failure "dispatch.sh rejects unknown profile" "$SKILL_DIR/scripts/dispatch.sh" bogus-profile "$TMPDIR/target" "test" +run_expect_failure "dispatch.sh rejects nonexistent target dir" "$SKILL_DIR/scripts/dispatch.sh" plan "$TMPDIR/nope" "test" +run_expect_failure "dispatch.sh rejects unknown flags" "$SKILL_DIR/scripts/dispatch.sh" plan "$TMPDIR/target" "test" --bogus-flag + +echo "" +echo "[4] Root safety" +if [[ "${EUID:-$(id -u)}" -eq 0 ]]; then + set +e + "$SKILL_DIR/scripts/dispatch.sh" claws-out "$TMPDIR/target" "test" >/dev/null 2>&1 + exit_code=$? + set -e + if [[ "$exit_code" -eq 3 ]]; then + pass "claws-out blocked as root with exit code 3" + else + fail "claws-out root block exit code is $exit_code, expected 3" + fi + + unsafe_stderr=$("$SKILL_DIR/scripts/dispatch.sh" unsafe "$TMPDIR/target" "test" 2>&1 >/dev/null || true) + assert_contains "$unsafe_stderr" "deprecated" "'unsafe' alias prints deprecation notice" +else + echo " SKIP: not running as root" +fi + +echo "" +echo "[5] stream-json success path" +success_out=$(run_dispatch success "$TMPDIR/target" "fake prompt" --max-turns 3) +assert_contains "$success_out" "FOREMAN_STREAM_OK" "prints final result text" +assert_contains "$success_out" "[foreman] Stop reason: end_turn" "normalizes success stop reason" +assert_contains "$success_out" "[foreman] Stream:" "prints stream artifact path" +assert_contains "$(cat "$TMPDIR/claude-success.log")" "--output-format" "passes --output-format flag" +assert_contains "$(cat "$TMPDIR/claude-success.log")" "stream-json" "uses stream-json output" +assert_contains "$(cat "$TMPDIR/claude-success.log")" "--verbose" "passes --verbose" +assert_contains "$(cat "$TMPDIR/claude-success.log")" "FINAL-OUTPUT REQUIREMENT" "appends final-output guardrail" +assert_contains "$(cat "$TMPDIR/claude-success.log")" "Bash(git:*),Bash(ls:*)" "uses separate Bash allowlist entries" + +echo "" +echo "[5b] Optional extra add-dir roots" +: > "$TMPDIR/claude-success.log" +extra_dirs_out=$( + FOREMAN_EXTRA_ADD_DIRS="/Users/example:/opt/homebrew:/tmp" \ + run_dispatch success "$TMPDIR/target" "extra dirs prompt" --max-turns 3 +) +assert_contains "$extra_dirs_out" "FOREMAN_STREAM_OK" "extra add-dir run succeeds" +assert_contains "$(cat "$TMPDIR/claude-success.log")" "--add-dir" "passes --add-dir when FOREMAN_EXTRA_ADD_DIRS is set" +assert_contains "$(cat "$TMPDIR/claude-success.log")" "/Users/example" "passes first extra add-dir path" +assert_contains "$(cat "$TMPDIR/claude-success.log")" "/opt/homebrew" "passes second extra add-dir path" +assert_contains "$(cat "$TMPDIR/claude-success.log")" "/tmp" "passes third extra add-dir path" +assert_not_contains "$success_out" "--add-dir" "does not pass --add-dir by default" + +echo "" +echo "[6] Incomplete and diagnostic paths" +max_out=$(run_dispatch max_turns "$TMPDIR/target" "max prompt" --max-turns 15 || true) +assert_contains "$max_out" "[foreman] Stop reason: max_turns" "maps error_max_turns to max_turns" +assert_contains "$max_out" "Artifacts saved:" "saves max-turns artifact" +assert_contains "$max_out" "FOREMAN_PARTIAL" "prints partial result text" + +set +e +no_result_out=$(run_dispatch no_result "$TMPDIR/target" "no result prompt" --max-turns 2) +no_result_exit=$? +set -e +if [[ "$no_result_exit" -eq 42 ]]; then + pass "preserves claude exit code for no-result runs" +else + fail "no-result exit code is $no_result_exit, expected 42" +fi +assert_contains "$no_result_out" "no result event" "reports missing result event" +assert_contains "$no_result_out" "Artifacts saved:" "saves no-result artifact" + +denial_out=$(run_dispatch permission_denial "$TMPDIR/target" "denial prompt" --max-turns 3) +assert_contains "$denial_out" "Permission denials: 1" "prints permission-denial count" +assert_contains "$denial_out" "FOREMAN_DENIAL_OK" "prints result when permission denials exist" +assert_contains "$denial_out" "Artifacts saved:" "saves permission-denial artifact" + +echo "" +echo "[7] Progress redaction" +redaction_out=$(run_dispatch redaction "$TMPDIR/target" "redaction prompt" --max-turns 3) +assert_contains "$redaction_out" "https://example.com/path [args stripped]" "strips URL query and fragment in progress output" +assert_not_contains "$redaction_out" "SECRET_TOKEN" "does not leak URL token in progress output" + +echo "" +echo "[8] Env-backed Claude profiles" +cat > "$TMPDIR/profiles.json" <<'JSON' +{ + "active": "personal", + "profiles": { + "personal": { + "label": "Fake Personal", + "env_var": "FAKE_PERSONAL_TOKEN" + }, + "work": { + "label": "Fake Work", + "env_var": "FAKE_WORK_TOKEN" + } + } +} +JSON + +profile_out=$( + FOREMAN_CLAUDE_PROFILES_FILE="$TMPDIR/profiles.json" \ + FAKE_PERSONAL_TOKEN="personal-token" \ + FAKE_WORK_TOKEN="work-token" \ + run_dispatch success "$TMPDIR/target" "profile prompt" --profile work --max-turns 3 +) +assert_contains "$profile_out" "Auth lane: claude-cli (work; env \$FAKE_WORK_TOKEN)" "reports selected profile env var" +assert_contains "$(cat "$TMPDIR/claude-success.log")" "TOKEN=work-token" "exports requested profile token to Claude" + +missing_profile_out=$( + FOREMAN_CLAUDE_PROFILES_FILE="$TMPDIR/profiles.json" \ + FAKE_PERSONAL_TOKEN="personal-token" \ + FAKE_WORK_TOKEN="work-token" \ + run_dispatch success "$TMPDIR/target" "missing profile prompt" --profile nope --max-turns 3 || true +) +assert_contains "$missing_profile_out" "Unknown Claude profile: nope" "rejects unknown profile" + +missing_token_out=$( + FOREMAN_CLAUDE_PROFILES_FILE="$TMPDIR/profiles.json" \ + FAKE_PERSONAL_TOKEN="personal-token" \ + FAKE_WORK_TOKEN="" \ + run_dispatch success "$TMPDIR/target" "missing token prompt" --profile work --max-turns 3 || true +) +assert_contains "$missing_token_out" "Expected a token in \$FAKE_WORK_TOKEN" "rejects empty profile env var" + +echo "" +echo "[9] Claude profile auto-detection" +cat > "$TMPDIR/auto-profiles.json" <<'JSON' +{ + "active": "personal", + "profiles": { + "personal": { + "label": "Fake Personal", + "env_var": "FAKE_PERSONAL_TOKEN" + }, + "work": { + "label": "Fake Work", + "env_var": "FAKE_WORK_TOKEN" + } + } +} +JSON + +: > "$TMPDIR/claude-success.log" +auto_out=$( + unset CLAUDE_CODE_OAUTH_TOKEN + FOREMAN_CLAUDE_PROFILES_FILE="$TMPDIR/auto-profiles.json" \ + FAKE_PERSONAL_TOKEN="personal-token" \ + FAKE_WORK_TOKEN="work-token" \ + run_dispatch success "$TMPDIR/target" "auto profile prompt" --max-turns 3 +) +assert_contains "$auto_out" "Auto-detected 2 usable Claude profiles; using profile fallback lane." "reports auto-detected profile lane" +assert_contains "$auto_out" "Auth lane: claude-cli fallback (personal -> work) [auto-detected]" "auto-detected lane uses fallback order" +assert_contains "$(cat "$TMPDIR/claude-success.log")" "TOKEN=personal-token" "auto-detected lane exports active profile token" + +cat > "$TMPDIR/one-profile.json" <<'JSON' +{ + "active": "personal", + "profiles": { + "personal": { + "label": "Fake Personal", + "env_var": "FAKE_PERSONAL_TOKEN" + }, + "work": { + "label": "Fake Work", + "env_var": "FAKE_WORK_TOKEN" + } + } +} +JSON + +: > "$TMPDIR/claude-success.log" +one_profile_out=$( + unset CLAUDE_CODE_OAUTH_TOKEN + FOREMAN_CLAUDE_PROFILES_FILE="$TMPDIR/one-profile.json" \ + FAKE_PERSONAL_TOKEN="personal-token" \ + FAKE_WORK_TOKEN="" \ + run_dispatch success "$TMPDIR/target" "one profile prompt" --max-turns 3 +) +assert_contains "$one_profile_out" "Auth lane: inherited (ambient claude auth)" "one usable profile stays ambient by default" +assert_contains "$(cat "$TMPDIR/claude-success.log")" "TOKEN=" "one-profile ambient lane leaves token unset" +assert_not_contains "$(cat "$TMPDIR/claude-success.log")" "TOKEN=personal-token" "one-profile ambient lane does not export profile token" + +: > "$TMPDIR/claude-success.log" +ambient_token_out=$( + FOREMAN_CLAUDE_PROFILES_FILE="$TMPDIR/auto-profiles.json" \ + CLAUDE_CODE_OAUTH_TOKEN="ambient-token" \ + FAKE_PERSONAL_TOKEN="personal-token" \ + FAKE_WORK_TOKEN="work-token" \ + run_dispatch success "$TMPDIR/target" "ambient token prompt" --max-turns 3 +) +assert_contains "$ambient_token_out" "Auth lane: inherited (ambient claude auth)" "caller-provided CLAUDE_CODE_OAUTH_TOKEN suppresses auto-detection" +assert_contains "$(cat "$TMPDIR/claude-success.log")" "TOKEN=ambient-token" "ambient token is preserved when provided by caller" +assert_not_contains "$ambient_token_out" "Auto-detected" "does not report auto-detection when ambient token is provided" + +: > "$TMPDIR/claude-success.log" +no_profile_fallback_out=$( + unset CLAUDE_CODE_OAUTH_TOKEN + FOREMAN_CLAUDE_PROFILES_FILE="$TMPDIR/auto-profiles.json" \ + FAKE_PERSONAL_TOKEN="personal-token" \ + FAKE_WORK_TOKEN="work-token" \ + run_dispatch success "$TMPDIR/target" "no profile fallback prompt" --no-profile-fallback --max-turns 3 +) +assert_contains "$no_profile_fallback_out" "Auth lane: inherited (ambient claude auth)" "--no-profile-fallback without provider suppresses auto-detection" +assert_not_contains "$no_profile_fallback_out" "Auto-detected" "--no-profile-fallback does not report auto-detection" + +echo "not-json" > "$TMPDIR/malformed-profiles.json" +: > "$TMPDIR/claude-success.log" +malformed_out=$( + unset CLAUDE_CODE_OAUTH_TOKEN + FOREMAN_CLAUDE_PROFILES_FILE="$TMPDIR/malformed-profiles.json" \ + FAKE_PERSONAL_TOKEN="personal-token" \ + FAKE_WORK_TOKEN="work-token" \ + run_dispatch success "$TMPDIR/target" "malformed profile prompt" --max-turns 3 +) +assert_contains "$malformed_out" "Auth lane: inherited (ambient claude auth)" "malformed profiles file falls back to ambient by default" + +echo "" +echo "[10] Claude profile fallback" +cat > "$TMPDIR/fallback-profiles.json" <<'JSON' +{ + "active": "personal", + "profiles": { + "personal": { + "label": "Fake Personal", + "env_var": "FAKE_PERSONAL_TOKEN", + "cooldown_until": 0 + }, + "work": { + "label": "Fake Work", + "env_var": "FAKE_WORK_TOKEN", + "cooldown_until": 0 + } + } +} +JSON + +: > "$TMPDIR/claude-rate_limit_personal.log" +fallback_out=$( + FOREMAN_CLAUDE_PROFILES_FILE="$TMPDIR/fallback-profiles.json" \ + FOREMAN_CLAUDE_PROFILE_COOLDOWN_SECONDS=600 \ + FAKE_PERSONAL_TOKEN="personal-token" \ + FAKE_WORK_TOKEN="work-token" \ + run_dispatch rate_limit_personal "$TMPDIR/target" "fallback prompt" --provider claude-cli --max-turns 3 +) +assert_contains "$fallback_out" "Auth lane: claude-cli fallback (personal -> work)" "reports profile fallback order" +assert_contains "$fallback_out" "Auth attempt 1/2: personal" "prints first auth attempt" +assert_contains "$fallback_out" "Auth attempt 2/2: work" "prints second auth attempt" +assert_contains "$fallback_out" "retryable quota signal" "reports retryable quota fallback" +assert_contains "$fallback_out" "FOREMAN_FALLBACK_OK" "falls through to second profile result" +assert_contains "$(cat "$TMPDIR/claude-rate_limit_personal.log")" "TOKEN=personal-token" "tries active personal token first" +assert_contains "$(cat "$TMPDIR/claude-rate_limit_personal.log")" "TOKEN=work-token" "tries work token after personal quota" +cooldown_check=$( + python3 - "$TMPDIR/fallback-profiles.json" <<'PY' +import json, sys, time +data = json.load(open(sys.argv[1])) +cooldown = int(data["profiles"]["personal"].get("cooldown_until") or 0) +print("cooldown-set" if cooldown > int(time.time()) else "cooldown-missing") +PY +) +assert_contains "$cooldown_check" "cooldown-set" "records cooldown on failed profile" + +cat > "$TMPDIR/strict-profiles.json" <<'JSON' +{ + "active": "personal", + "profiles": { + "personal": { + "label": "Fake Personal", + "env_var": "FAKE_PERSONAL_TOKEN", + "cooldown_until": 0 + }, + "work": { + "label": "Fake Work", + "env_var": "FAKE_WORK_TOKEN", + "cooldown_until": 0 + } + } +} +JSON + +: > "$TMPDIR/claude-rate_limit_personal.log" +strict_out=$( + FOREMAN_CLAUDE_PROFILES_FILE="$TMPDIR/strict-profiles.json" \ + FAKE_PERSONAL_TOKEN="personal-token" \ + FAKE_WORK_TOKEN="work-token" \ + run_dispatch rate_limit_personal "$TMPDIR/target" "strict prompt" --profile personal --max-turns 3 || true +) +assert_contains "$strict_out" "Auth lane: claude-cli (personal; env \$FAKE_PERSONAL_TOKEN)" "explicit profile remains strict" +assert_contains "$strict_out" "You've hit your session limit" "strict profile surfaces original quota result" +assert_not_contains "$(cat "$TMPDIR/claude-rate_limit_personal.log")" "TOKEN=work-token" "strict profile does not try fallback token" + +echo "" +echo "=== Results: $PASS passed, $FAIL failed ===" + +if [[ "$FAIL" -gt 0 ]]; then + exit 1 +fi diff --git a/skills/moltmaster/SKILL.md b/skills/moltmaster/SKILL.md index 38c1999..0d48c74 100644 --- a/skills/moltmaster/SKILL.md +++ b/skills/moltmaster/SKILL.md @@ -1,6 +1,10 @@ --- name: moltmaster -description: Use for OpenClaw OAuth / auth-profile refresh operations, Auth Molt, Moltmaster, Codex OAuth refresh, Claude/Anthropic OAuth profile refresh planning, or any task involving refreshing, inspecting, or pruning OpenClaw auth-profile credentials without running openclaw doctor. Triggers on Auth Molt, Moltmaster, Codex OAuth refresh, auth-profile refresh, force-expired-for-refresh, openclaw credential rotation. +description: > + Use for OpenClaw OAuth profile refresh operations, Auth Molt, Moltmaster, + Codex OAuth refresh, Claude or Anthropic OAuth profile refresh planning, or + tasks involving OpenClaw auth profile inspection, pruning, or refresh without + running openclaw doctor. --- # Moltmaster @@ -72,6 +76,7 @@ Covers: dry-run, flag refusals (8 cases), profile validation, prune dry-run. For | Variable | Purpose | |---|---| +| `AUTH_MOLT_SDK_PATH` / `MOLTMASTER_SDK_PATH` | Override OpenClaw plugin SDK runtime path | | `AUTH_MOLT_STORE_PATH` | Override auth store path | | `AUTH_MOLT_BACKUP_DIR` | Override backup directory | | `AUTH_MOLT_STATE_FILE` | Override cooldown state file | diff --git a/skills/moltmaster/scripts/moltmaster.mjs b/skills/moltmaster/scripts/moltmaster.mjs index 69f4672..4772378 100644 --- a/skills/moltmaster/scripts/moltmaster.mjs +++ b/skills/moltmaster/scripts/moltmaster.mjs @@ -7,13 +7,22 @@ import path from 'node:path'; import process from 'node:process'; const __dirname = path.dirname(new URL(import.meta.url).pathname); -const SDK_CANDIDATE_PATHS = [ - '/usr/local/lib/node_modules/openclaw/dist/plugin-sdk/agent-runtime.js', - '/usr/lib/node_modules/openclaw/dist/plugin-sdk/agent-runtime.js', -]; -const SDK_PATH = process.env.AUTH_MOLT_SDK_PATH - ?? SDK_CANDIDATE_PATHS.find((candidate) => fs.existsSync(candidate)) - ?? SDK_CANDIDATE_PATHS[0]; + +function resolveSdkPath() { + const sdkSuffix = 'openclaw/dist/plugin-sdk/agent-runtime.js'; + const candidates = [ + process.env.AUTH_MOLT_SDK_PATH, + process.env.MOLTMASTER_SDK_PATH, + process.env.HOMEBREW_PREFIX ? path.join(process.env.HOMEBREW_PREFIX, 'lib', 'node_modules', sdkSuffix) : null, + '/opt/homebrew/lib/node_modules/openclaw/dist/plugin-sdk/agent-runtime.js', + '/usr/local/lib/node_modules/openclaw/dist/plugin-sdk/agent-runtime.js', + '/usr/lib/node_modules/openclaw/dist/plugin-sdk/agent-runtime.js', + ].filter(Boolean); + + return candidates.find((candidate) => fs.existsSync(candidate)) ?? candidates[0]; +} + +const SDK_PATH = resolveSdkPath(); const DEFAULT_STORE_PATH = path.join(os.homedir(), '.openclaw', 'agents', 'main', 'agent', 'auth-profiles.json'); const STORE_PATH = process.env.AUTH_MOLT_STORE_PATH ?? DEFAULT_STORE_PATH; const SKILL_DIR = path.resolve(__dirname, '..'); @@ -22,10 +31,12 @@ const STATE_FILE = process.env.AUTH_MOLT_STATE_FILE ?? path.join(SKILL_DIR, '.re const CODEX_PROFILE_ALLOWLIST = [ /^openai-codex:[^/\s]+@edge\.app$/, /^openai-codex:pearson\.jaredb@gmail\.com$/, + /^openai-codex:you@example\.com$/, ]; const CODEX_EMAIL_ALLOWLIST = [ /@edge\.app$/, /^pearson\.jaredb@gmail\.com$/, + /^you@example\.com$/, ]; const WARN_WITHIN_MS = 14 * 24 * 60 * 60 * 1000; const COOLDOWN_MS = 60_000; @@ -127,7 +138,7 @@ function validateTarget(profileId, profile) { if (!allowlistedProfileId(profileId)) throw new Error(`Refusing non-allowlisted profile id: ${profileId}`); if (!profile || profile.provider !== 'openai-codex') throw new Error(`Refusing non-Codex provider for ${profileId}`); if (profile.type !== 'oauth') throw new Error(`Refusing non-OAuth profile for ${profileId}: ${profile.type}`); - if (profile.email && !allowlistedEmail(profile.email)) throw new Error(`Refusing unexpected email for ${profileId}`); + if (profile.email && !allowlistedEmail(profile.email)) throw new Error(`Refusing unexpected email domain for ${profileId}`); } function ensureBackupDir() { diff --git a/skills/moltmaster/scripts/moltmaster.test.mjs b/skills/moltmaster/scripts/moltmaster.test.mjs index f6a0843..d21f1ae 100644 --- a/skills/moltmaster/scripts/moltmaster.test.mjs +++ b/skills/moltmaster/scripts/moltmaster.test.mjs @@ -199,7 +199,7 @@ run('non-edge.app email rejected', ['--dry-run', '--profile', 'openai-codex:bade storePath: mixedStorePath, ...testEnv(), expectExit: 1, - expectStderr: ['unexpected email'], + expectStderr: ['unexpected email domain'], }); run('non-allowlisted profile ID rejected', ['--dry-run', '--profile', 'anthropic:badid'], { storePath: mixedStorePath, diff --git a/skills/shell-swap/.github/CODEOWNERS b/skills/shell-swap/.github/CODEOWNERS new file mode 100644 index 0000000..37028cf --- /dev/null +++ b/skills/shell-swap/.github/CODEOWNERS @@ -0,0 +1 @@ +* @clawSean diff --git a/skills/shell-swap/BASELINE_TEST_AUDIT.md b/skills/shell-swap/BASELINE_TEST_AUDIT.md new file mode 100644 index 0000000..7791001 --- /dev/null +++ b/skills/shell-swap/BASELINE_TEST_AUDIT.md @@ -0,0 +1,32 @@ +# Baseline Test Audit — shell-swap + +## What exists + +`scripts/test.sh` — 71 offline assertions covering: + +1. **SKILL.md structure** — file exists, has YAML frontmatter with `name` and `description` +2. **Script validity** — `switch.sh` exists and passes `bash -n` syntax check +3. **Model resolution** — configured aliases, full ids, raw provider/model ids, multi-segment providers, and provider/model ambiguity rejection +4. **Session rewrite correctness** — model/provider overrides are stamped together, stale fallback fields are removed, `auto` sessions are preserved, nested report/origin objects are untouched, and codex harness pins are cleared when switching out of codex lanes +5. **Scoping** — `--agent`, `--agent current`, unknown agent rejection, and fleet-primary updates +6. **Cron mutation** — legacy cron updates are opt-in and malformed stores abort before writes +7. **Safety** — backups, atomic writes, malformed-store preflight, dry-run no-write behavior, and missing-config failures +8. **Session override modes** — `--think`, `--fast`, `default` clearing, `gateway` dry-run, `offline` edits, and combined model+override runs + +## How to run + +```bash +bash scripts/test.sh +``` + +## Last run + +- **Date:** 2026-05-09 +- **Result:** 71/71 passed, 0 failed +- **Environment:** Linux, bash, python3 + +## Remaining gaps + +- No live Gateway `sessions.patch` call in the hermetic suite; gateway mode is dry-run tested and offline mode is fully file-tested +- No full end-to-end slash-command test from Telegram/chat command to script execution +- No cron-store migration support beyond the legacy `cron/jobs.json` path diff --git a/skills/shell-swap/RESEARCH-harness-pin-gap.md b/skills/shell-swap/RESEARCH-harness-pin-gap.md new file mode 100644 index 0000000..2192983 --- /dev/null +++ b/skills/shell-swap/RESEARCH-harness-pin-gap.md @@ -0,0 +1,100 @@ +# Research: The Harness-Pin Gap (and the restart requirement) + +*Logged 2026-06-24 after a live incident on JPop's Telegram DM. Status: root cause +confirmed; one mechanism (restart requirement) reasoned, not yet source-proven — +flagged below as an open question.* + +## TL;DR + +Shell-swap stamps `{model, provider}` across config + every session store, and that +works for normal switches. But it does **not** touch a deeper, third field: +`agentHarnessId` (the execution-harness pin). When a session is pinned to a harness +whose provider is dead/over-quota (our case: `codex`), swapping the model leaves the +session still bound to the dead harness — so it keeps failing even though `status` +shows the new model. Neither shell-swap nor an in-chat `/model` command clears that +pin. The only fix was an out-of-band edit to the session store, which then required a +gateway restart to take effect. + +## What actually happened + +1. JPop's DM (`agent:mainelobster:telegram:direct:6566057320`) was pinned to + `agentHarnessId: "codex"`. +2. The Codex provider was over usage limit → every turn errored with "codex out of usage". +3. A fleet shell-swap to `claude-cli/claude-sonnet-4-6` updated the model everywhere. + `status` reported Sonnet. **The DM still threw the Codex error.** +4. Running the per-session model-setter (programmatic equivalent of `/model`) also did + not fix it. Checked the store afterward: `agentHarnessId: "codex"` was still present. +5. Fix that worked: cleared the codex harness pins directly in the session store, then + **restarted the gateway**. DM then resolved to `claude-cli/sonnet` cleanly. + +## Why the model swap wasn't enough — the three layers + +A session's effective runtime is resolved from (at least) three distinct fields: + +| Layer | Field(s) | Written by shell-swap? | Written by `/model`? | +|-------|----------|------------------------|----------------------| +| Model | `model`, `modelOverride` | ✅ yes | ✅ yes | +| Provider | `modelProvider`, `providerOverride` | ✅ yes | ✅ yes | +| **Harness pin** | `agentHarnessId` | ❌ **no** | ❌ **no** | + +The harness pin sits *underneath* model/provider. If it points at a dead harness, the +session is stuck regardless of what model you select. This is the entire gap. + +## Verified evidence (this incident) + +- `grep -c agentHarnessId openclaw.json` → **0**. The pin was never in config; it lived + per-session in the session store (`sessions/openclaw.sqlite`). +- Diffing live `openclaw.json` against its `.bak` showed exactly **one** line changed: + `agents.defaults.model.primary` `openai/gpt-5.5` → `claude-cli/claude-sonnet-4-6`. + i.e. the config swap genuinely did not, and could not, address the pin. +- The pin clearing was done in the session store, not config. + +## Why the restart was needed — in-band vs out-of-band + +This is the conceptual key, and it generalizes: + +- **In-band edit (hot, no restart):** an in-chat `/model` command is processed by the + *live* session while it's awake handling the message. It rewrites its own state in RAM + and on disk in one step. The owner of the state makes the edit, so nothing is stale. +- **Out-of-band edit (cold, needs restart):** clearing the pin meant reaching into the + session store on disk *from outside* the live session. The DM was still holding the old + pin in the gateway's memory and had no idea the backing file changed. Only a restart + forces a fresh re-read. + +The kicker that makes the cold path unavoidable: the hot path (`/model`) only writes +model/provider, never `agentHarnessId`. There is **no in-chat command that clears a +harness pin**. So the out-of-band store edit was the only available route — and that +route is exactly the one that requires a restart. + +> **Open question (not yet source-proven):** the "live session holds state in RAM, an +> out-of-band disk edit is invisible until reload" model is the standard pattern and fits +> every observation here, but I have not traced the exact OpenClaw code path that loads +> and caches session harness bindings at boot. Worth confirming in source before treating +> as gospel. The bundled `dist/` is minified; check upstream source. + +## Future exploration / potential skill improvements + +1. **Harness-pin awareness in shell-swap.** When the resolved target's runtime differs + from a session's existing `agentHarnessId`, the swap arguably should clear or re-stamp + the pin (it's the whole reason a "successful" swap can leave a session broken). Options: + - Auto-detect runtime change and clear/repair `agentHarnessId` as part of the stamp. + - Add an opt-in `--clear-harness-pin` (or `--repair-harness`) flag for the + escape-a-dead-harness case, kept off by default to avoid surprising behavior. +2. **Session-store target discrepancy.** SKILL.md step 2 targets + `agents/*/sessions/sessions.json`, but the live pin this incident touched was in + `sessions/openclaw.sqlite`. Confirm which store is authoritative on the current + OpenClaw build and whether the script is even walking the right file for harness data. + If the store migrated to sqlite, the script's session walk may be stale. +3. **Restart-required signalling.** Because clearing a pin is inherently out-of-band, the + skill should explicitly tell the operator a restart is mandatory after a harness-pin + repair (vs. a normal model swap, where sessions pick up the new model on next turn). +4. **Upstream schema fix candidate.** Consider whether OpenClaw should expose an in-band + way to clear/override a harness pin (a `/harness` command or `/model` clearing a stale + pin when the runtime changes), which would remove the need for out-of-band edits + + restarts entirely. Possible contribution lane. + +## Related + +- Memory note: `shell-swap-codex-harness-pin-gap` (auto-memory index). +- SKILL.md "What it does" step 2 (model/provider stamping) — the place a pin-aware step + would slot in. diff --git a/skills/shell-swap/scripts/test.sh b/skills/shell-swap/scripts/test.sh new file mode 100755 index 0000000..f2f713a --- /dev/null +++ b/skills/shell-swap/scripts/test.sh @@ -0,0 +1,324 @@ +#!/usr/bin/env bash +# Hermetic regression suite for shell-swap/scripts/switch.sh. +# +# Builds a synthetic OpenClaw dir (config + multi-agent session stores) in a +# temp dir, runs switch.sh against it via OPENCLAW_DIR, and asserts behaviour. +# No dependency on the live ~/.openclaw. Run: bash scripts/test.sh + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SWITCH="$SCRIPT_DIR/switch.sh" +PASS=0 +FAIL=0 + +red() { printf '\033[31m%s\033[0m\n' "$1"; } +grn() { printf '\033[32m%s\033[0m\n' "$1"; } + +ok() { PASS=$((PASS+1)); grn " ok - $1"; } +bad() { FAIL=$((FAIL+1)); red " FAIL - $1"; [[ -n "${2:-}" ]] && echo " $2"; } + +# assert_eq +assert_eq() { [[ "$2" == "$3" ]] && ok "$1" || bad "$1" "expected [$2] got [$3]"; } +# assert_contains +assert_contains() { [[ "$3" == *"$2"* ]] && ok "$1" || bad "$1" "missing [$2] in output"; } +# assert_exit +assert_exit() { [[ "$2" == "$3" ]] && ok "$1" || bad "$1" "expected exit $2 got $3"; } + +# jq-free JSON probe via python +jget() { python3 -c "import json,sys; d=json.load(open(sys.argv[1])) +exec('v=d'+sys.argv[2]); print(v)" "$1" "$2" 2>/dev/null; } + +make_fixture() { + local root="$1" + mkdir -p "$root/agents/mainelobster/sessions" \ + "$root/agents/clawdia/sessions" \ + "$root/agents/empty/sessions" + cat > "$root/openclaw.json" <<'JSON' +{ + "agents": { + "defaults": { + "model": { "primary": "openai/gpt-5.5", "fallbacks": [] }, + "models": { + "openai/gpt-5.5": { "alias": "gpt" }, + "claude-cli/claude-opus-4-8": { "alias": "opus", "agentRuntime": { "id": "claude-cli" } }, + "anthropic/claude-opus-4-6": { "alias": "opus-4.6", "agentRuntime": { "id": "claude-cli" } }, + "anthropic/claude-opus-4-8": { "alias": "opus-8", "agentRuntime": { "id": "openclaw" } }, + "openai/gpt-5.5-codex": { "alias": "gpt-codex", "agentRuntime": { "id": "codex" } }, + "venice/minimax-m25": { "alias": "minimax" }, + "nvidia/moonshotai/kimi-k2.5": { "alias": "kimi" }, + "xai/grok-4.3": { "alias": "Grok" }, + "ollama/phi4-mini": {} + } + } + } +} +JSON + + # mainelobster: a mix of states the rewrite must handle correctly. + cat > "$root/agents/mainelobster/sessions/sessions.json" <<'JSON' +{ + "agent:mainelobster:s1": { + "model": "gpt-5.5", "modelProvider": "openai", "modelOverrideSource": "auto", + "thinkingLevel": "off", "fastMode": false, + "systemPromptReport": { "model": "gpt-5.5", "provider": "openai" }, + "contextBudgetStatus": { "model": "gpt-5.5" }, + "origin": { "provider": "telegram" } + }, + "agent:mainelobster:s2_diverged": { + "model": "claude-opus-4-8", "modelProvider": "anthropic", + "modelOverride": "claude-opus-4-8", "providerOverride": "anthropic", + "modelOverrideSource": "user", "thinkingLevel": "high" + }, + "agent:mainelobster:s3_auto": { + "model": "auto", "modelProvider": "nadirclaw" + }, + "agent:mainelobster:s4_fallback": { + "model": "grok-4.3", "modelProvider": "xai", "modelOverrideSource": "auto", + "modelOverrideFallbackOriginProvider": "anthropic", + "modelOverrideFallbackOriginModel": "claude-opus-4-6", + "modelOverrideFallbackNotice": "fell back due to billing" + }, + "agent:mainelobster:s5_ontarget": { + "model": "claude-opus-4-8", "modelProvider": "claude-cli", "modelOverrideSource": "user" + }, + "agent:mainelobster:s6_codexpin": { + "model": "gpt-5.5", "modelProvider": "openai", "modelOverrideSource": "user", + "agentHarnessId": "codex", "agentRuntimeOverride": "codex", "liveModelSwitchPending": true + } +} +JSON + + cat > "$root/agents/clawdia/sessions/sessions.json" <<'JSON' +{ + "agent:clawdia:s1": { "model": "gpt-5.5", "modelProvider": "openai", "modelOverrideSource": "auto", "thinkingLevel": "low" } +} +JSON + + echo '{}' > "$root/agents/empty/sessions/sessions.json" +} + +run() { OPENCLAW_DIR="$ROOT" bash "$SWITCH" "$@" 2>&1; } + +# --------------------------------------------------------------------------- +echo "== resolution ==" +ROOT="$(mktemp -d)"; make_fixture "$ROOT" + +out="$(run opus --dry-run)" +assert_contains "alias opus -> full id" "Resolved full id : claude-cli/claude-opus-4-8" "$out" +assert_contains "alias opus -> model" "Session model : claude-opus-4-8" "$out" +assert_contains "alias opus -> provider" "Session provider : claude-cli" "$out" + +out="$(run opus-4.6 --dry-run)" +assert_contains "agentRuntime: opus-4.6 provider is claude-cli not anthropic" "Session provider : claude-cli" "$out" + +out="$(run opus-8 --dry-run)" +assert_contains "agentRuntime: opus-8 provider is openclaw" "Session provider : openclaw" "$out" + +out="$(run xai/grok-4.3 --dry-run)" +assert_contains "full id passthrough provider" "Session provider : xai" "$out" +assert_contains "full id passthrough model" "Session model : grok-4.3" "$out" + +out="$(run kimi --dry-run)" +assert_contains "multi-slash provider" "Session provider : nvidia" "$out" +assert_contains "multi-slash model" "Session model : moonshotai/kimi-k2.5" "$out" + +out="$(run minimax --dry-run)" +assert_contains "cross-provider alias (venice)" "Session provider : venice" "$out" + +run gpt-5.5 --dry-run >/dev/null 2>&1; assert_exit "bare model rejected" 3 "$?" +run anything/at-all-novel --dry-run >/dev/null 2>&1; assert_exit "novel provider/model passes (exit 0)" 0 "$?" +run --think big --dry-run >/dev/null 2>&1; assert_exit "invalid think rejected" 1 "$?" +run --fast turbo --dry-run >/dev/null 2>&1; assert_exit "invalid fast rejected" 1 "$?" +rm -rf "$ROOT" + +# --------------------------------------------------------------------------- +echo "== session rewrite correctness (real run) ==" +ROOT="$(mktemp -d)"; make_fixture "$ROOT" +S="$ROOT/agents/mainelobster/sessions/sessions.json" +run opus >/dev/null + +assert_eq "s1 model switched" "claude-opus-4-8" "$(jget "$S" "['agent:mainelobster:s1']['model']")" +assert_eq "s1 provider stamped to runtime" "claude-cli" "$(jget "$S" "['agent:mainelobster:s1']['modelProvider']")" +assert_eq "s1 source flipped (model changed)" "user" "$(jget "$S" "['agent:mainelobster:s1']['modelOverrideSource']")" +assert_eq "nested systemPromptReport.model UNTOUCHED" "gpt-5.5" "$(jget "$S" "['agent:mainelobster:s1']['systemPromptReport']['model']")" +assert_eq "nested contextBudgetStatus.model UNTOUCHED" "gpt-5.5" "$(jget "$S" "['agent:mainelobster:s1']['contextBudgetStatus']['model']")" +assert_eq "nested origin.provider UNTOUCHED" "telegram" "$(jget "$S" "['agent:mainelobster:s1']['origin']['provider']")" + +assert_eq "s2 diverged provider REPAIRED" "claude-cli" "$(jget "$S" "['agent:mainelobster:s2_diverged']['modelProvider']")" +assert_eq "s2 diverged providerOverride REPAIRED" "claude-cli" "$(jget "$S" "['agent:mainelobster:s2_diverged']['providerOverride']")" + +assert_eq "s3 auto model PRESERVED" "auto" "$(jget "$S" "['agent:mainelobster:s3_auto']['model']")" +assert_eq "s3 auto provider PRESERVED" "nadirclaw" "$(jget "$S" "['agent:mainelobster:s3_auto']['modelProvider']")" + +assert_eq "s4 stale fallback origin removed" "None" "$(jget "$S" ".get('agent:mainelobster:s4_fallback',{}).get('modelOverrideFallbackOriginProvider','None')" 2>/dev/null || echo None)" + +# s5 was already exactly on target -> source must NOT be touched (provenance) +assert_eq "s5 on-target source preserved" "user" "$(jget "$S" "['agent:mainelobster:s5_ontarget']['modelOverrideSource']")" +rm -rf "$ROOT" + +# verify stale-field removal more directly +ROOT="$(mktemp -d)"; make_fixture "$ROOT"; S="$ROOT/agents/mainelobster/sessions/sessions.json" +run opus >/dev/null +hasstale="$(python3 -c "import json;d=json.load(open('$S'));s=d['agent:mainelobster:s4_fallback'];print(any(k.startswith('modelOverrideFallback') for k in s))")" +assert_eq "s4 all stale fallback fields gone" "False" "$hasstale" +rm -rf "$ROOT" + +# --------------------------------------------------------------------------- +echo "== harness/runtime pin clearing (codex deadlock fix) ==" +# Switching a codex-pinned session to a non-codex lane must DELETE the stale +# agentHarnessId / agentRuntimeOverride / liveModelSwitchPending pins, else the +# session keeps routing to the dead codex harness and deadlocks. +ROOT="$(mktemp -d)"; make_fixture "$ROOT"; S="$ROOT/agents/mainelobster/sessions/sessions.json" +run opus >/dev/null +assert_eq "codex->claude clears agentHarnessId" "None" "$(jget "$S" ".get('agent:mainelobster:s6_codexpin',{}).get('agentHarnessId','None')")" +assert_eq "codex->claude clears agentRuntimeOverride" "None" "$(jget "$S" ".get('agent:mainelobster:s6_codexpin',{}).get('agentRuntimeOverride','None')")" +assert_eq "codex->claude clears liveModelSwitchPending" "None" "$(jget "$S" ".get('agent:mainelobster:s6_codexpin',{}).get('liveModelSwitchPending','None')")" +assert_eq "codex->claude switched model" "claude-opus-4-8" "$(jget "$S" "['agent:mainelobster:s6_codexpin']['model']")" +rm -rf "$ROOT" + +# Switching INTO a codex lane (provider resolves to agentRuntime.id "codex") +# must KEEP the harness pin (it's valid there) but still resolve the pending +# live-switch flag, since file-surgery hard-sets the model. +ROOT="$(mktemp -d)"; make_fixture "$ROOT"; S="$ROOT/agents/mainelobster/sessions/sessions.json" +run gpt-codex >/dev/null +assert_eq "->codex keeps agentHarnessId" "codex" "$(jget "$S" ".get('agent:mainelobster:s6_codexpin',{}).get('agentHarnessId','None')")" +assert_eq "->codex still clears liveModelSwitchPending" "None" "$(jget "$S" ".get('agent:mainelobster:s6_codexpin',{}).get('liveModelSwitchPending','None')")" +rm -rf "$ROOT" + +# --------------------------------------------------------------------------- +echo "== scoping ==" +ROOT="$(mktemp -d)"; make_fixture "$ROOT" +run bogus-alias-xyz --agent mainelobster --dry-run >/dev/null 2>&1 +assert_exit "bogus alias rejected before agent handling" 3 "$?" +run opus --agent mainelobster >/dev/null +assert_eq "scoped run leaves config primary" "openai/gpt-5.5" "$(jget "$ROOT/openclaw.json" "['agents']['defaults']['model']['primary']")" +assert_eq "scoped run switched target agent" "claude-opus-4-8" "$(jget "$ROOT/agents/mainelobster/sessions/sessions.json" "['agent:mainelobster:s1']['model']")" +assert_eq "scoped run left OTHER agent alone" "gpt-5.5" "$(jget "$ROOT/agents/clawdia/sessions/sessions.json" "['agent:clawdia:s1']['model']")" +rm -rf "$ROOT" + +ROOT="$(mktemp -d)"; make_fixture "$ROOT" +run opus --agent ghost >/dev/null 2>&1; assert_exit "unknown agent hard-errors" 4 "$?" +assert_eq "unknown agent made NO config change" "openai/gpt-5.5" "$(jget "$ROOT/openclaw.json" "['agents']['defaults']['model']['primary']")" +rm -rf "$ROOT" + +ROOT="$(mktemp -d)"; make_fixture "$ROOT" +out="$(OPENCLAW_DIR="$ROOT" OPENCLAW_MCP_AGENT_ID=mainelobster bash "$SWITCH" opus --agent current --dry-run 2>&1)" +assert_contains "--agent current resolves env agent" "Agent scope : mainelobster" "$out" +rm -rf "$ROOT" + +echo "== fleet run updates primary ==" +ROOT="$(mktemp -d)"; make_fixture "$ROOT" +run opus >/dev/null +assert_eq "fleet run updates config primary" "claude-cli/claude-opus-4-8" "$(jget "$ROOT/openclaw.json" "['agents']['defaults']['model']['primary']")" +assert_eq "allowlist NOT clobbered (still 9 entries)" "9" "$(jget "$ROOT/openclaw.json" "['agents']['defaults']['models'].__len__()")" +rm -rf "$ROOT" + +# --------------------------------------------------------------------------- +echo "== cron (opt-in) ==" +ROOT="$(mktemp -d)"; make_fixture "$ROOT" +mkdir -p "$ROOT/cron" +cat > "$ROOT/cron/jobs.json" <<'JSON' +{ "jobs": [ + { "name": "daily-digest", "payload": { "model": "gpt-5.5" } }, + { "name": "already-target", "payload": { "model": "claude-cli/claude-opus-4-8" } } +] } +JSON +run opus --crons >/dev/null +assert_eq "cron payload rewritten to full id" "claude-cli/claude-opus-4-8" "$(jget "$ROOT/cron/jobs.json" "['jobs'][0]['payload']['model']")" +[[ -f "$ROOT/cron/jobs.json.bak" ]] && ok "cron backup created" || bad "cron backup created" +rm -rf "$ROOT" + +# without --crons the cron file is untouched +ROOT="$(mktemp -d)"; make_fixture "$ROOT"; mkdir -p "$ROOT/cron" +echo '{ "jobs": [ { "name": "j", "payload": { "model": "gpt-5.5" } } ] }' > "$ROOT/cron/jobs.json" +run opus >/dev/null +assert_eq "cron untouched without --crons" "gpt-5.5" "$(jget "$ROOT/cron/jobs.json" "['jobs'][0]['payload']['model']")" +rm -rf "$ROOT" + +# malformed cron with --crons must abort in pre-flight BEFORE any write +ROOT="$(mktemp -d)"; make_fixture "$ROOT"; mkdir -p "$ROOT/cron" +echo '{ broken cron' > "$ROOT/cron/jobs.json" +run opus --crons >/dev/null 2>&1; assert_exit "malformed cron aborts (exit 5)" 5 "$?" +assert_eq "cron-abort left config primary unchanged" "openai/gpt-5.5" "$(jget "$ROOT/openclaw.json" "['agents']['defaults']['model']['primary']")" +assert_eq "cron-abort left sessions unchanged" "gpt-5.5" "$(jget "$ROOT/agents/mainelobster/sessions/sessions.json" "['agent:mainelobster:s1']['model']")" +rm -rf "$ROOT" + +# --------------------------------------------------------------------------- +echo "== safety: backups, atomicity, pre-validation, dry-run ==" +ROOT="$(mktemp -d)"; make_fixture "$ROOT" +run opus >/dev/null +[[ -f "$ROOT/openclaw.json.bak" ]] && ok "config backup created" || bad "config backup created" +[[ -f "$ROOT/agents/mainelobster/sessions/sessions.json.bak" ]] && ok "sessions backup created" || bad "sessions backup created" +leftover="$(find "$ROOT" -name '.shellswap-*' | wc -l | tr -d ' ')" +assert_eq "no leftover tmp files (atomic)" "0" "$leftover" +python3 -c "import json;json.load(open('$ROOT/agents/mainelobster/sessions/sessions.json'))" 2>/dev/null && ok "result is valid JSON" || bad "result is valid JSON" +rm -rf "$ROOT" + +# pre-validation: a malformed store aborts before ANY write +ROOT="$(mktemp -d)"; make_fixture "$ROOT" +echo '{ this is not json' > "$ROOT/agents/clawdia/sessions/sessions.json" +run opus >/dev/null 2>&1; assert_exit "malformed store aborts (exit 5)" 5 "$?" +assert_eq "abort left config primary unchanged" "openai/gpt-5.5" "$(jget "$ROOT/openclaw.json" "['agents']['defaults']['model']['primary']")" +assert_eq "abort left good store unchanged" "gpt-5.5" "$(jget "$ROOT/agents/mainelobster/sessions/sessions.json" "['agent:mainelobster:s1']['model']")" +rm -rf "$ROOT" + +# dry-run writes nothing +ROOT="$(mktemp -d)"; make_fixture "$ROOT" +before="$(md5sum "$ROOT/openclaw.json" "$ROOT/agents/mainelobster/sessions/sessions.json")" +run opus --dry-run >/dev/null +after="$(md5sum "$ROOT/openclaw.json" "$ROOT/agents/mainelobster/sessions/sessions.json")" +assert_eq "dry-run modifies nothing" "$before" "$after" +[[ -f "$ROOT/openclaw.json.bak" ]] && bad "dry-run must not write backups" || ok "dry-run writes no backups" +rm -rf "$ROOT" + +# --------------------------------------------------------------------------- +echo "== session override modes ==" +ROOT="$(mktemp -d)"; make_fixture "$ROOT" +S="$ROOT/agents/mainelobster/sessions/sessions.json" +CS="$ROOT/agents/clawdia/sessions/sessions.json" +run --think high --fast auto --session-mode offline >/dev/null +assert_eq "mode-only does not change config primary" "openai/gpt-5.5" "$(jget "$ROOT/openclaw.json" "['agents']['defaults']['model']['primary']")" +assert_eq "offline think set target agent" "high" "$(jget "$S" "['agent:mainelobster:s1']['thinkingLevel']")" +assert_eq "offline fast auto set" "auto" "$(jget "$S" "['agent:mainelobster:s1']['fastMode']")" +assert_eq "offline think set other agent by default" "high" "$(jget "$CS" "['agent:clawdia:s1']['thinkingLevel']")" +rm -rf "$ROOT" + +ROOT="$(mktemp -d)"; make_fixture "$ROOT" +S="$ROOT/agents/mainelobster/sessions/sessions.json" +CS="$ROOT/agents/clawdia/sessions/sessions.json" +run --think default --fast default --agent mainelobster --session-mode offline >/dev/null +assert_eq "default clears thinking override" "None" "$(jget "$S" ".get('agent:mainelobster:s1',{}).get('thinkingLevel','None')")" +assert_eq "default clears fast override" "None" "$(jget "$S" ".get('agent:mainelobster:s1',{}).get('fastMode','None')")" +assert_eq "scoped override leaves other agent alone" "low" "$(jget "$CS" "['agent:clawdia:s1']['thinkingLevel']")" +rm -rf "$ROOT" + +ROOT="$(mktemp -d)"; make_fixture "$ROOT" +out="$(run --think max --dry-run)" +assert_contains "gateway dry-run used for warm-safe mode" "would patch 7 session(s) through Gateway sessions.patch" "$out" +assert_eq "gateway dry-run leaves config primary" "openai/gpt-5.5" "$(jget "$ROOT/openclaw.json" "['agents']['defaults']['model']['primary']")" +rm -rf "$ROOT" + +ROOT="$(mktemp -d)"; make_fixture "$ROOT" +S="$ROOT/agents/mainelobster/sessions/sessions.json" +run opus --think x-high --fast on --agent mainelobster --session-mode offline >/dev/null +assert_eq "model+think normalized x-high" "xhigh" "$(jget "$S" "['agent:mainelobster:s1']['thinkingLevel']")" +assert_eq "model+fast normalized on" "True" "$(jget "$S" "['agent:mainelobster:s1']['fastMode']")" +assert_eq "model still switched with overrides" "claude-opus-4-8" "$(jget "$S" "['agent:mainelobster:s1']['model']")" +rm -rf "$ROOT" + +# missing config +ROOT="$(mktemp -d)" +run opus --dry-run >/dev/null 2>&1; assert_exit "missing config clean exit 1" 1 "$?" +rm -rf "$ROOT" + +# --------------------------------------------------------------------------- +echo "" +echo "== bash -n syntax ==" +bash -n "$SWITCH" && ok "switch.sh parses" || bad "switch.sh parses" + +echo "" +echo "================================" +echo " PASS: $PASS FAIL: $FAIL" +echo "================================" +[[ "$FAIL" -eq 0 ]] diff --git a/skills/web-use/BASELINE_TEST_AUDIT.md b/skills/web-use/BASELINE_TEST_AUDIT.md new file mode 100644 index 0000000..4a47990 --- /dev/null +++ b/skills/web-use/BASELINE_TEST_AUDIT.md @@ -0,0 +1,24 @@ +# Baseline Test Audit - web-use + +## Scope + +Baseline structural checks for the local `web-use` skill. + +## Current Checks + +Run: + +```bash +bash skills/web-use/scripts/test.sh +``` + +The test script verifies: + +- frontmatter name matches `web-use` +- Browserless extraction helper parses and exposes `--help` without credentials +- Browserless session helper parses and exposes `--help` without credentials +- Browserless media-request helper parses and exposes `--help` without credentials +- TinyFish helper parses and exposes `--help` without optional dependencies + +Credentialed live extraction tests are intentionally not part of baseline because +they consume external service credits and depend on target-site behavior. diff --git a/skills/web-use/ROADMAP.md b/skills/web-use/ROADMAP.md new file mode 100644 index 0000000..6379030 --- /dev/null +++ b/skills/web-use/ROADMAP.md @@ -0,0 +1,15 @@ +# ROADMAP - web-use + +## Browser Automation Consolidation + +- [x] Consolidate browser-control routing, tab hygiene, stale ref recovery, and + login/manual-blocker guidance into `SKILL.md`. +- [x] Keep `web-use` as the single model-visible local source of truth for + web/browser work. +- [x] Remove the local `plugin-skills/browser-automation` symlink instead of + keeping a shim or duplicate skill entry. +- When OpenClaw's bundled native `browser-automation` skill changes upstream, + compare it as reference material and fold useful updates into `web-use` + instead of re-adding a companion skill. +- Add a smoke test playbook for a harmless page navigation and screenshot/check + flow. diff --git a/skills/web-use/SKILL.md b/skills/web-use/SKILL.md index 4f50beb..4c36b2d 100644 --- a/skills/web-use/SKILL.md +++ b/skills/web-use/SKILL.md @@ -1,6 +1,12 @@ --- name: "web-use" -description: "Use for any task that touches a web page or web data. Routes between web_search/web_fetch, Browserless/TinyFish/protected extraction, OpenClaw browser, and live browser context for login, 2FA, CAPTCHA, current tabs, carts, or checkout. Read before declaring any page blocked — Cloudflare, CAPTCHA, and \"Just a moment\" pages get Browserless /unblock first. Domain skills keep site-specific policy." +description: > + Default routing layer for all web work: pick the lightest path among + web_search, web_fetch, OpenClaw browser control, Browserless /unblock, and + TinyFish. Read before browsing, scraping, multi-step browser flows, or before + declaring any page blocked; Cloudflare, CAPTCHA, and "Just a moment" pages get + /unblock first. Covers live browser context for login, 2FA, CAPTCHA, current + tabs, carts, and checkout. --- # Web Use @@ -8,12 +14,32 @@ description: "Use for any task that touches a web page or web data. Routes betwe Use this as the default routing layer for web work. Decide the lightest dependable path before reaching for a browser or protected extraction backend. -This skill answers two separate questions: +This skill answers three separate questions: 1. **Data path:** how should the page or site data be retrieved? 2. **Browser context:** whose browser, device, tab, or login state matters? +3. **Browser control:** once the OpenClaw `browser` tool is the right path, how + should it be driven reliably? -Keep those questions separate even when one workflow needs both. +Keep these separate even when one workflow needs more than one. + +## Consolidation Note + +OpenClaw also ships a native bundled `browser-automation` skill under its +browser extension. In this workspace, do not expose that as a separate local +skill, shim, or `plugin-skills/browser-automation` symlink. The useful browser +control mechanics are consolidated here so web routing, browser context, and +browser operation have one model-visible source of truth. + +OpenClaw's skills CLI may recreate `~/.openclaw/plugin-skills/browser-automation` +while enumerating extra skills, even when `skills.entries.browser-automation.enabled=false`. +Treat that symlink as generated index state, not a source of truth. The durable +state is the config disable plus this consolidation note; if the symlink +reappears, leave it disabled or remove it again instead of using it. + +If the bundled skill changes upstream, compare it as reference material and +fold any useful updates into `web-use` instead of resurrecting a second local +skill entry. ## Quick Route @@ -22,26 +48,32 @@ Keep those questions separate even when one workflow needs both. | Search, discovery, quick source finding | `web_search` | | Read a simple public URL | `web_fetch` | | Inspect a JS-rendered page visually | OpenClaw `browser` | -| Extract protected or bot-gated data | `references/extraction-backends.md` | -| Use a current tab, login, 2FA, CAPTCHA, extension, cart, or checkout | `references/context-device.md` | +| Drive a multi-step browser flow, login, tabs, cart, checkout, recovery | OpenClaw `browser` — see Browser Tool Control | +| Extract protected or bot-gated data | Browserless `/unblock` first, then `references/extraction-backends.md` | +| Use a current tab, login, 2FA, CAPTCHA, extension, or account state | `references/context-device.md` | +| Resolve, download, or transcribe a video or media URL | `video` skill if present, else a local resolver (yt-dlp/ffprobe) then a headless-render fallback | | Site-specific paid API or domain policy | Relevant domain skill | +**Cloudflare / anti-bot stop rule:** a page showing "Just a moment", "Checking +your browser", "We'll have you designing again soon", CAPTCHA, Turnstile, +Access Denied, or similar bot-gate copy is not a final block. Run Browserless +`/unblock` before saying the page cannot be accessed. Only call it blocked after +`/unblock` has failed and login/current-browser escalation is not appropriate. + Do not use a browser when search or fetch is enough. Do not use Browserless, TinyFish, or paid APIs when a simple tool can finish the task. -## Logged-in sessions are NOT user-device-only (read this) +## Logged-In Sessions Are Not User-Device-Only -A logged-in browser session can live on the **agent side**, not just the user's -machine. The VPS managed `openclaw` browser (`profile="openclaw"`, `target="host"`, -CDP `:18800`, persistent `userDataDir`) can hold durable logins — e.g. it is -signed into Amazon as Jared — and is frequently the **primary** lane for -cart/checkout/order-history/account tasks. The user's own device browser -(Mac node) is often a **fallback** and may be logged into nothing. +A logged-in browser session can live on the agent side, not just the user's +machine. A managed OpenClaw browser profile can hold durable logins and may be +the primary lane for cart, checkout, order-history, and account tasks. The user's +own device browser is a fallback when the agent-side session is logged out or +the user wants to co-interact. -So when a task needs a logged-in session: check the agent-side managed browser -FIRST, verify login state on the live page, and only fall back to the user -device when the agent session is logged out or the user wants to co-interact. -Do not assume "logged in" ⇒ "the user's current browser." +When a task needs a logged-in session, check the agent-side managed browser +first, verify login state on the live page, then fall back to the user device if +needed. Do not assume "logged in" means "the user's current browser." ## Routing Rules @@ -50,7 +82,8 @@ Do not assume "logged in" ⇒ "the user's current browser." 3. Treat paid/API-credit backends as deliberate choices, not defaults. 4. Keep interactive human-visible browsing separate from server-side extraction. 5. Keep site-specific policy in the relevant domain skill. -6. A logged-in session may be agent-side; verify login state rather than routing by device assumption. +6. A logged-in session may be agent-side; verify login state instead of routing + by device assumption. ## Data Path Decision @@ -62,6 +95,9 @@ Use native OpenClaw tools first when they are enough: - `web_fetch` for a specific public URL that does not need JavaScript or login. - OpenClaw `browser` for simple visual inspection when rendering matters. +If any lightweight path lands on an anti-bot wall, immediately switch to +Browserless `/unblock`; do not treat the lightweight failure as terminal. + ### 2. Protected Server-Side Extraction Use Browserless when the core problem is getting through protection and @@ -71,6 +107,11 @@ extracting page data without a human-visible browser: - protected pages that a normal fetch or local headless browser cannot read - structured extraction from a hard page +For Cloudflare, Turnstile, CAPTCHA-like holding pages, or branded anti-bot pages, +prefer Browserless `/unblock` as the first server-side extraction mode. Use +`/stealth/bql` when you specifically need programmable BQL behavior or when +`/unblock` does not expose enough useful content. + Use Browserless `/session` sparingly. It is not the default for one-off reads. Use it only when repeated same-site BQL/CDP work genuinely benefits from persisted cookies, localStorage, sessionStorage, or cache. Session URLs include @@ -112,13 +153,98 @@ Plain-English labels: - On your device - On my side -Note: "On my side" (agent device) can be a **logged-in** session, not just a -fresh one. For account/cart/checkout work, prefer the agent-side logged-in -managed browser first when it holds the needed login. - Read `references/context-device.md` for the compact 2 x 2 browser-mode matrix and phrasing guide. +## Browser Tool Control + +Once the OpenClaw `browser` tool is the right path and the task is anything +beyond a single page check, use this loop. + +### Operating Loop + +1. Check browser state before acting: + - `openclaw browser doctor` or `action="status"` when the browser/plugin setup itself may be broken. + - `action="status"` for availability. + - `action="profiles"` if login state or profile choice matters. + - `action="tabs"` before opening a new tab if retries/timeouts may have left windows behind. +2. Prefer stable tab handles: + - Open important tabs with `label`, for example `label="meet"`. + - After `action="tabs"` or `action="open"`, store `suggestedTargetId` and pass it as `targetId` in later calls. + - `suggestedTargetId` is the label when one exists, otherwise the stable `tabId` handle like `t1`. + - Avoid relying on raw DevTools `targetId` except for immediate diagnostics; it can change under Chromium target replacement. +3. Read before clicking: + - Use `action="snapshot"` on the intended `targetId`. + - Use the same `targetId` for follow-up actions so refs stay on the same tab. + - For durable Playwright refs, request `refs="aria"` when supported. If you receive `axN` refs from `snapshotFormat="aria"`, use them only after that same snapshot call; stale or unbound `axN` refs fail fast and need a fresh snapshot. + - Use `urls=true` when link text is ambiguous or a direct navigation target would avoid brittle clicks. + - Use `labels=true` on snapshot or screenshot when visual position matters. +4. Act narrowly: + - Prefer `action="act"` with a ref from the latest snapshot. + - After navigation, modal changes, or form submission, snapshot again before the next action. + - Avoid blind waits. Wait for visible UI state when possible. +5. Report real blockers: + - If the page needs login, permission, CAPTCHA, 2FA, camera/microphone approval, or another manual step, stop and tell the user exactly what is needed. + - Do not claim the browser is not logged in just because the current page shows a permission or onboarding dialog. Inspect the visible UI first. + +### Tab Hygiene + +Before creating a tab for a named task, list tabs and reuse an existing matching +label or URL when it is still usable. + +```json +{ "action": "tabs" } +``` + +If no suitable tab exists: + +```json +{ "action": "open", "url": "https://example.com", "label": "task" } +``` + +Then target it by label: + +```json +{ "action": "snapshot", "targetId": "task", "refs": "aria" } +``` + +If a retry creates duplicates, close the extras by `tabId`: + +```json +{ "action": "close", "targetId": "t3" } +``` + +Do not pass bare numbers like `"2"` as `targetId`. Numeric tab positions are only +for the CLI `openclaw browser tab select 2` helper; browser tool calls need a +`suggestedTargetId`, label, `tabId`, or raw target id. + +### Stale Ref Recovery + +If an action fails with a missing or stale ref: + +1. Snapshot the same `targetId` again. +2. Find the current visible control. +3. Retry once with the new ref. +4. If the UI moved to a blocker state, report the blocker instead of looping. + +### Existing User Browser + +Use `profile="user"` only when existing cookies/login matter. This attaches to +the user's running Chromium-based browser. + +For `profile="user"` and other existing-session profiles, omit `timeoutMs` on +`act:type`, `evaluate`, `hover`, `scrollIntoView`, `drag`, `select`, and `fill`; +that driver rejects per-call timeout overrides for those actions. + +### Google Meet Notes + +When creating or joining a Meet: + +- Treat camera/microphone permission screens as progress, not login failure. +- If asked whether people can hear you, click the microphone option when voice is required. +- If Google asks for sign-in, 2FA, account chooser confirmation, or permission that needs user approval, report the exact manual action. +- Use one labeled tab per meeting flow, for example `label="meet"`, and reuse it during retries. + ## Escalation Patterns ### Public Page @@ -126,35 +252,38 @@ and phrasing guide. 1. `web_search` if the URL/source is unknown. 2. `web_fetch` if the URL is known and likely static. 3. OpenClaw `browser` if JavaScript/rendering matters. +4. If any step hits Cloudflare, Turnstile, CAPTCHA-like, Access Denied, or a + branded bot-gate page, run Browserless `/unblock` before reporting failure. ### Protected Page Data -1. Try Browserless `/stealth/bql` or `/unblock`. -2. Use Browserless `/session` only for repeated same-site work that benefits +1. Try Browserless `/unblock` first for Cloudflare/anti-bot pages. +2. Try Browserless `/stealth/bql` when BQL behavior is needed or `/unblock` does + not expose useful content. +3. Use Browserless `/session` only for repeated same-site work that benefits from persisted remote state. -3. Use TinyFish Browser API / CDP when deeper remote browser automation is needed. -4. Use a site-specific API only when the domain skill approves the tradeoff. -5. Switch to interactive browser work when login, CAPTCHA/2FA, visible review, +4. Use TinyFish Browser API / CDP when deeper remote browser automation is needed. +5. Use a site-specific API only when the domain skill approves the tradeoff. +6. Switch to interactive browser work when login, CAPTCHA/2FA, visible review, or extension state matters. -6. If the site remains blocked, say so plainly. +7. If the site remains blocked after the relevant Browserless path has actually + run, say so plainly and name what was tried. ### Current Tab Or Logged-In Browser -1. Confirm that the task needs a live browser/session. -2. Check the agent-side managed browser first — it may already hold the login - (e.g. Amazon). Verify login state on the live page. -3. Use the appropriate browser profile/node path for the target device; the - user's device browser is a fallback when the agent session is unavailable. -4. If the requested mode is unavailable, say which capability is missing and +1. Confirm that the task needs the user's live browser/session. +2. Use the available browser profile/node path for the target device. +3. If the requested mode is unavailable, say which capability is missing and choose the closest acceptable fallback. ## Bundled Helper Scripts Use these from this skill directory when repeatable execution helps: -- `scripts/browserless_extract.py` - Browserless `content`, `unblock`, or `stealth-bql` +- `scripts/browserless_extract.py` - Browserless `content`, `unblock`, or `stealth-bql` using stdlib HTTP; reads `BROWSERLESS_TOKEN` or legacy `BROWSERLESS_API_KEY`; redacts tokens, retries transient failures, returns bounded excerpts, metadata, and generic media candidates. For Cloudflare/anti-bot pages, pass `--mode unblock` first. Note: OG/Twitter `meta` and `media_candidates` only populate in `content`/`unblock` modes — the default `stealth-bql` returns title/body text only, so pass `--mode content` or `--mode unblock` when you need metadata or media URLs. +- `scripts/browserless_media_requests.py` - Browserless `/function` network media discovery for rendered pages where media URLs appear only after playback/rendering - `scripts/browserless_session.py` - opt-in persistent Browserless session helper with redacted output and 0600 session files -- `scripts/tinyfish_browser_extract.py` - TinyFish Browser API / CDP extraction helper +- `scripts/tinyfish_browser_extract.py` - optional TinyFish Browser API / CDP extraction helper See `references/backends.md.example` for credential and command templates. @@ -163,3 +292,7 @@ See `references/backends.md.example` for credential and command templates. - `references/context-device.md` - browser context/device/session matrix - `references/extraction-backends.md` - backend ladder and safety notes - `references/backends.md.example` - public-safe credential and command template + +## Baseline Checks + +Run `bash scripts/test.sh` after editing this skill. diff --git a/skills/web-use/references/backends.md.example b/skills/web-use/references/backends.md.example index 3d8a2c7..db1e0c2 100644 --- a/skills/web-use/references/backends.md.example +++ b/skills/web-use/references/backends.md.example @@ -13,7 +13,7 @@ Copy this file to `backends.md` and fill in the env var names or 1Password refer | Browserless (`/session`) | opt-in persistent protected extraction session | No | use sparingly only when cookies/localStorage/sessionStorage/cache must persist across repeated same-site BQL/CDP calls | | TinyFish Browser API (CDP session) | remote stealth browser primitive for brittle or multi-step flows | Secondary / situational | works on protected pages when used as a browser session | | Site-specific structured data API | clean structured fields for one site | No | domain skill owns credential, cost policy, and when it is worth a call | -| OpenClaw browser | interactive human-visible browsing | No | use for login, 2FA, manual review, checkout | +| OpenClaw browser | interactive human-visible browsing | No | use for login, 2FA, manual review, cart, checkout, or account state | --- @@ -33,6 +33,9 @@ export BROWSERLESS_TOKEN="..." export TINYFISH_API_KEY="..." ``` +`BROWSERLESS_API_KEY` is accepted as a legacy fallback by the local helper +scripts, but new config should use `BROWSERLESS_TOKEN`. + --- ## Policy @@ -96,6 +99,18 @@ python3 skills/web-use/scripts/browserless_session.py stop \ Do not paste the session file contents, `browserQL`, `connect`, or `stop` URLs into chat or logs. They include the Browserless token and should be treated as bearer credentials. +### Browserless media-request helper + +```bash +BROWSERLESS_TOKEN="" \ + python3 skills/web-use/scripts/browserless_media_requests.py \ + +``` + +Use this when a page exposes media only after rendering or playback, such as a +public social post/Reel where `yt-dlp` failed. Returned CDN URLs are temporary; +store source URL, method, fetch time, and summarized media metadata by default. + ### TinyFish helper ```bash diff --git a/skills/web-use/references/extraction-backends.md b/skills/web-use/references/extraction-backends.md index fa630de..c5ee2dd 100644 --- a/skills/web-use/references/extraction-backends.md +++ b/skills/web-use/references/extraction-backends.md @@ -72,7 +72,7 @@ session: - 2FA - CAPTCHA/manual solve - visual confirmation -- final shopping steps +- final shopping steps or other user-confirmed account actions ## Protected Page Escalation @@ -87,6 +87,7 @@ When doing repeatable operational work, prefer the bundled helpers before rewriting one-off curl/CDP glue: - `../scripts/browserless_extract.py` +- `../scripts/browserless_media_requests.py` - `../scripts/browserless_session.py` - `../scripts/tinyfish_browser_extract.py` diff --git a/skills/web-use/scripts/browserless_extract.py b/skills/web-use/scripts/browserless_extract.py index ebca44b..759b5cf 100755 --- a/skills/web-use/scripts/browserless_extract.py +++ b/skills/web-use/scripts/browserless_extract.py @@ -2,12 +2,12 @@ """Fetch a page through Browserless and return normalized extraction JSON. Examples: - BROWSERLESS_TOKEN="" \ - python3 skills/web-use/scripts/browserless_extract.py \ + BROWSERLESS_TOKEN="" \ + python3 scripts/browserless_extract.py \ https://example.com/protected-page - BROWSERLESS_TOKEN="" \ - python3 skills/web-use/scripts/browserless_extract.py \ + BROWSERLESS_TOKEN="" \ + python3 scripts/browserless_extract.py \ https://example.com/protected-page \ --mode unblock """ @@ -19,14 +19,94 @@ import os import re import sys +import time from html import unescape from typing import Any - -import requests +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen DEFAULT_HOST = "https://production-sfo.browserless.io" DEFAULT_TIMEOUT = 120 DEFAULT_SNIPPET_CHARS = 3000 +DEFAULT_MAX_RETRIES = 3 +DEFAULT_MEDIA_LIMIT = 20 +RETRYABLE_STATUS = {429, 500, 502, 503, 504} +BACKOFF_BASE = 1.5 +BACKOFF_CAP = 20.0 +ERROR_BODY_CHARS = 500 +META_KEYS = ( + "og:title", + "og:description", + "og:image", + "og:video", + "og:video:url", + "og:video:secure_url", + "twitter:title", + "twitter:description", + "twitter:image", +) +MEDIA_RE = re.compile( + r"https?:\\?/\\?/[^\"'<>\s]+?\.(?:mp4|m3u8|webm|mov|m4v)(?:[^\"'<>\s]*)?", + re.I, +) + + +class BrowserlessRequestError(RuntimeError): + """Raised when Browserless returns a non-2xx response after retries.""" + + +def redact(text: str, token: str | None) -> str: + if token and text: + text = text.replace(token, "") + return re.sub(r"([?&]token=)[^&\s]+", r"\1", text or "") + + +def retry_after(exc: HTTPError) -> float | None: + raw = exc.headers.get("Retry-After") if exc.headers else None + if raw and raw.isdigit(): + return float(raw) + return None + + +def post_browserless( + url: str, + payload: dict[str, Any], + timeout: int, + token: str | None, + max_retries: int, +) -> tuple[int, str]: + body = json.dumps(payload).encode("utf-8") + last_error = "unknown error" + + for attempt in range(max_retries + 1): + request = Request( + url, + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urlopen(request, timeout=timeout) as response: + return response.status, response.read().decode("utf-8", errors="replace") + except HTTPError as exc: + error_body = redact(exc.read().decode("utf-8", errors="replace"), token) + last_error = f"HTTP {exc.code}: {error_body[:ERROR_BODY_CHARS]}" + if exc.code not in RETRYABLE_STATUS or attempt == max_retries: + raise BrowserlessRequestError(last_error) from None + delay = retry_after(exc) or min(BACKOFF_BASE * (2**attempt), BACKOFF_CAP) + except URLError as exc: + last_error = redact(str(exc.reason), token) + if attempt == max_retries: + raise BrowserlessRequestError(last_error) from None + delay = min(BACKOFF_BASE * (2**attempt), BACKOFF_CAP) + time.sleep(delay) + + raise BrowserlessRequestError(last_error) + + +def browserless_url(host: str, path: str, token: str) -> str: + return f"{host}{path}?{urlencode({'token': token})}" def clean_text(text: str) -> str: @@ -47,6 +127,38 @@ def extract_title_from_html(html: str) -> str: return clean_text(match.group(1)) if match else "" +def extract_meta(html: str) -> dict[str, str]: + meta: dict[str, str] = {} + for key in META_KEYS: + pattern_a = ( + r"]+(?:property|name)=[\"']%s[\"'][^>]+content=[\"']([^\"']*)[\"']" + % re.escape(key) + ) + pattern_b = ( + r"]+content=[\"']([^\"']*)[\"'][^>]+(?:property|name)=[\"']%s[\"']" + % re.escape(key) + ) + match = re.search(pattern_a, html, re.I) or re.search(pattern_b, html, re.I) + if match: + meta[key] = clean_text(match.group(1)) + return meta + + +def normalize_media_url(url: str) -> str: + return unescape(url).replace("\\/", "/") + + +def extract_media_candidates(html: str, limit: int) -> list[str]: + seen: list[str] = [] + for match in MEDIA_RE.findall(html): + media_url = normalize_media_url(match) + if media_url not in seen: + seen.append(media_url) + if len(seen) >= limit: + break + return seen + + def signals(title: str, body: str) -> dict[str, bool]: combined = f"{title}\n{body}" return { @@ -68,62 +180,50 @@ def signals(title: str, body: str) -> dict[str, bool]: } -def sanitize_error(error: Exception, token: str | None) -> str: - message = str(error) - if token: - message = message.replace(token, "") - message = re.sub(r"([?&]token=)[^&\s]+", r"\1", message) - return message - - -def request_content(token: str, url: str, host: str, timeout: int) -> dict[str, Any]: - response = requests.post( - f"{host}/content", - params={"token": token}, - json={"url": url}, - timeout=timeout, - ) - response.raise_for_status() - html = response.text +def html_result(provider_mode: str, url: str, status: int, html: str, media_limit: int) -> dict[str, Any]: title = extract_title_from_html(html) body = strip_html(html) return { "provider": "browserless", - "mode": "content", + "mode": provider_mode, "url": url, - "status_code": response.status_code, + "status_code": status, "title": title, + "meta": extract_meta(html), + "media_candidates": extract_media_candidates(html, media_limit), "body": body, **signals(title, body), } -def request_unblock(token: str, url: str, host: str, timeout: int) -> dict[str, Any]: - response = requests.post( - f"{host}/unblock", - params={"token": token}, - json={"url": url, "browserWSEndpoint": False}, - timeout=timeout, +def request_content(args: argparse.Namespace) -> dict[str, Any]: + status, html = post_browserless( + browserless_url(args.host, "/content", args.token), + {"url": args.url}, + args.timeout, + args.token, + args.max_retries, + ) + return html_result("content", args.url, status, html, args.media_limit) + + +def request_unblock(args: argparse.Namespace) -> dict[str, Any]: + status, response_text = post_browserless( + browserless_url(args.host, "/unblock", args.token), + {"url": args.url, "browserWSEndpoint": False}, + args.timeout, + args.token, + args.max_retries, ) - response.raise_for_status() - payload = response.json() + payload = json.loads(response_text) html = payload.get("content") or "" - title = extract_title_from_html(html) - body = strip_html(html) - return { - "provider": "browserless", - "mode": "unblock", - "url": url, - "status_code": response.status_code, - "title": title, - "body": body, - "browser_ws_endpoint": payload.get("browserWSEndpoint"), - **signals(title, body), - } + result = html_result("unblock", args.url, status, html, args.media_limit) + result["browser_ws_endpoint"] = payload.get("browserWSEndpoint") + return result -def request_stealth_bql(token: str, url: str, host: str, timeout: int, solve: bool) -> dict[str, Any]: - solve_fragment = "solve(type: cloudflare) { found solved time }" if solve else "" +def request_stealth_bql(args: argparse.Namespace) -> dict[str, Any]: + solve_fragment = "solve(type: cloudflare) { found solved time }" if args.solve else "" query = ( "mutation Extract($url: String!) { " "goto(url: $url, waitUntil: networkIdle) { status } " @@ -132,14 +232,14 @@ def request_stealth_bql(token: str, url: str, host: str, timeout: int, solve: bo 'body: text(selector: "body") { text } ' "}" ) - response = requests.post( - f"{host}/stealth/bql", - params={"token": token}, - json={"query": query, "variables": {"url": url}}, - timeout=timeout, + _, response_text = post_browserless( + browserless_url(args.host, "/stealth/bql", args.token), + {"query": query, "variables": {"url": args.url}}, + args.timeout, + args.token, + args.max_retries, ) - response.raise_for_status() - payload = response.json() + payload = json.loads(response_text) if payload.get("errors"): raise RuntimeError(json.dumps(payload["errors"], indent=2)) data = payload.get("data") or {} @@ -148,9 +248,11 @@ def request_stealth_bql(token: str, url: str, host: str, timeout: int, solve: bo return { "provider": "browserless", "mode": "stealth-bql", - "url": url, + "url": args.url, "status_code": (data.get("goto") or {}).get("status"), "title": title, + "meta": {}, + "media_candidates": [], "body": body, "solve": data.get("solve"), **signals(title, body), @@ -158,10 +260,7 @@ def request_stealth_bql(token: str, url: str, host: str, timeout: int, solve: bo def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("url", help="Target URL") parser.add_argument( "--mode", @@ -170,8 +269,14 @@ def parse_args() -> argparse.Namespace: help="Browserless surface to use", ) parser.add_argument("--host", default=DEFAULT_HOST, help="Browserless host") - parser.add_argument("--token", default=os.environ.get("BROWSERLESS_TOKEN"), help="Browserless API token") + parser.add_argument( + "--token", + default=os.environ.get("BROWSERLESS_TOKEN") or os.environ.get("BROWSERLESS_API_KEY"), + help="Browserless API token", + ) parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, help="Request timeout in seconds") + parser.add_argument("--max-retries", type=int, default=DEFAULT_MAX_RETRIES) + parser.add_argument("--media-limit", type=int, default=DEFAULT_MEDIA_LIMIT, help="Maximum media URL candidates to emit") parser.add_argument( "--snippet-chars", type=int, @@ -194,17 +299,17 @@ def main() -> int: try: if args.mode == "content": - result = request_content(args.token, args.url, args.host, args.timeout) + result = request_content(args) elif args.mode == "unblock": - result = request_unblock(args.token, args.url, args.host, args.timeout) + result = request_unblock(args) else: - result = request_stealth_bql(args.token, args.url, args.host, args.timeout, args.solve) + result = request_stealth_bql(args) except Exception as exc: # noqa: BLE001 print( json.dumps( { "ok": False, - "error": sanitize_error(exc, args.token), + "error": redact(str(exc), args.token)[:ERROR_BODY_CHARS], "provider": "browserless", "mode": args.mode, "url": args.url, diff --git a/skills/web-use/scripts/browserless_media_requests.py b/skills/web-use/scripts/browserless_media_requests.py new file mode 100755 index 0000000..2d64065 --- /dev/null +++ b/skills/web-use/scripts/browserless_media_requests.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Discover media network requests from a rendered page via Browserless /function. + +This is intentionally generic web-use plumbing. It does not decide what kind of +social/video workflow should happen next; callers such as the video skill consume +the normalized JSON. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +DEFAULT_HOST = "https://production-sfo.browserless.io" +DEFAULT_TIMEOUT = 120 +DEFAULT_WAIT_MS = 6000 +DEFAULT_GOTO_TIMEOUT_MS = 90000 + + +def post_json(url: str, payload: dict[str, Any], timeout: int) -> tuple[int, str]: + body = json.dumps(payload).encode("utf-8") + request = Request( + url, + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urlopen(request, timeout=timeout) as response: + return response.status, response.read().decode("utf-8", errors="replace") + except HTTPError as exc: + error_body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Browserless HTTP {exc.code}: {error_body[:1000]}") from exc + except URLError as exc: + raise RuntimeError(f"Browserless request failed: {exc.reason}") from exc + + +def browserless_url(host: str, path: str, token: str) -> str: + return f"{host}{path}?{urlencode({'token': token})}" + + +FUNCTION_CODE = r""" +export default async function({ page, context }) { + const seen = new Map(); + const mediaTypes = /video|audio|mpegurl|mp4|webm|m3u8/i; + const mediaUrl = /\.(mp4|m4v|webm|mov|m3u8|mp3|m4a)(\?|$)/i; + const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); + + const remember = (url, source, headers = {}) => { + if (!url || seen.has(url)) return; + const contentType = headers['content-type'] || headers['Content-Type'] || ''; + if (!mediaTypes.test(contentType) && !mediaUrl.test(url)) return; + seen.set(url, { url, source, content_type: contentType || null }); + }; + + page.on('response', async response => { + try { + remember(response.url(), 'response', response.headers()); + } catch (err) {} + }); + + await page.goto(context.url, { waitUntil: 'networkidle2', timeout: context.gotoTimeoutMs || 90000 }); + + if (context.play) { + await page.evaluate(async () => { + for (const video of Array.from(document.querySelectorAll('video'))) { + try { video.muted = true; await video.play(); } catch (err) {} + } + for (const button of Array.from(document.querySelectorAll('button, [role="button"]')).slice(0, 20)) { + const label = `${button.ariaLabel || ''} ${button.textContent || ''}`.toLowerCase(); + if (/play|watch|reel|video/.test(label)) { + try { button.click(); } catch (err) {} + } + } + }); + } + + await sleep(context.waitMs || 6000); + + const domMedia = await page.evaluate(() => Array.from(document.querySelectorAll('video, audio, source')) + .map(el => el.currentSrc || el.src) + .filter(Boolean)); + for (const url of domMedia) remember(url, 'dom'); + + const title = await page.title().catch(() => ''); + return { + ok: true, + url: context.url, + title, + candidates: Array.from(seen.values()), + candidate_count: seen.size + }; +} +""" + + +def parse_browserless_function_response(text: str) -> dict[str, Any]: + payload = json.loads(text) + if isinstance(payload, dict) and "data" in payload and isinstance(payload["data"], dict): + return payload["data"] + if isinstance(payload, dict): + return payload + raise RuntimeError("Unexpected Browserless /function response shape") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("url", help="Target page URL") + parser.add_argument("--host", default=DEFAULT_HOST, help="Browserless host") + parser.add_argument( + "--token", + default=os.environ.get("BROWSERLESS_TOKEN") or os.environ.get("BROWSERLESS_API_KEY"), + help="Browserless token; defaults to BROWSERLESS_TOKEN or BROWSERLESS_API_KEY", + ) + parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, help="HTTP timeout in seconds") + parser.add_argument("--wait-ms", type=int, default=DEFAULT_WAIT_MS, help="Post-load wait time in ms") + parser.add_argument("--goto-timeout-ms", type=int, default=DEFAULT_GOTO_TIMEOUT_MS, help="Browser goto timeout in ms") + parser.add_argument("--play", action="store_true", help="Try muted playback/clicks before collecting media") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if not args.token: + print( + json.dumps( + { + "ok": False, + "error": "Missing Browserless token. Set BROWSERLESS_TOKEN or BROWSERLESS_API_KEY.", + }, + indent=2, + ), + file=sys.stderr, + ) + return 2 + + try: + _, response_text = post_json( + browserless_url(args.host, "/function", args.token), + { + "code": FUNCTION_CODE, + "context": { + "url": args.url, + "waitMs": args.wait_ms, + "gotoTimeoutMs": args.goto_timeout_ms, + "play": args.play, + }, + }, + args.timeout, + ) + result = parse_browserless_function_response(response_text) + result.setdefault("ok", True) + result.setdefault("provider", "browserless") + result.setdefault("mode", "function-media-requests") + print(json.dumps(result, indent=2)) + return 0 + except Exception as exc: # noqa: BLE001 + print(json.dumps({"ok": False, "error": str(exc), "provider": "browserless", "mode": "function-media-requests", "url": args.url}, indent=2)) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/web-use/scripts/browserless_session.py b/skills/web-use/scripts/browserless_session.py index e7c2c5d..f67d85e 100755 --- a/skills/web-use/scripts/browserless_session.py +++ b/skills/web-use/scripts/browserless_session.py @@ -41,8 +41,9 @@ import re import sys from typing import Any - -import requests +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen DEFAULT_HOST = "https://production-sfo.browserless.io" DEFAULT_TIMEOUT = 120 @@ -50,6 +51,44 @@ STD_STREAMS = {"-", "/dev/stdout", "/dev/stderr", "/dev/fd/1", "/dev/fd/2"} +class BrowserlessRequestError(RuntimeError): + """Raised when Browserless returns a failed HTTP response.""" + + +def with_query(url: str, params: dict[str, str] | None = None) -> str: + if not params: + return url + separator = "&" if "?" in url else "?" + return f"{url}{separator}{urlencode(params)}" + + +def request_json( + method: str, + url: str, + payload: dict[str, Any] | None, + timeout: int, +) -> tuple[int, Any]: + body = json.dumps(payload).encode("utf-8") if payload is not None else None + request = Request( + url, + data=body, + headers={"Content-Type": "application/json"} if body is not None else {}, + method=method, + ) + try: + with urlopen(request, timeout=timeout) as response: + text = response.read().decode("utf-8", errors="replace") + try: + return response.status, json.loads(text) if text else {} + except json.JSONDecodeError: + return response.status, {"raw": text} + except HTTPError as exc: + error_body = exc.read().decode("utf-8", errors="replace") + raise BrowserlessRequestError(f"HTTP {exc.code}: {error_body[:1000]}") from exc + except URLError as exc: + raise BrowserlessRequestError(str(exc.reason)) from exc + + def token_from_url(url: Any) -> str | None: if not isinstance(url, str): return None @@ -189,14 +228,12 @@ def cmd_create(args: argparse.Namespace) -> int: raise ValueError("--process-keep-alive-ms must be <= --ttl-ms.") body["processKeepAlive"] = args.process_keep_alive_ms - response = requests.post( - f"{args.host}/session", - params=params, - json=body, - timeout=args.timeout, + _, session = request_json( + "POST", + with_query(f"{args.host}/session", params), + body, + args.timeout, ) - response.raise_for_status() - session = response.json() if not isinstance(session, dict): raise ValueError("Browserless /session did not return a JSON object.") @@ -208,7 +245,7 @@ def cmd_create(args: argparse.Namespace) -> int: cleanup = "not attempted" if stop_url: try: - requests.delete(stop_url, timeout=args.timeout) + request_json("DELETE", stop_url, None, args.timeout) cleanup = "stopped" except Exception: # noqa: BLE001 cleanup = "stop failed" @@ -270,19 +307,15 @@ def cmd_query(args: argparse.Namespace) -> int: if not isinstance(variables, dict): raise ValueError("--variables-json must be a JSON object.") - response = requests.post( + status_code, payload = request_json( + "POST", browserql, - json={"query": query_text, "variables": variables}, - timeout=args.timeout, + {"query": query_text, "variables": variables}, + args.timeout, ) - status_code = response.status_code - try: - payload: Any = response.json() - except ValueError: - payload = {"raw": response.text} errors = payload.get("errors") if isinstance(payload, dict) else None - ok = response.ok and not errors + ok = 200 <= status_code < 300 and not errors emit( { "ok": ok, @@ -323,11 +356,15 @@ def cmd_stop(args: argparse.Namespace) -> int: "Session file has no stop URL; nothing to delete remotely." ) - response = requests.delete(stop_url, timeout=args.timeout) - status_code = response.status_code - stopped = response.ok + try: + status_code, _ = request_json("DELETE", stop_url, None, args.timeout) + stopped = 200 <= status_code < 300 + except BrowserlessRequestError as exc: + status_match = re.search(r"HTTP (\d+):", str(exc)) + status_code = int(status_match.group(1)) if status_match else None + stopped = False # A 404 means the session is already gone, so the local file is moot too. - removable = response.ok or status_code == 404 + removable = stopped or status_code == 404 file_deleted = False retained_reason = None @@ -413,9 +450,9 @@ def add_common(parser: argparse.ArgumentParser) -> None: ) parser.add_argument( "--token", - default=os.environ.get("BROWSERLESS_TOKEN"), + default=os.environ.get("BROWSERLESS_TOKEN") or os.environ.get("BROWSERLESS_API_KEY"), help=( - "Browserless API token (env BROWSERLESS_TOKEN). Required for create; " + "Browserless API token (env BROWSERLESS_TOKEN or BROWSERLESS_API_KEY). Required for create; " "query/stop read the token-bearing URL from the session file." ), ) diff --git a/skills/web-use/scripts/test.sh b/skills/web-use/scripts/test.sh new file mode 100755 index 0000000..62dd550 --- /dev/null +++ b/skills/web-use/scripts/test.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +pass=0 +fail=0 + +check() { + local name="$1" + shift + if "$@" >/tmp/web-use-test.out 2>/tmp/web-use-test.err; then + printf 'ok - %s\n' "$name" + pass=$((pass + 1)) + else + printf 'not ok - %s\n' "$name" + sed 's/^/ /' /tmp/web-use-test.err + fail=$((fail + 1)) + fi +} + +echo "=== web-use skill baseline tests ===" + +check "Frontmatter name matches directory name" grep -Eq '^name:[[:space:]]*"?web-use"?[[:space:]]*$' SKILL.md +check "browserless_extract helper parses" python3 -m py_compile scripts/browserless_extract.py +check "browserless_extract help works without credentials" python3 scripts/browserless_extract.py --help +check "browserless_extract help mentions retry option" bash -c 'python3 scripts/browserless_extract.py --help | grep -q -- "--max-retries"' +check "browserless_extract help mentions media limit option" bash -c 'python3 scripts/browserless_extract.py --help | grep -q -- "--media-limit"' +check "browserless_session helper parses" python3 -m py_compile scripts/browserless_session.py +check "browserless_session help works without credentials" python3 scripts/browserless_session.py --help +check "browserless_media_requests helper parses" python3 -m py_compile scripts/browserless_media_requests.py +check "browserless_media_requests help works without credentials" python3 scripts/browserless_media_requests.py --help +check "tinyfish helper parses" python3 -m py_compile scripts/tinyfish_browser_extract.py +check "tinyfish help works without optional deps" python3 scripts/tinyfish_browser_extract.py --help + +rm -f /tmp/web-use-test.out /tmp/web-use-test.err + +echo "Passed: $pass" +echo "Failed: $fail" + +if [ "$fail" -ne 0 ]; then + exit 1 +fi diff --git a/skills/web-use/scripts/tinyfish_browser_extract.py b/skills/web-use/scripts/tinyfish_browser_extract.py index 3f327b0..684cd91 100755 --- a/skills/web-use/scripts/tinyfish_browser_extract.py +++ b/skills/web-use/scripts/tinyfish_browser_extract.py @@ -16,9 +16,8 @@ import re import sys from typing import Any - -import requests -import websockets +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen CREATE_SESSION_URL = "https://api.browser.tinyfish.ai/" DEFAULT_TIMEOUT_SECONDS = 300 @@ -26,6 +25,32 @@ DEFAULT_SNIPPET_CHARS = 3000 +class TinyFishRequestError(RuntimeError): + """Raised when TinyFish returns a failed HTTP response.""" + + +def post_json(url: str, payload: dict[str, Any], headers: dict[str, str], timeout: int) -> tuple[int, Any]: + body = json.dumps(payload).encode("utf-8") + request = Request( + url, + data=body, + headers={**headers, "Content-Type": "application/json"}, + method="POST", + ) + try: + with urlopen(request, timeout=timeout) as response: + text = response.read().decode("utf-8", errors="replace") + try: + return response.status, json.loads(text) if text else {} + except json.JSONDecodeError: + return response.status, {"raw": text} + except HTTPError as exc: + error_body = exc.read().decode("utf-8", errors="replace") + raise TinyFishRequestError(f"HTTP {exc.code}: {error_body[:1000]}") from exc + except URLError as exc: + raise TinyFishRequestError(str(exc.reason)) from exc + + def sanitize_error(error: Exception, api_key: str | None, session: dict[str, Any] | None = None) -> str: message = str(error) secrets = [api_key] @@ -62,6 +87,13 @@ def signals(title: str, body: str) -> dict[str, bool]: async def evaluate_via_cdp(cdp_url: str, target_url: str, wait_seconds: int, snippet_chars: int) -> dict[str, Any]: + try: + import websockets + except ModuleNotFoundError as exc: + raise RuntimeError( + "TinyFish CDP extraction requires the optional Python package `websockets`." + ) from exc + async with websockets.connect(cdp_url, max_size=20_000_000) as ws: next_id = 0 @@ -157,14 +189,16 @@ def main() -> int: session: dict[str, Any] | None = None try: - response = requests.post( + status_code, session = post_json( CREATE_SESSION_URL, - headers={"X-API-Key": args.api_key, "Content-Type": "application/json"}, - json={"url": args.url, "timeout_seconds": args.session_timeout_seconds}, - timeout=60, + {"url": args.url, "timeout_seconds": args.session_timeout_seconds}, + {"X-API-Key": args.api_key}, + 60, ) - response.raise_for_status() - session = response.json() + if not (200 <= status_code < 300): + raise TinyFishRequestError(f"HTTP {status_code}: {session}") + if not isinstance(session, dict): + raise TinyFishRequestError("TinyFish response was not a JSON object") cdp_url = session.get("cdp_url") if not cdp_url: raise RuntimeError("TinyFish response missing cdp_url")