diff --git a/.github/scripts/visual-diff.mjs b/.github/scripts/visual-diff.mjs index d4821c0c7b..8f2f588f33 100644 --- a/.github/scripts/visual-diff.mjs +++ b/.github/scripts/visual-diff.mjs @@ -55,7 +55,7 @@ const AUTH_TOKEN = process.env.CLAWMETRY_VISUAL_DIFF_TOKEN || ""; // switchTab() name here, in CANONICAL_TABS, and in PR_SCREENSHOT_TABS. // `overview` is the implicit default -- listed first for a `root` baseline. const DEFAULT_TABS = - "overview,flow,brain,usage,crons,memory,security,subagents,transcripts,logs,skills,models,approvals,alerts,notifications,limits,clusters,history,channels,dives,harness,inventory,nemoclaw,policy,selfevolve,swimlane,tool-catalog,tracing,turn-anatomy,version-impact,context-economics,agents,evals,bench"; + "overview,flow,brain,usage,crons,memory,security,subagents,transcripts,logs,skills,models,approvals,alerts,notifications,limits,clusters,history,channels,dives,harness,inventory,nemoclaw,guard,policy,selfevolve,swimlane,tool-catalog,tracing,turn-anatomy,version-impact,context-economics,agents,evals,bench"; const TABS = (process.env.PR_SCREENSHOT_TABS || DEFAULT_TABS) .split(",") .map((p) => p.trim()) diff --git a/.github/workflows/pr-screenshots.yml b/.github/workflows/pr-screenshots.yml index efb3c44be2..3c1165fad0 100644 --- a/.github/workflows/pr-screenshots.yml +++ b/.github/workflows/pr-screenshots.yml @@ -38,7 +38,7 @@ env: # clawmetry/templates/tabs/*.html and be reachable via window.switchTab(). # See routes/* for the API endpoints each tab calls. # Must stay in sync with CANONICAL_TABS in tests/test_e2e_oss_all_tabs.py. - PR_SCREENSHOT_TABS: "overview,flow,brain,usage,crons,memory,security,subagents,transcripts,logs,skills,models,approvals,alerts,notifications,limits,clusters,history,channels,dives,harness,inventory,nemoclaw,policy,selfevolve,swimlane,tool-catalog,tracing,turn-anatomy,version-impact,context-economics,agents,evals,bench" + PR_SCREENSHOT_TABS: "overview,flow,brain,usage,crons,memory,security,subagents,transcripts,logs,skills,models,approvals,alerts,notifications,limits,clusters,history,channels,dives,harness,inventory,nemoclaw,guard,policy,selfevolve,swimlane,tool-catalog,tracing,turn-anatomy,version-impact,context-economics,agents,evals,bench" HEAD_PORT: "8082" BASE_PORT: "8081" # Local-store fast-paths must be on so the dashboard renders the seeded @@ -58,7 +58,7 @@ jobs: timeout-minutes: 20 # Non-blocking: the bot is allowed to fail without holding up a merge. continue-on-error: true - # Trial-end hard block is default-ON in code — opt out so the screenshot + # Trial-end hard block is default-ON in code -- opt out so the screenshot # bot can render the actual observability tabs instead of the paywall # overlay. Hard-block coverage lives in tests/test_trial_hard_block.py. env: @@ -343,19 +343,20 @@ jobs: if: ${{ always() && !github.event.pull_request.head.repo.fork }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Bind event data to an env var rather than interpolating it into the - # script, so no `run:` block in this repo expands ${{ github.event.* }}. + # Bind all event data and context to env vars rather than interpolating + # directly into the run: block -- prevents expression injection. PR: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} run: | set -e MARKER="" BODY="$(printf '%s\n\n%s' "$MARKER" "$(cat /tmp/visual-diff-body.md)")" - existing_id=$(gh api -X GET "repos/${{ github.repository }}/issues/${PR}/comments" --paginate \ + existing_id=$(gh api -X GET "repos/${REPO}/issues/${PR}/comments" --paginate \ --jq "map(select(.body | contains(\"${MARKER}\"))) | .[0].id // empty") if [ -n "$existing_id" ]; then - gh api -X PATCH "repos/${{ github.repository }}/issues/comments/${existing_id}" -f body="$BODY" + gh api -X PATCH "repos/${REPO}/issues/comments/${existing_id}" -f body="$BODY" else - gh pr comment "$PR" --repo "${{ github.repository }}" --body "$BODY" + gh pr comment "$PR" --repo "${REPO}" --body "$BODY" fi # -- 12. Belt + suspenders: upload PNGs as artefact ---------------------- diff --git a/CLAUDE.md b/CLAUDE.md index 0bd3cf4637..fb3cf7edbe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -200,6 +200,8 @@ DEBUG=1 # Enable debug logging - **Control plane that defaults to observation** — ClawMetry is NOT read-only, and hasn't been for a long time. It already kills, pauses, blocks and reroutes running agents through five surfaces: approval denial → session kill (`clawmetry/approvals.py`), POSIX signals across the agent's descendant tree (`clawmetry/process_control.py`), HITL pause → proxy `503` (`routes/hitl.py`), the enforcement proxy's budget block / loop detection / model routing (`clawmetry/proxy.py`), and cron CRUD via gateway RPC (`routes/crons.py`). Do NOT reject a feature because "we're read-only" — that rule is retired. The rule that replaces it is **no surprise writes**: every write is (a) user-initiated or declared in a policy the user wrote, (b) scoped to a single session, (c) reversible where physics allows, and (d) attributed in the approvals audit table. Reads need no permission; writes need all four. **Fail open on entitlement, closed on policy.** If a licence/entitlement lookup errors or is ambiguous, the agent KEEPS RUNNING — only a policy the user actually declared may block or kill. A billing bug must never stop a customer's agent. (`clawmetry/entitlements.py` already defaults to GRACE, where every `allows_*` returns `True`; new control features inherit that posture and assert it in a test.) + **Capability is answered per session, from one place.** `process_control.runtime_control_support(runtime, session_id, cwd)` is the single verdict the Guard tab, the daemon and the actuator all read; never re-derive it. Three axes vary independently and each has bitten us: (1) **OS** — POSIX uses signals, **Windows uses the native equivalents** (`NtSuspendProcess`/`NtResumeProcess`, a console Ctrl+C from a detached helper, `taskkill /T` → `TerminateProcess`). Declare ctypes `argtypes`/`restype` on every Win32 call — the default `c_int` restype truncates a 64-bit `HANDLE` and the whole path fails silently. A Windows Ctrl+C reaches the console, not one pid; say so in the UI. (2) **Session, not runtime** — a Cursor *CLI* session is a real process tree and is controllable; a Cursor *editor* conversation shares the one IDE process and is not. Ask the resolver, don't refuse a runtime wholesale. (3) **OpenClaw pause** — there is no pause primitive; the HITL flag is enforced *only* by `clawmetry/proxy.py`, so with no proxy running a "pause" changes nothing. Probe `enforcement_proxy_status()` and report `advisory_only` rather than claiming the agent was held. A control that cannot work says why next to a disabled button — never ship one that quietly does nothing. + **Policies escalate over time.** A Guard policy may carry `steps` (`[{action, after_secs}, …]`, capped at `policy_engine.MAX_LADDER_STEPS`) so the response can be *pause now, kill in 5 minutes if still stuck*. Rung 0 fires on the match; rung *n* is due `after_secs` after rung *n-1* **actually fired**; a rung only fires if the session is **still matching** that tick. The durable latch is `(session_id, policy_id, step_index)` — widening it was mandatory, since a two-column latch lets rung 1 overwrite rung 0 and a restart replays a ladder ending in `kill`. Every rung passes the same locks: a ladder can never reach a process a plain policy could not. A policy with no `steps` is a one-rung ladder, which is why every pre-ladder rule is unchanged. - **Acceptance criteria are traceable to tests** — every criterion in `docs/acceptance_criteria.json` (mirrored from 8090 Software Factory) must be declared by at least one test under `tests/`. CI enforces it as a one-way ratchet; see FLYWHEEL.md §1g. Drift Bot catches "this diff contradicts a Blueprint"; this catches "untouched code stopped satisfying a criterion", which is the class that produced `$0.00` cost windows and ghost sessions. `make ac-report` to see where you stand. - **Never delete a hook you did not write** — `~/.claude/settings.json` has other writers (GitLens's `gk ai hook install claude-code --force`, `numbat`, the user) and ClawMetry itself writes it from three places. Every removal path goes through `clawmetry/hook_ownership.py` at **hook** granularity, never entry granularity: a foreign writer may have merged its command into the same entry as ours, and the daemon gate's reinstall runs every ~2s, so an entry-level drop deletes someone else's hook within seconds. Installed hook timeouts are clamped (`CLAWMETRY_HOOK_TIMEOUT_MAX_S`, default 8h) — on Copilot, whose `preToolUse` gate is fail-closed, an unbounded wait on a wedged hook is a denial of service against the user's own agent. `docs/HOOK_COEXISTENCE.md`; harness `scripts/hook_collision_matrix.py`. - **A user's repository is read, never written** — `clawmetry/git_outcomes.py` is the only place ClawMetry runs `git` against a directory the operator chose, and it routes every invocation through one chokepoint that rejects anything outside an allowlist of read-only plumbing subcommands (`log`, `rev-list`, `blame`, `cat-file`, `rev-parse`, `for-each-ref`, `show-ref`, `ls-files`, plus `config --get` and `remote get-url`). A `fetch` added "just to freshen state" raises `UnsafeGitCommand` rather than shipping. Add a new git call by adding it to that allowlist, with a test, or not at all. diff --git a/clawmetry/audit.py b/clawmetry/audit.py index 9161fe8849..3e8b51ce30 100644 --- a/clawmetry/audit.py +++ b/clawmetry/audit.py @@ -101,7 +101,9 @@ def record_audit( finally: conn.close() except Exception as exc: # pragma: no cover - defensive - logger.warning("audit: record failed (%s): %s", event_type, exc) + logger.warning("audit: record failed (%s): %s", + str(event_type or "")[:64].replace("\r", " ").replace("\n", " "), + exc) def audit_event( @@ -133,7 +135,9 @@ def audit_event( details["source"] = source record_audit(action, actor=actor, target=target, details=details) except Exception as exc: # pragma: no cover - defensive; record_audit already guards - logger.warning("audit: audit_event failed (%s): %s", action, exc) + logger.warning("audit: audit_event failed (%s): %s", + str(action or "")[:64].replace("\r", " ").replace("\n", " "), + exc) def read_audit_log( diff --git a/clawmetry/guard_actuator.py b/clawmetry/guard_actuator.py new file mode 100644 index 0000000000..ca0baf732a --- /dev/null +++ b/clawmetry/guard_actuator.py @@ -0,0 +1,129 @@ +"""Guard actuator — the ONE path from a decision to a process. + +Both ways ClawMetry can act on an agent end here: a human pressing +Pause / Resume / Stop / Kill in the Guard tab (``routes/guard.py``) and the +daemon's policy pass firing a rung of a policy (``sync._apply_guard_policies``). +Because they share this function, an automatic pause and a hand-pressed +pause are identical to the agent process — resume included, which used to +bypass the shared path and call the signal helper directly. + +The module is deliberately small and free of module-level imports from the +daemon so it can be read on its own: the pieces it composes (the HITL pause +flag, the OpenClaw CLI task cancel, the per-platform signal helpers) are +looked up at call time. That also keeps the daemon's monkeypatch seams +(``sync._hitl_set_pause``, ``sync._openclaw_cancel_task``) working exactly +as before. + +Never raises: every outcome is a structured result the caller records +verbatim, and the ``detail`` field is always a fixed token, never exception +text, because the dict can reach an HTTP response. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Dict + +log = logging.getLogger("clawmetry.guard_actuator") + +# Actions the actuator understands. ``resume`` is control-only: no policy +# can request it (it is not in ``policy_engine.ACTIONS``); a human decides +# that. +ACTIONS = ("pause", "resume", "stop", "kill") + + +def _log_token(value: Any, limit: int = 128) -> str: + """One log-safe token from a caller-supplied value: line breaks removed + so a crafted id cannot forge extra log lines, length bounded.""" + return str(value or "")[:limit].replace("\r", " ").replace("\n", " ") + + +def guard_actuate(runtime: str, session_id: str, cwd: str, + action: str) -> Dict[str, Any]: + """Send the signal for one policy decision, or one human button press. + + Deliberately mirrors ``sync._run_process_control`` (the cloud-relayed + path) including its OpenClaw special-casing, so all three ways of + reaching a process — cloud relay, local button, automatic policy — do + exactly the same thing to it. + + Returns ``{ok, detail, ...}``. ``detail`` is one of a fixed set of + tokens (``paused_via_proxy_hitl``, ``unsupported_no_primitive``, + ``cwd_mismatch_rejected``, ``actuator_error``, ``no-op``, or the signal + helper's own token) so it can be shown to an operator as-is. + """ + import clawmetry.process_control as _pc + import clawmetry.sync as _s + + rt = (runtime or "").strip().lower() + act = (action or "").strip().lower() + sid = str(session_id or "") + + # When an HTTP handler supplies cwd, validate it against the session's + # recorded location before passing it to any signal helper. The daemon + # supplies cwd from the session record itself, so this is a no-op for + # automatic policy actions; it closes the injection path for the HTTP + # handler (routes/guard.py also canonicalises, but defence-in-depth here). + if cwd: + try: + import clawmetry.local_store as _ls + rec = _ls.get_store().get_session_location(sid) + recorded_cwd = (rec or {}).get("cwd") or "" + if recorded_cwd and ( + os.path.realpath(cwd) != os.path.realpath(recorded_cwd) + ): + log.warning( + "guard actuate cwd mismatch for %s: supplied=%s recorded=%s", + _log_token(sid), _log_token(cwd, 200), + _log_token(recorded_cwd, 200), + ) + return {"ok": False, "detail": "cwd_mismatch_rejected"} + except Exception: # noqa: BLE001 + pass # No recorded cwd — allow; the caller's own validation is enough + + try: + if act == "pause": + _s._hitl_set_pause(sid, True) + if rt == "openclaw": + # OpenClaw has no pause primitive. The HITL flag file is the + # only lever, and the ONLY thing that enforces it is the + # optional enforcement proxy. Claiming "the proxy refuses + # further LLM calls" on a node with no proxy reported a + # stopped agent that was still running — so ask first and + # report what actually happened. + cap = _pc.openclaw_pause_capability() + return {"ok": bool(cap["effective"]), + "detail": ("paused_via_proxy_hitl" if cap["effective"] + else "unsupported_no_primitive"), + "mechanism": cap["mechanism"], + "advisory_only": not cap["effective"], + "note": cap["detail"]} + return _pc.pause_session(rt, sid, cwd) + if act in ("stop", "kill"): + _s._hitl_set_pause(sid, True) + if rt == "openclaw": + cr = _s._openclaw_cancel_task(sid) + return {"ok": bool(cr.get("ok")), "action": "cancel", + "scope_pending": bool(cr.get("scope_pending")), + "detail": (cr.get("error") or "task cancel requested")} + mode = "stop" if act == "stop" else "kill" + return _pc.kill_session(rt, sid, cwd, mode=mode) + if act == "resume": + _s._hitl_set_pause(sid, False) + if rt == "openclaw": + cap = _pc.openclaw_pause_capability() + return {"ok": bool(cap["effective"]), + "detail": ("resumed_via_proxy_hitl" if cap["effective"] + else "nothing_was_holding_this_session"), + "mechanism": cap["mechanism"], + "advisory_only": not cap["effective"], + "note": cap["detail"]} + return _pc.resume_session(rt, sid, cwd) + except Exception: # noqa: BLE001 — never raise into the daemon tick + # The exception text stays in the log; the returned detail is a fixed + # token because this dict is recorded and can reach an HTTP response. + log.exception("guard actuator %s failed for %s", + _log_token(act, 32), _log_token(sid)) + return {"ok": False, "detail": "actuator_error"} + return {"ok": False, "detail": "no-op"} diff --git a/clawmetry/local_store.py b/clawmetry/local_store.py index 538d400152..f8e293f2ff 100644 --- a/clawmetry/local_store.py +++ b/clawmetry/local_store.py @@ -1612,6 +1612,63 @@ def _on_disk_bytes() -> int: PRIMARY KEY (repo_root, sha) ) """, + # ── Guard policies (detector incident -> enforcement action) ────────── + # Authored in the dashboard's Guard panel, evaluated by the daemon in + # ``sync.py::_emit_detector_incidents`` via ``clawmetry.policy_engine``. + # Rules are LOCAL: they never leave the node, because the actuator they + # drive (process_control signals) only works on the node itself. + """ + CREATE TABLE IF NOT EXISTS session_policy ( + policy_id VARCHAR PRIMARY KEY, + name VARCHAR, + enabled BOOLEAN DEFAULT TRUE, + scope_runtime VARCHAR DEFAULT '', + scope_agent_id VARCHAR DEFAULT '', + trigger_kind VARCHAR DEFAULT '', + min_severity VARCHAR DEFAULT 'info', + min_repeat INTEGER DEFAULT 0, + min_duration_s INTEGER DEFAULT 0, + min_spend_usd DOUBLE DEFAULT 0, + action VARCHAR DEFAULT 'monitor', + -- Escalation ladder as JSON: [{"action","after_secs"}, ...]. + -- Empty/NULL means the single `action` above, which is why every + -- pre-ladder policy keeps working untouched. + steps VARCHAR DEFAULT '', + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL + ) + """, + "CREATE INDEX IF NOT EXISTS idx_session_policy_enabled ON session_policy(enabled)", + # Threshold on the estimated cost of the FLAGGED STRETCH, as opposed to + # min_spend_usd which is the whole session's bill. "Kill anything burning + # more than $5 while stuck" is the rule people actually want, and it needs + # this column. ALTER (not a new CREATE) so existing stores pick it up. + "ALTER TABLE session_policy ADD COLUMN IF NOT EXISTS min_spend_at_risk_usd DOUBLE DEFAULT 0", + # Every policy decision the daemon reached, acted on or not. Doubles as + # the DURABLE one-shot latch: the (session_id, policy_id, step_index) PK + # means each RUNG of a policy's escalation ladder fires at most once per + # session even across a daemon restart, so a restart can never re-kill a + # session it already acted on, and can never replay a ladder from rung 0. + # A policy with no ladder is a one-rung ladder at step_index 0, which is + # exactly the old (session_id, policy_id) latch. + """ + CREATE TABLE IF NOT EXISTS policy_actions ( + session_id VARCHAR NOT NULL, + policy_id VARCHAR NOT NULL, + step_index INTEGER NOT NULL DEFAULT 0, + runtime VARCHAR, + action VARCHAR, + kind VARCHAR, + reason VARCHAR, + evidence BLOB, + enforced BOOLEAN DEFAULT FALSE, + result_ok BOOLEAN DEFAULT FALSE, + result_detail VARCHAR, + created_at BIGINT NOT NULL, + PRIMARY KEY (session_id, policy_id, step_index) + ) + """, + "CREATE INDEX IF NOT EXISTS idx_policy_actions_created ON policy_actions(created_at DESC)", ] @@ -1712,6 +1769,10 @@ def _secs(v): # Absolute root the relative `path` hangs off, so a cloud viewer can show # where on disk the file lives without re-deriving it from the runtime. ("memory_blobs", "root", "VARCHAR"), + # Guard escalation ladders: JSON [{"action","after_secs"}, ...]. Empty on + # existing rows, which normalize_steps() reads as the single `action` — + # so every policy authored before ladders keeps its exact behaviour. + ("session_policy", "steps", "VARCHAR DEFAULT ''"), ] # ── Integrity / hash-chain (Issue #2200) ──────────────────────────────────── @@ -1770,6 +1831,60 @@ def _apply_migrations(conn) -> None: } if col not in existing_cols: conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} {decl}") + if "policy_actions" in existing_tables: + _migrate_policy_actions_ladder(conn) + + +def _migrate_policy_actions_ladder(conn) -> None: + """Widen the Guard latch from (session, policy) to (session, policy, step). + + Escalation ladders need each RUNG to latch independently: with the old + two-column PK, rung 1 would overwrite rung 0's row, the ladder would lose + its place, and a daemon restart would replay it from the top — meaning a + ladder ending in ``kill`` could re-fire. + + DuckDB cannot ALTER a primary key, so this rebuilds the table. Idempotent + via the ``step_index`` column probe: on an already-migrated store it does + nothing. Existing rows land at ``step_index = 0``, which is where a + single-action policy belongs. Failure is contained by the caller (which + logs and continues), and a store that stays on the old shape keeps + working — it just cannot climb past rung 0. + """ + cols = {row[1] for row in conn.execute( + "PRAGMA table_info('policy_actions')").fetchall()} + if "step_index" in cols: + return + conn.execute(""" + CREATE TABLE policy_actions__ladder ( + session_id VARCHAR NOT NULL, + policy_id VARCHAR NOT NULL, + step_index INTEGER NOT NULL DEFAULT 0, + runtime VARCHAR, + action VARCHAR, + kind VARCHAR, + reason VARCHAR, + evidence BLOB, + enforced BOOLEAN DEFAULT FALSE, + result_ok BOOLEAN DEFAULT FALSE, + result_detail VARCHAR, + created_at BIGINT NOT NULL, + PRIMARY KEY (session_id, policy_id, step_index) + ) + """) + conn.execute(""" + INSERT INTO policy_actions__ladder ( + session_id, policy_id, step_index, runtime, action, kind, reason, + evidence, enforced, result_ok, result_detail, created_at + ) + SELECT session_id, policy_id, 0, runtime, action, kind, reason, + evidence, enforced, result_ok, result_detail, created_at + FROM policy_actions + """) + conn.execute("DROP TABLE policy_actions") + conn.execute("ALTER TABLE policy_actions__ladder RENAME TO policy_actions") + conn.execute("CREATE INDEX IF NOT EXISTS idx_policy_actions_created " + "ON policy_actions(created_at DESC)") + log.info("local store: policy_actions migrated to a per-step Guard latch") # ── v7 dedup migration (#1232) ─────────────────────────────────────────────── @@ -1945,6 +2060,29 @@ def _to_blob(value: Any) -> bytes | None: return str(value).encode("utf-8", errors="replace") +def _from_blob(value: Any) -> Any: + """Inverse of :func:`_to_blob` — decode a DuckDB BLOB back to a Python + value. JSON is parsed; anything else comes back as a string. Never + raises: an undecodable blob returns ``None`` so one bad row cannot break + a whole result set.""" + if value is None: + return None + try: + raw = (bytes(value).decode("utf-8", errors="replace") + if isinstance(value, (bytes, bytearray)) else str(value)) + except Exception: + return None + raw = raw.strip() + if not raw: + return None + if raw[0] in "{[": + try: + return json.loads(raw) + except Exception: + return raw + return raw + + # DuckDB defaults to threads == CPU core count, so a single aggregate query # fans out across every core (observed: a 12-core box pegged at ~200% CPU just # re-running query_aggregates). ClawMetry is an observability sidecar, not a @@ -3444,33 +3582,37 @@ def get_session_location( try: if agent_type: rows = self._fetch( - "SELECT cwd, git_branch, metadata FROM sessions " + "SELECT session_id, cwd, git_branch, metadata FROM sessions " "WHERE agent_type = ? AND session_id = ? LIMIT 1", [str(agent_type), sid], ) else: rows = self._fetch( - "SELECT cwd, git_branch, metadata FROM sessions " + "SELECT session_id, cwd, git_branch, metadata FROM sessions " "WHERE session_id = ? LIMIT 1", [sid], ) except Exception: log.debug("local store: get_session_location failed for %s", - sid, exc_info=True) + sid.replace("\r", " ").replace("\n", " "), exc_info=True) return None if not rows: return None row = rows[0] meta: dict[str, Any] = {} - if row[2]: + if row[3]: try: - decoded = json.loads(row[2]) + decoded = json.loads(row[3]) if isinstance(decoded, dict): meta = decoded except Exception: pass - return {"session_id": sid, "cwd": row[0], "git_branch": row[1], - "metadata": meta} + # ``session_id`` is the STORED value (the row's own column), not the + # caller's argument echoed back: control handlers act on this copy so + # a request can name a session but never supply the string a process + # is located or signalled with. + return {"session_id": str(row[0] or sid), "cwd": row[1], + "git_branch": row[2], "metadata": meta} def apply_session_attention(self, items: list[dict[str, Any]]) -> int: """Publish the daemon's INFERRED "needs you" pass onto session rows. @@ -5003,6 +5145,297 @@ def query_repo_activity( d["details"] = text except UnicodeDecodeError: d["details"] = None + # ── Guard policies ──────────────────────────────────────────────────── + def upsert_session_policy(self, policy: dict) -> None: + """Create or update one Guard policy. + + Permissive by design (repo rule: never crash on bad input) — a row + missing ``policy_id`` is dropped rather than raised on, and an + unknown ``action`` is coerced to the safe default ``monitor`` so a + malformed write can never escalate to a kill. + """ + if not isinstance(policy, dict): + return + pid = str(policy.get("policy_id") or "").strip()[:128] + if not pid: + return + from clawmetry.policy_engine import ACTIONS as _ACTIONS + action = str(policy.get("action") or "monitor").strip().lower() + if action not in _ACTIONS: + action = "monitor" + sev = str(policy.get("min_severity") or "info").strip().lower() + # ``critical`` is the money/irreversibility tier: incidents whose + # spend at risk crosses the threshold, and the behavioural findings + # that outlive the session (a disabled protection, a root delete). + if sev not in ("info", "warning", "critical"): + sev = "info" + now_ms = int(time.time() * 1000) + + def _i(key): + try: + return max(0, int(policy.get(key) or 0)) + except (TypeError, ValueError): + return 0 + + def _f(key): + try: + return max(0.0, float(policy.get(key) or 0)) + except (TypeError, ValueError): + return 0.0 + + # Normalize the ladder through the engine so what is STORED is what + # will be EVALUATED — validating here and again at read time would let + # the two drift. A malformed rung is dropped once, at the door. + from clawmetry.policy_engine import normalize_steps as _norm_steps + _steps = _norm_steps(policy) + # A one-rung ladder that merely repeats `action` carries no + # information, so store '' and let it read back as a plain policy. + _steps_json = "" + if len(_steps) > 1 or (_steps and _steps[0]["action"] != action): + _steps_json = json.dumps(_steps, separators=(",", ":"))[:4000] + + with self._write_lock: + self._conn.execute(""" + INSERT INTO session_policy ( + policy_id, name, enabled, scope_runtime, scope_agent_id, + trigger_kind, min_severity, min_repeat, min_duration_s, + min_spend_usd, min_spend_at_risk_usd, action, steps, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (policy_id) DO UPDATE SET + name = excluded.name, + enabled = excluded.enabled, + scope_runtime = excluded.scope_runtime, + scope_agent_id = excluded.scope_agent_id, + trigger_kind = excluded.trigger_kind, + min_severity = excluded.min_severity, + min_repeat = excluded.min_repeat, + min_duration_s = excluded.min_duration_s, + min_spend_usd = excluded.min_spend_usd, + min_spend_at_risk_usd = excluded.min_spend_at_risk_usd, + action = excluded.action, + steps = excluded.steps, + updated_at = excluded.updated_at + """, [ + pid, + str(policy.get("name") or "")[:200], + bool(policy.get("enabled", True)), + str(policy.get("scope_runtime") or "")[:64], + str(policy.get("scope_agent_id") or "")[:64], + str(policy.get("trigger_kind") or "")[:64], + sev, + _i("min_repeat"), + _i("min_duration_s"), + _f("min_spend_usd"), + _f("min_spend_at_risk_usd"), + action, + _steps_json, + now_ms, + now_ms, + ]) + + def delete_session_policy(self, policy_id: str) -> int: + """Delete one Guard policy. Returns rows removed (0 or 1).""" + pid = str(policy_id or "").strip() + if not pid: + return 0 + with self._write_lock: + before = self._conn.execute( + "SELECT COUNT(*) FROM session_policy WHERE policy_id = ?", + [pid]).fetchone() + self._conn.execute( + "DELETE FROM session_policy WHERE policy_id = ?", [pid]) + return int(before[0]) if before else 0 + + def query_session_policies(self, enabled_only: bool = False) -> list: + """All Guard policies, newest-updated first. + + Returns ``[]`` on any error so a policy-table problem degrades to + "no enforcement" rather than breaking the daemon tick. + """ + try: + sql = ("SELECT policy_id, name, enabled, scope_runtime, " + "scope_agent_id, trigger_kind, min_severity, min_repeat, " + "min_duration_s, min_spend_usd, min_spend_at_risk_usd, " + "action, steps, created_at, updated_at FROM session_policy") + if enabled_only: + sql += " WHERE enabled = TRUE" + sql += " ORDER BY updated_at DESC" + rows = self._conn.execute(sql).fetchall() + except Exception: + return [] + cols = ["policy_id", "name", "enabled", "scope_runtime", + "scope_agent_id", "trigger_kind", "min_severity", "min_repeat", + "min_duration_s", "min_spend_usd", "min_spend_at_risk_usd", + "action", "steps", "created_at", "updated_at"] + out = [] + for r in rows: + d = dict(zip(cols, r)) + # Hand back a real list, never the raw JSON column: the engine + # tolerates both but the API and the UI should only ever see one. + raw = d.get("steps") + d["steps"] = [] + if isinstance(raw, str) and raw.strip(): + try: + parsed = json.loads(raw) + if isinstance(parsed, list): + d["steps"] = parsed + except Exception: + pass + out.append(d) + return out + + def policy_already_fired(self, session_id: str, policy_id: str, + step_index: int = 0) -> bool: + """Durable one-shot latch, per RUNG of the policy's ladder. + + True when this policy has already reached a decision at this rung for + this session. Survives daemon restarts (the in-memory alternative + would let a restart re-kill a session it already acted on). Fails + CLOSED — any read error returns True, i.e. we decline to act rather + than risk acting twice. + + ``step_index`` defaults to 0, so a caller that predates escalation + ladders keeps the exact old semantics: a single-action policy fires + at most once per session. + """ + sid = str(session_id or "").strip() + pid = str(policy_id or "").strip() + if not sid or not pid: + return True + try: + step = max(0, int(step_index or 0)) + except (TypeError, ValueError): + return True # fail closed on an unreadable rung + try: + row = self._conn.execute( + "SELECT 1 FROM policy_actions WHERE session_id = ? " + "AND policy_id = ? AND step_index = ? LIMIT 1", + [sid, pid, step]).fetchone() + return row is not None + except Exception: + return True + + def query_policy_ladder_state(self, limit: int = 2000) -> dict: + """How far each escalation ladder has climbed. + + Returns ``{session_id: {policy_id: {"last_step", "last_fired_at"}}}`` + where ``last_fired_at`` is epoch SECONDS (the engine's clock unit; + the column is milliseconds). + + This is what makes a ladder durable: the daemon reads it each tick and + hands it to ``policy_engine.evaluate``, so a restart resumes at the + rung it had reached instead of replaying the ladder from the top — a + replay that, for a ladder ending in ``kill``, would mean killing a + session twice. + + ``{}`` on any error, which reads as "no ladder has started". That is + the safe direction: every ladder is treated as being at rung 0, and + rung 0 is still protected by the per-step latch. + """ + try: + lim = max(1, min(int(limit or 2000), 20000)) + except (TypeError, ValueError): + lim = 2000 + try: + rows = self._conn.execute( + "SELECT session_id, policy_id, MAX(step_index) AS last_step, " + "MAX(created_at) AS last_fired_at FROM policy_actions " + "GROUP BY session_id, policy_id " + "ORDER BY last_fired_at DESC LIMIT ?", [lim]).fetchall() + except Exception: + return {} + out: dict = {} + for sid, pid, last_step, last_fired_ms in rows: + if not sid or not pid: + continue + try: + step = int(last_step or 0) + fired = float(last_fired_ms or 0) / 1000.0 + except (TypeError, ValueError): + continue + out.setdefault(str(sid), {})[str(pid)] = { + "last_step": step, + "last_fired_at": fired, + } + return out + + def record_policy_action(self, session_id: str, policy_id: str, + runtime: str = "", action: str = "", + kind: str = "", reason: str = "", + evidence: Any = None, enforced: bool = False, + result_ok: bool = False, + result_detail: str = "", + step_index: int = 0) -> None: + """Record one policy decision (acted on or not) at one ladder rung. + + Written for EVERY decision including dry-run ``monitor`` ones, which + is what makes monitor mode honest: the user can read exactly what + would have fired. Also serves as the latch (PK conflict = already + fired), so this must be written before/regardless of whether the + actuator succeeded. + + ``step_index`` is part of the PK, so each rung of a ladder gets its + own durable row. Re-recording the SAME rung (the daemon writes once + before acting and once after, to fill in the result) updates that row + rather than inserting a second one. + """ + sid = str(session_id or "").strip()[:128] + pid = str(policy_id or "").strip()[:128] + if not sid or not pid: + return + try: + step = max(0, int(step_index or 0)) + except (TypeError, ValueError): + step = 0 + try: + with self._write_lock: + self._conn.execute(""" + INSERT INTO policy_actions ( + session_id, policy_id, step_index, runtime, action, + kind, reason, evidence, enforced, result_ok, + result_detail, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (session_id, policy_id, step_index) DO UPDATE SET + result_ok = excluded.result_ok, + result_detail = excluded.result_detail, + enforced = excluded.enforced + """, [ + sid, pid, step, + str(runtime or "")[:64], + str(action or "")[:32], + str(kind or "")[:64], + str(reason or "")[:400], + _to_blob(evidence) if evidence is not None else None, + bool(enforced), + bool(result_ok), + str(result_detail or "")[:400], + int(time.time() * 1000), + ]) + except Exception: + return + + def query_policy_actions(self, limit: int = 50) -> list: + """Recent policy decisions, newest first. ``[]`` on any error.""" + try: + lim = max(1, min(int(limit or 50), 500)) + except (TypeError, ValueError): + lim = 50 + try: + rows = self._conn.execute( + "SELECT session_id, policy_id, step_index, runtime, action, " + "kind, reason, evidence, enforced, result_ok, result_detail, " + "created_at FROM policy_actions ORDER BY created_at DESC " + "LIMIT ?", [lim]).fetchall() + except Exception: + return [] + cols = ["session_id", "policy_id", "step_index", "runtime", "action", + "kind", "reason", "evidence", "enforced", "result_ok", + "result_detail", "created_at"] + out = [] + for r in rows: + d = dict(zip(cols, r)) + d["evidence"] = _from_blob(d.get("evidence")) out.append(d) return out diff --git a/clawmetry/policy_engine.py b/clawmetry/policy_engine.py new file mode 100644 index 0000000000..962704ccc3 --- /dev/null +++ b/clawmetry/policy_engine.py @@ -0,0 +1,483 @@ +"""Guard policies — turn a detector incident into an enforcement decision. + +This module is the wire between two halves that already existed but never +touched: :mod:`clawmetry.detectors` (which finds agents that have gone off +track) and :mod:`clawmetry.process_control` (which can pause/stop/kill them). +Before this, the only edge between detection and action was a human noticing a +banner and pressing Stop. + +**This module is pure.** ``evaluate()`` does no I/O, opens no store, sends no +signals. It takes incidents + policies + facts and returns decisions. The +daemon (``sync.py::_emit_detector_incidents``) does the reading, the +dispatching and the auditing. Keeping it pure means the whole matching +surface is unit-testable without a daemon, a DuckDB file or a live agent — +the same split ``detectors.py`` uses. + +A policy row (see ``local_store.session_policy``):: + + { + "policy_id": str, + "enabled": bool, + "scope_runtime": str, # "" = every runtime + "scope_agent_id": str, # "" = every agent + "trigger_kind": str, # "" = any detector kind + "min_severity": "info" | "warning", + "min_repeat": int, # incident count must be >= this + "min_duration_s": int, # session bad for at least this long + "min_spend_usd": float, # session cost must be >= this + "min_spend_at_risk_usd": float, # the FLAGGED STRETCH must be worth + # >= this (an estimate the detector attaches; + # 0 or missing means the threshold is unused) + "action": "monitor" | "alert" | "pause" | "stop" | "kill", + "steps": list, # OPTIONAL escalation ladder, see below + } + +**Escalation ladders.** A single action fired once is not how operations +actually respond to a stuck agent — the real shape is *pause it, tell me, +give it five minutes, then kill it if it is still stuck*. ``steps`` expresses +that as an ordered list:: + + "steps": [ + {"action": "pause", "after_secs": 0}, + {"action": "alert", "after_secs": 0}, + {"action": "kill", "after_secs": 300} + ] + +Semantics, chosen so a ladder can never act faster than a plain policy: + +* Step 0 fires when the policy first matches; its ``after_secs`` is ignored + (use ``min_duration_s`` for a delay *before* the first action). +* Step *n* becomes due ``after_secs`` seconds after step *n-1* actually + fired — not after the incident started — so a ladder measures the time the + agent was given to recover. +* A due step only fires if the session is STILL matching this tick. That is + what makes "kill if still stuck" mean *still stuck*: if the detector stops + reporting, the ladder simply stops. +* Every step passes through the same three locks as a plain action. A + ``kill`` step on a node with ``CLAWMETRY_POLICY_ENFORCE=0`` is recorded as + a dry run exactly like a ``kill`` policy would be. +* The durable latch is per ``(session, policy, step)``, so a daemon restart + mid-ladder resumes at the right rung instead of replaying it. + +A policy with no ``steps`` is a one-step ladder built from its ``action``, +which is why every existing policy keeps behaving identically. + +All thresholds are AND-ed. An unset threshold (0) never blocks a match, so a +policy with everything zeroed fires on the first matching incident. + +A decision:: + + { + "policy_id": str, + "session_id": str, + "runtime": str, + "cwd": str, + "action": str, + "kind": str, # detector kind that triggered it + "reason": str, # plain words, shown in the UI and audit row + "evidence": dict, # the numbers that satisfied the thresholds + "step_index": int, # which rung of the ladder this is (0-based) + "step_count": int, # how many rungs the ladder has + "is_final_step": bool, # nothing escalates after this one + "next_action": str, # "" when this was the last rung + "next_after_secs": int, # how long until the next rung becomes due + } + +Safety invariants enforced here (the daemon adds two more — the enforce env +flag and the one-shot latch): + +* **At most one decision per session.** Several policies can match the same + session; firing each would mean several signals at one process. The + strongest action wins, ties broken by ``policy_id`` so the choice is + deterministic and reproducible in tests. +* **``monitor`` is a real decision, not a skip.** It returns a decision the + daemon records to the audit trail without acting. That is what makes + dry-run honest: you can see exactly what *would* have fired. +""" +from __future__ import annotations + +from typing import Any, Dict, Iterable, List, Optional + +# Action ladder, weakest first. Order IS the escalation order and the +# strongest-wins comparison; do not reorder without updating the UI copy. +ACTIONS = ("monitor", "alert", "pause", "stop", "kill") +_ACTION_RANK = {name: i for i, name in enumerate(ACTIONS)} + +# Actions that actually signal the agent's process. Everything below `pause` +# only writes rows. The daemon uses this to decide whether the enforce flag +# and the latch apply. +ACTUATING_ACTIONS = frozenset({"pause", "stop", "kill"}) + +# ``critical`` is the tier the detectors reserve for two things: an incident +# whose spend at risk crossed ``detectors.CRITICAL_SPEND_USD``, and a +# behavioural finding that outlives the session (a disabled protection, a +# recursive delete at a home root). A policy can require it, which is how you +# write "kill only the expensive or irreversible ones". +_SEVERITY_RANK = {"info": 0, "warning": 1, "critical": 2} + + +def action_rank(action: str) -> int: + """Position on the escalation ladder; unknown actions sort weakest.""" + return _ACTION_RANK.get(str(action or "").strip().lower(), -1) + + +def is_actuating(action: str) -> bool: + """True when the action sends a signal to a real process.""" + return str(action or "").strip().lower() in ACTUATING_ACTIONS + + +def _severity_rank(sev: Any) -> int: + return _SEVERITY_RANK.get(str(sev or "warning").strip().lower(), 1) + + +def _incident_count(incident: Dict[str, Any]) -> int: + """Best-effort 'how many times' number behind an incident. + + Detectors put their count under different evidence keys depending on the + kind (a loop counts repeats, no-progress counts tool calls, a repeated + failure counts failures). We take the largest of the known keys so one + ``min_repeat`` threshold reads sensibly against every detector. + """ + ev = incident.get("evidence") + if not isinstance(ev, dict): + return 0 + best = 0 + for key in ("repeat_count", "repeats", "total_tool_calls", "tool_calls", + "failure_count", "failures", "count"): + try: + val = int(ev.get(key) or 0) + except (TypeError, ValueError): + continue + if val > best: + best = val + return best + + +def _as_float(value: Any) -> float: + try: + return float(value or 0) + except (TypeError, ValueError): + return 0.0 + + +def _as_int(value: Any) -> int: + try: + return int(value or 0) + except (TypeError, ValueError): + return 0 + + +def _scope_matches(policy_value: Any, actual: Any) -> bool: + """Empty scope means 'all'. Comparison is case-insensitive.""" + want = str(policy_value or "").strip().lower() + if not want: + return True + return want == str(actual or "").strip().lower() + + +# A ladder longer than this is almost certainly a mistake, and each rung is a +# durable row plus a latch check per tick. Extra rungs are dropped, not +# rejected, so a bad rule degrades instead of disabling the whole policy. +MAX_LADDER_STEPS = 8 + +# Upper bound on a rung's delay (24h). A typo of 300000 instead of 300 would +# otherwise park a ladder forever with no sign anything was wrong. +MAX_STEP_DELAY_SECS = 86400 + + +def normalize_steps(policy: Dict[str, Any]) -> List[Dict[str, Any]]: + """The policy's escalation ladder as ``[{"action", "after_secs"}, ...]``. + + A policy with no usable ``steps`` becomes a ONE-step ladder built from its + ``action``, which is what keeps every pre-ladder policy behaving exactly + as before. + + Malformed rungs are DROPPED rather than coerced. Coercing an unrecognised + action to a default would silently change what a rule does to someone's + agent; dropping it means the ladder is shorter than authored, which the + UI can show. Step 0's delay is forced to 0 (see the module docstring) and + every delay is clamped to ``MAX_STEP_DELAY_SECS``. + """ + fallback = str(policy.get("action") or "monitor").strip().lower() + if fallback not in _ACTION_RANK: + fallback = "monitor" + + raw = policy.get("steps") + if isinstance(raw, str): + # The store round-trips steps as JSON text; tolerate either form so a + # policy read straight from DuckDB and one posted from the UI behave + # identically. + import json + try: + raw = json.loads(raw) + except Exception: # noqa: BLE001 + raw = None + + steps: List[Dict[str, Any]] = [] + if isinstance(raw, (list, tuple)): + for entry in raw: + if len(steps) >= MAX_LADDER_STEPS: + break + if not isinstance(entry, dict): + continue + act = str(entry.get("action") or "").strip().lower() + if act not in _ACTION_RANK: + continue + delay = _as_int(entry.get("after_secs")) + delay = max(0, min(delay, MAX_STEP_DELAY_SECS)) + steps.append({"action": act, + "after_secs": 0 if not steps else delay}) + + if not steps: + return [{"action": fallback, "after_secs": 0}] + return steps + + +def _due_step(steps: List[Dict[str, Any]], state: Optional[Dict[str, Any]], + now: float) -> Optional[int]: + """Which rung, if any, should fire right now. ``None`` = nothing due. + + ``state`` is what the store knows about this ``(session, policy)`` pair: + ``{"last_step": int, "last_fired_at": float-epoch-seconds}``. No state + means the ladder has not started, so rung 0 is due. + + Returns None when the ladder is finished or the next rung's delay has not + elapsed. A missing/garbage ``last_fired_at`` is treated as "just fired", + which DELAYS the next rung rather than firing it early — the safe way to + be wrong when the next rung might be a kill. + """ + if not steps: + return None + if not isinstance(state, dict): + return 0 + last = _as_int(state.get("last_step")) + if state.get("last_step") is None: + return 0 + nxt = last + 1 + if nxt >= len(steps): + return None # ladder exhausted + try: + fired_at = float(state.get("last_fired_at") or 0) + except (TypeError, ValueError): + fired_at = 0.0 + if fired_at <= 0: + fired_at = float(now) # unknown -> wait the full delay from now + if float(now) - fired_at < float(steps[nxt]["after_secs"]): + return None + return nxt + + +def _match(policy: Dict[str, Any], incident: Dict[str, Any], + facts: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Return the evidence dict when `policy` matches, else None. + + Returning the evidence rather than a bool means the decision carries the + exact numbers that satisfied each threshold, so the audit row can say + *why* rather than just *that*. + """ + if not policy.get("enabled", True): + return None + + action = str(policy.get("action") or "monitor").strip().lower() + if action not in _ACTION_RANK: + return None # unknown action: refuse rather than guess + + kind = str(incident.get("kind") or "").strip() + want_kind = str(policy.get("trigger_kind") or "").strip() + if want_kind and want_kind != kind: + return None + + if not _scope_matches(policy.get("scope_runtime"), incident.get("runtime")): + return None + if not _scope_matches(policy.get("scope_agent_id"), facts.get("agent_id")): + return None + + if _severity_rank(incident.get("severity")) < _severity_rank( + policy.get("min_severity") or "info"): + return None + + count = _incident_count(incident) + min_repeat = _as_int(policy.get("min_repeat")) + if min_repeat and count < min_repeat: + return None + + duration_s = _as_float(facts.get("bad_for_seconds")) + min_duration = _as_float(policy.get("min_duration_s")) + if min_duration and duration_s < min_duration: + return None + + spend = _as_float(facts.get("cost_usd")) + min_spend = _as_float(policy.get("min_spend_usd")) + if min_spend and spend < min_spend: + return None + + # What the flagged stretch — not the whole session — is estimated to have + # cost. The detector computes it; we carry it into the decision so the + # audit row can answer "was acting on this worth it?" in dollars. + spend_at_risk = _as_float(incident.get("spend_at_risk_usd")) + min_at_risk = _as_float(policy.get("min_spend_at_risk_usd")) + if min_at_risk and spend_at_risk < min_at_risk: + return None + + return { + "count": count, + "bad_for_seconds": int(duration_s), + "cost_usd": round(spend, 4), + "spend_at_risk_usd": round(spend_at_risk, 4), + "spend_basis": str(incident.get("spend_basis") or "unknown"), + "severity": str(incident.get("severity") or "warning"), + "thresholds": { + "min_repeat": min_repeat, + "min_duration_s": int(min_duration), + "min_spend_usd": round(min_spend, 4), + "min_spend_at_risk_usd": round(min_at_risk, 4), + "min_severity": str(policy.get("min_severity") or "info"), + }, + } + + +def _reason(policy: Dict[str, Any], incident: Dict[str, Any], + evidence: Dict[str, Any], step_index: int = 0, + steps: Optional[List[Dict[str, Any]]] = None) -> str: + """Plain-words explanation, shown in the UI and stored in the audit row. + + Deliberately states the observation and the threshold it crossed, so a + user reading it later can tell whether the policy was well-tuned. For a + multi-rung ladder it also says which rung this is and what happens next, + because "pause" reads very differently when the next line is "then kill + in 5m if still stuck". + """ + steps = steps or normalize_steps(policy) + idx = max(0, min(int(step_index), len(steps) - 1)) + action = steps[idx]["action"] + title = str(incident.get("title") or incident.get("kind") or "incident") + bits: List[str] = [] + if evidence.get("count"): + bits.append(f"{evidence['count']} events") + if evidence.get("bad_for_seconds"): + bits.append(f"{int(evidence['bad_for_seconds'] // 60)}m without progress") + if evidence.get("cost_usd"): + bits.append(f"${evidence['cost_usd']:.2f} spent") + if evidence.get("spend_at_risk_usd"): + # The number that decides whether this was worth acting on. + bits.append(f"~${evidence['spend_at_risk_usd']:.2f} at risk") + detail = ", ".join(bits) if bits else "threshold met" + verb = "would " + action if action == "monitor" else action + text = f"{title} ({detail}) -> {verb}" + if len(steps) > 1: + text += f" [step {idx + 1}/{len(steps)}]" + if idx + 1 < len(steps): + nxt = steps[idx + 1] + text += (f", then {nxt['action']} in " + f"{_humanize_secs(int(nxt['after_secs']))} if still matching") + return text[:400] + + +def _humanize_secs(secs: int) -> str: + """``300`` -> ``5m``. Short enough to sit inside a 400-char reason.""" + secs = max(0, int(secs)) + if secs < 60: + return f"{secs}s" + if secs < 3600: + return f"{secs // 60}m" + return f"{secs // 3600}h" + + +def evaluate(incidents: Iterable[Dict[str, Any]], + policies: Iterable[Dict[str, Any]], + session_facts: Optional[Dict[str, Dict[str, Any]]] = None, + ladder_state: Optional[Dict[str, Dict[str, Dict[str, Any]]]] = None, + now: Optional[float] = None, + ) -> List[Dict[str, Any]]: + """Match incidents against policies and return at most one decision per + session (the strongest matching action wins). + + ``session_facts`` maps ``session_id`` -> ``{"cost_usd", "bad_for_seconds", + "runtime", "cwd", "agent_id"}``. A missing entry is treated as all-zero + facts, which means spend/duration thresholds simply never match for that + session rather than matching by accident. + + ``ladder_state`` maps ``session_id -> policy_id -> + {"last_step", "last_fired_at"}`` — how far each ladder has already got, + read from the audit table by the daemon. Omitting it means every matching + policy is at rung 0, which is exactly the pre-ladder behaviour. + + ``now`` is the evaluation clock in epoch seconds, injected so ladder + timing is testable without sleeping. Defaults to wall-clock. + + Pure: no I/O, no exceptions raised for bad input rows (malformed policies + and incidents are skipped, matching the repo's never-crash-on-bad-input + rule). + """ + import time as _time + + facts_by_session = session_facts or {} + state_by_session = ladder_state or {} + clock = float(now) if now is not None else _time.time() + policy_list = [p for p in (policies or []) if isinstance(p, dict)] + if not policy_list: + return [] + + best_by_session: Dict[str, Dict[str, Any]] = {} + + for incident in incidents or []: + if not isinstance(incident, dict): + continue + session_id = str(incident.get("session_id") or "").strip() + if not session_id: + continue # an incident with no session cannot be acted on + facts = facts_by_session.get(session_id) or {} + session_state = state_by_session.get(session_id) or {} + + for policy in policy_list: + evidence = _match(policy, incident, facts) + if evidence is None: + continue + + policy_id = str(policy.get("policy_id") or "") + # Which rung of this policy's ladder is due right now. A ladder + # that has finished, or whose next rung has not come round yet, + # yields no decision at all this tick. + steps = normalize_steps(policy) + step_index = _due_step(steps, session_state.get(policy_id), clock) + if step_index is None: + continue + step = steps[step_index] + action = step["action"] + is_final = step_index >= len(steps) - 1 + nxt = None if is_final else steps[step_index + 1] + + candidate = { + "policy_id": policy_id, + "session_id": session_id, + "runtime": str(incident.get("runtime") or facts.get("runtime") or ""), + "cwd": str(facts.get("cwd") or ""), + "action": action, + "kind": str(incident.get("kind") or ""), + "reason": _reason(policy, incident, evidence, + step_index=step_index, steps=steps), + "evidence": evidence, + "step_index": step_index, + "step_count": len(steps), + "is_final_step": is_final, + "next_action": "" if nxt is None else nxt["action"], + "next_after_secs": 0 if nxt is None else int(nxt["after_secs"]), + } + + current = best_by_session.get(session_id) + if current is None: + best_by_session[session_id] = candidate + continue + # Strongest action wins; deterministic tie-break on policy_id so + # the same inputs always produce the same decision. + cur_rank = action_rank(current["action"]) + new_rank = action_rank(action) + if new_rank > cur_rank or ( + new_rank == cur_rank and policy_id < current["policy_id"]): + best_by_session[session_id] = candidate + + # Stable output ordering: strongest first, then session id. + return sorted( + best_by_session.values(), + key=lambda d: (-action_rank(d["action"]), d["session_id"]), + ) diff --git a/clawmetry/process_control.py b/clawmetry/process_control.py index ff048853ae..64ce63364b 100644 --- a/clawmetry/process_control.py +++ b/clawmetry/process_control.py @@ -12,9 +12,13 @@ * **Dependency-light & host-testable.** No Flask, no DuckDB, no cloud imports. ``psutil`` is used *if available* (import-guarded) and we degrade to ``ps`` / ``lsof`` shelling otherwise, so OSS keeps deps minimal. -* **Cross-platform.** macOS and Linux are first-class. Windows / other POSIX - return an honest ``unsupported`` result rather than guessing (POSIX job-control - signals like SIGSTOP/SIGCONT do not exist on Windows). +* **Cross-platform.** macOS, Linux AND Windows are first-class. Windows has no + POSIX job-control signals, so each action maps to its native equivalent: + pause/resume -> ``NtSuspendProcess``/``NtResumeProcess`` (what psutil's + ``suspend()`` calls), stop -> a console Ctrl+C delivered from a short-lived + helper process, kill -> ``taskkill /T`` then ``TerminateProcess`` over the + tree. Every other platform still returns an honest ``unsupported`` result + rather than guessing. * **Never crashes.** A missing file, a dead pid, or a permission error returns ``ok=False`` with a ``reason`` — it never raises into the caller. Respects the never-hang contract: every wait is bounded, no unbounded loops. @@ -42,6 +46,7 @@ import logging import os +import re import signal import subprocess import sys @@ -58,8 +63,217 @@ _IS_MACOS = sys.platform == "darwin" _IS_LINUX = sys.platform.startswith("linux") +_IS_WINDOWS = os.name == "nt" _POSIX = os.name == "posix" and (_IS_MACOS or _IS_LINUX) +# Platforms where the actuators are implemented at all. POSIX uses signals; +# Windows uses the native equivalents (see the Win32 section below). Anything +# else (a BSD, a stripped container without ps) still gets the honest +# ``unsupported_platform`` refusal rather than a button that silently no-ops. +_CONTROLLABLE_PLATFORM = _POSIX or _IS_WINDOWS + + +def platform_support() -> Dict[str, Any]: + """What this OS can actually do, for the UI to state plainly. + + ``routes/guard.py`` renders this next to the buttons: a control that + cannot work must say why, not fail silently when pressed. + """ + if _POSIX: + return {"controllable": True, "platform": sys.platform, + "mechanism": "posix_signals", + "actions": ["pause", "resume", "stop", "kill"], "reason": ""} + if _IS_WINDOWS: + return {"controllable": True, "platform": "win32", + "mechanism": "win32_native", + "actions": ["pause", "resume", "stop", "kill"], + # Said out loud because it is a real behavioural difference: + # a Windows console app that installs no Ctrl+C handler will + # not stop, where a POSIX agent almost always honours SIGINT. + "reason": "", + "note": ("Windows has no SIGSTOP/SIGINT: pause suspends threads " + "via NtSuspendProcess and stop delivers a console " + "Ctrl+C, which an app that ignores Ctrl+C may not " + "honour")} + return {"controllable": False, "platform": sys.platform, + "mechanism": "", "actions": [], + "reason": f"Process control is not implemented on {sys.platform}"} + + +# ────────────────────────────────────────────────────────────────────────── +# The shared guard every public control helper runs through +# ────────────────────────────────────────────────────────────────────────── +def _guarded(action_name: str, runtime: str, session_id: str, cwd: str, + fn) -> Dict[str, Any]: + """Resolve the session, run the pid-reuse guard, then call ``fn(pid)``. + + Returns a structured result. Never raises. ``fn`` is one of the signal + helpers (stop_turn / graceful_kill / pause / resume). + """ + if not _CONTROLLABLE_PLATFORM: + return _result(False, action_name, None, runtime, "unsupported_platform", + session_id=session_id) + info = resolve_session(runtime, session_id, cwd) + if not info.get("ok"): + return _result(False, action_name, None, runtime, + info.get("reason") or "unresolved", + session_id=session_id, unsupported=info.get("unsupported")) + pid = info["pid"] + ok, reason = verify_pid(pid, info.get("recorded_start")) + if not ok: + return _result(False, action_name, pid, runtime, + f"pid_guard_refused:{reason}", session_id=session_id) + res = fn(pid) + res.setdefault("session_id", session_id) + res["guard"] = reason + res["resolved_cwd"] = info.get("cwd") + return res + + +# ────────────────────────────────────────────────────────────────────────── +# Capability answers — what can we ACTUALLY do to this session, right now +# +# One place, because the answer has three independent axes (the OS, the +# runtime, and — for OpenClaw — whether the enforcement proxy is in the loop) +# and every caller needs the same verdict. ``routes/guard.py`` renders it next +# to the buttons and ``sync.py`` records it on the policy decision, so a +# control that cannot work says why instead of failing silently when pressed. +# ────────────────────────────────────────────────────────────────────────── +_CLAWMETRY_HOME = os.path.join(os.path.expanduser("~"), ".clawmetry") +_PROXY_PID_FILE = os.path.join(_CLAWMETRY_HOME, "proxy.pid") + + +def enforcement_proxy_status() -> Dict[str, Any]: + """Is the optional enforcement proxy actually running on this node? + + Reads ``~/.clawmetry/proxy.pid`` directly rather than importing + ``clawmetry.proxy`` — this module stays dependency-light, and the pid file + IS the contract (``proxy.run_proxy`` writes it, ``proxy.proxy_status`` + reads it the same way). A stale pid file is treated as not-running. + """ + try: + with open(_PROXY_PID_FILE, "r") as fh: + pid = int((fh.read() or "").strip()) + except Exception: # noqa: BLE001 — absent / unreadable / not a number + return {"running": False, "pid": None, "reason": "no proxy pid file"} + if pid <= 0: + return {"running": False, "pid": None, "reason": "invalid proxy pid file"} + if is_alive(pid): + return {"running": True, "pid": pid, "reason": ""} + return {"running": False, "pid": pid, "reason": "stale proxy pid file"} + + +def openclaw_pause_capability() -> Dict[str, Any]: + """What an OpenClaw "pause" actually does on this node. + + OpenClaw has no pause primitive. All ClawMetry can do is write the HITL + flag file ``~/.clawmetry/hitl/pause_``, and the ONLY thing + that enforces it is ``clawmetry.proxy._is_session_hitl_paused`` — so when + the enforcement proxy is not running, that file changes nothing at all. + + This distinction is the whole point of the function. Reporting "the proxy + refuses further LLM calls" on a node with no proxy is a pause that claims + to have stopped an agent that is still running, which is worse than + refusing outright. + """ + proxy = enforcement_proxy_status() + if proxy.get("running"): + return { + "effective": True, + "mechanism": "proxy_hitl", + "proxy_pid": proxy.get("pid"), + "detail": ("OpenClaw has no pause primitive; the enforcement " + "proxy holds this session's LLM calls while the HITL " + "pause flag is set"), + } + return { + "effective": False, + "mechanism": "none", + "proxy_pid": None, + "detail": ("OpenClaw has no pause primitive and the enforcement proxy " + "is not running on this node, so the HITL pause flag is " + "recorded but nothing enforces it — the agent keeps " + "running. Use Stop (gateway task cancel) instead, or start " + "the proxy with `clawmetry proxy start`."), + } + + +def runtime_control_support(runtime: str, session_id: str = "", + cwd: str = "") -> Dict[str, Any]: + """Per-session control capability: ``{controllable, actions, reason, …}``. + + Answered per SESSION, not per runtime, because two of them differ session + by session: + + * ``cursor`` — a CLI session (``cursor-agent``) is a real process tree and + IS controllable; a conversation inside the Cursor editor shares the one + IDE process and is not. Only the resolver can tell them apart, so we ask + it rather than blanket-refusing the runtime (which is what the Guard tab + used to do, hiding the buttons for sessions that would have worked). + * ``openclaw`` — Stop works (gateway task cancel), Pause depends on + whether the enforcement proxy is in the loop right now. + + Never raises: any resolver error degrades to "not controllable, here's + why". + """ + rt = (runtime or "").strip().lower() + plat = platform_support() + if not plat.get("controllable"): + return {"controllable": False, "actions": [], "runtime": rt, + "reason": plat.get("reason", ""), "platform": plat} + + if rt == "openclaw": + # Stop/kill go through the OpenClaw CLI task cancel in sync.py, not + # through signals, so they work regardless of the resolver. + pause_cap = openclaw_pause_capability() + actions = ["stop", "kill"] + if pause_cap["effective"]: + actions = ["pause", "resume"] + actions + return {"controllable": True, "runtime": rt, "actions": actions, + "reason": "", "no_pause": not pause_cap["effective"], + "pause_capability": pause_cap, + "note": pause_cap["detail"], "platform": plat} + + if rt in SPLIT_SUPPORT_RUNTIMES: + info = resolve_session(rt, session_id, cwd) + if info.get("ok"): + return {"controllable": True, "runtime": rt, + "actions": ["pause", "resume", "stop", "kill"], + "reason": "", "resolved_pid": info.get("pid"), + "platform": plat} + return {"controllable": False, "runtime": rt, "actions": [], + "reason": _SPLIT_SUPPORT_REASONS.get( + info.get("reason") or "", + info.get("reason") or "session could not be located"), + "platform": plat} + + if rt == "claude_code" or rt in SUPPORTED_RUNTIMES: + return {"controllable": True, "runtime": rt, + "actions": ["pause", "resume", "stop", "kill"], + "reason": "", "platform": plat} + + return {"controllable": False, "runtime": rt, "actions": [], + "reason": f"No signal support for {rt or 'unknown runtime'}", + "platform": plat} + + +# Resolver reasons rendered as something an operator can act on. +_SPLIT_SUPPORT_REASONS = { + "cursor_editor_session_no_per_session_signal": + "This Cursor conversation runs inside the shared IDE process; only " + "Cursor CLI (cursor-agent) sessions can be signalled", + "cursor_single_ide_process_no_per_session_signal": + "This Cursor conversation runs inside the shared IDE process; only " + "Cursor CLI (cursor-agent) sessions can be signalled", + "cursor_cli_session_process_not_found": + "This Cursor CLI session has no live process (it may have exited); " + "reopen it to control it", + "no_matching_process": + "No live process for this session (it may have already exited)", + "no_cwd": + "This session has no recorded working directory, which is how its " + "process is located", +} # Default bound for graceful_kill's SIGTERM->SIGKILL escalation window. _DEFAULT_GRACE_SECS = 5.0 @@ -166,6 +380,8 @@ def _proc_start_epoch(pid: int) -> Optional[float]: return float(_psutil.Process(int(pid)).create_time()) except Exception: # noqa: BLE001 - dead/zombie/perm return None + if _IS_WINDOWS: + return _win_proc_start_epoch(pid) if _IS_LINUX: try: with open(f"/proc/{int(pid)}/stat", "r") as fh: @@ -196,6 +412,381 @@ def _linux_btime() -> Optional[float]: return None +# ────────────────────────────────────────────────────────────────────────── +# Win32 primitives +# +# Windows has no POSIX job-control signals, so each action maps to the native +# equivalent. Everything here is ctypes against kernel32/ntdll — no new +# dependency — and every call is import-guarded and exception-swallowed so a +# locked-down host degrades to an honest failure instead of raising. +# +# pause/resume NtSuspendProcess / NtResumeProcess. This is exactly what +# psutil's Process.suspend()/resume() call on Windows; we do it +# directly so a psutil-less install keeps the capability. +# stop A console Ctrl+C. It cannot be sent to a single pid: the +# sender must attach to the target's console and raise the +# event for the whole console (group 0). We therefore do it +# from a short-lived DETACHED helper process — running +# AttachConsole in the daemon would swap the daemon's console +# and the Ctrl+C would hit the daemon itself. +# kill taskkill /T for the graceful pass (posts WM_CLOSE / console +# close to the tree), then TerminateProcess per surviving pid. +# ────────────────────────────────────────────────────────────────────────── +_WIN_PROCESS_TERMINATE = 0x0001 +_WIN_PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 +_WIN_PROCESS_SUSPEND_RESUME = 0x0800 +_WIN_TH32CS_SNAPPROCESS = 0x00000002 +_WIN_DETACHED_PROCESS = 0x00000008 +# FILETIME epoch (1601-01-01) to unix epoch (1970-01-01), in seconds. +_WIN_FILETIME_EPOCH_DELTA = 11644473600.0 + + +_WIN_K32 = None +_WIN_K32_TRIED = False + +# Allowlist of ntdll routines _win_ntdll_call may invoke. Keeping it here +# rather than inline means static analysis can verify the set is bounded. +_WIN_NTDLL_ALLOWED = frozenset({"NtSuspendProcess", "NtResumeProcess"}) + + +def _win_kernel32(): + """kernel32 with argtypes/restypes declared, or None off Windows. + + Declaring the prototypes is NOT optional. ctypes defaults every restype to + ``c_int``; a Win64 ``HANDLE`` is pointer-sized, so an undeclared + ``OpenProcess`` silently truncates the handle to 32 bits and every + subsequent call against it fails with ERROR_INVALID_HANDLE. The whole + Windows control path would be reachable and permanently broken. + + Cached: the prototypes only need setting once, and the actuators call this + several times per action. + """ + global _WIN_K32, _WIN_K32_TRIED + if _WIN_K32 is not None or _WIN_K32_TRIED: + return _WIN_K32 + _WIN_K32_TRIED = True + if not _IS_WINDOWS: + return None + try: + import ctypes + from ctypes import wintypes + + k = ctypes.WinDLL("kernel32", use_last_error=True) + k.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + k.OpenProcess.restype = wintypes.HANDLE + k.CloseHandle.argtypes = [wintypes.HANDLE] + k.CloseHandle.restype = wintypes.BOOL + k.TerminateProcess.argtypes = [wintypes.HANDLE, wintypes.UINT] + k.TerminateProcess.restype = wintypes.BOOL + k.GetProcessTimes.argtypes = [ + wintypes.HANDLE, ctypes.POINTER(wintypes.FILETIME), + ctypes.POINTER(wintypes.FILETIME), + ctypes.POINTER(wintypes.FILETIME), + ctypes.POINTER(wintypes.FILETIME), + ] + k.GetProcessTimes.restype = wintypes.BOOL + k.CreateToolhelp32Snapshot.argtypes = [wintypes.DWORD, wintypes.DWORD] + k.CreateToolhelp32Snapshot.restype = wintypes.HANDLE + # Process32FirstW/NextW take a LPPROCESSENTRY32W we declare locally; + # c_void_p is the honest stand-in for "pointer to that struct". + k.Process32FirstW.argtypes = [wintypes.HANDLE, ctypes.c_void_p] + k.Process32FirstW.restype = wintypes.BOOL + k.Process32NextW.argtypes = [wintypes.HANDLE, ctypes.c_void_p] + k.Process32NextW.restype = wintypes.BOOL + _WIN_K32 = k + return k + except Exception: # noqa: BLE001 + return None + + +def _win_open_process(access: int, pid: int): + """OpenProcess handle for ``pid``, or None. Caller must CloseHandle.""" + k = _win_kernel32() + if k is None: + return None + try: + handle = k.OpenProcess(int(access), False, int(pid)) + return handle or None + except Exception: # noqa: BLE001 + return None + + +def _win_close_handle(handle) -> None: + k = _win_kernel32() + if k is None or not handle: + return + try: + k.CloseHandle(handle) + except Exception: # noqa: BLE001 + pass + + +def _win_proc_start_epoch(pid: int) -> Optional[float]: + """Process creation time as a unix epoch, via GetProcessTimes. + + This is what makes the pid-reuse guard work on a psutil-less Windows box. + Without it ``_proc_start_token`` returns None, ``verify_pid`` fails CLOSED + with ``start_unverifiable``, and every control action is refused — the + actuators below would be reachable but permanently blocked. + """ + if not _IS_WINDOWS or pid is None or int(pid) <= 0: + return None + try: + import ctypes + from ctypes import wintypes + except Exception: # noqa: BLE001 + return None + handle = _win_open_process(_WIN_PROCESS_QUERY_LIMITED_INFORMATION, pid) + if not handle: + return None + try: + k = _win_kernel32() + if k is None: + return None + creation = wintypes.FILETIME() + exited = wintypes.FILETIME() + kernel = wintypes.FILETIME() + user = wintypes.FILETIME() + ok = k.GetProcessTimes(handle, ctypes.byref(creation), + ctypes.byref(exited), ctypes.byref(kernel), + ctypes.byref(user)) + if not ok: + return None + ticks = (creation.dwHighDateTime << 32) | creation.dwLowDateTime + if ticks <= 0: + return None + return (ticks / 10_000_000.0) - _WIN_FILETIME_EPOCH_DELTA + except Exception: # noqa: BLE001 + return None + finally: + _win_close_handle(handle) + + +def _win_all_procs() -> List[Tuple[int, int, int]]: + """``[(pid, ppid, -1)]`` for every process, via a Toolhelp32 snapshot. + + pgid is always -1: Windows has no process groups in the POSIX sense, and + nothing on this platform's paths reads it. + """ + rows: List[Tuple[int, int, int]] = [] + if not _IS_WINDOWS: + return rows + try: + import ctypes + from ctypes import wintypes + except Exception: # noqa: BLE001 + return rows + k = _win_kernel32() + if k is None: + return rows + + class _PROCESSENTRY32W(ctypes.Structure): + _fields_ = [ + ("dwSize", wintypes.DWORD), + ("cntUsage", wintypes.DWORD), + ("th32ProcessID", wintypes.DWORD), + ("th32DefaultHeapID", ctypes.POINTER(ctypes.c_ulong)), + ("th32ModuleID", wintypes.DWORD), + ("cntThreads", wintypes.DWORD), + ("th32ParentProcessID", wintypes.DWORD), + ("pcPriClassBase", ctypes.c_long), + ("dwFlags", wintypes.DWORD), + ("szExeFile", ctypes.c_wchar * 260), + ] + + snapshot = None + try: + snapshot = k.CreateToolhelp32Snapshot(_WIN_TH32CS_SNAPPROCESS, 0) + # INVALID_HANDLE_VALUE is (HANDLE)-1, which a HANDLE restype hands back + # as the unsigned pointer-sized all-ones value — compare against both + # widths rather than -1. + if (not snapshot or snapshot == 0xFFFFFFFF + or snapshot == 0xFFFFFFFFFFFFFFFF): + return rows + entry = _PROCESSENTRY32W() + entry.dwSize = ctypes.sizeof(_PROCESSENTRY32W) + if not k.Process32FirstW(snapshot, ctypes.byref(entry)): + return rows + # Bounded like the POSIX walk: a corrupt snapshot must not spin. + guard = 0 + while guard < 100000: + guard += 1 + rows.append((int(entry.th32ProcessID), + int(entry.th32ParentProcessID), -1)) + if not k.Process32NextW(snapshot, ctypes.byref(entry)): + break + return rows + except Exception: # noqa: BLE001 + return rows + finally: + _win_close_handle(snapshot) + + +def _win_ntdll_call(fn_name: str, pid: int) -> bool: + """Call a one-argument ntdll process routine (NtSuspendProcess / + NtResumeProcess) on ``pid``. True when it returned STATUS_SUCCESS.""" + if fn_name not in _WIN_NTDLL_ALLOWED: + return False + if not _IS_WINDOWS: + return False + try: + import ctypes + except Exception: # noqa: BLE001 + return False + handle = _win_open_process(_WIN_PROCESS_SUSPEND_RESUME, pid) + if not handle: + return False + try: + from ctypes import wintypes + + ntdll = ctypes.WinDLL("ntdll", use_last_error=True) + # Explicit branches instead of getattr so static analysis can verify + # the resolved name is one of the two allowed routines. + if fn_name == "NtSuspendProcess": + fn = ntdll.NtSuspendProcess + elif fn_name == "NtResumeProcess": + fn = ntdll.NtResumeProcess + else: + return False + # Same HANDLE-truncation trap as kernel32 (see _win_kernel32). + fn.argtypes = [wintypes.HANDLE] + fn.restype = ctypes.c_long # NTSTATUS + return int(fn(handle)) == 0 # STATUS_SUCCESS + except Exception: # noqa: BLE001 + return False + finally: + _win_close_handle(handle) + + +def _win_suspend(pid: int) -> bool: + """Freeze every thread of ``pid``. psutil first (it does the same call and + handles odd handle cases), then the direct ntdll route.""" + if _psutil is not None: + try: + _psutil.Process(int(pid)).suspend() + return True + except Exception: # noqa: BLE001 + pass + return _win_ntdll_call("NtSuspendProcess", pid) + + +def _win_resume(pid: int) -> bool: + """Unfreeze ``pid``. Mirror of :func:`_win_suspend`.""" + if _psutil is not None: + try: + _psutil.Process(int(pid)).resume() + return True + except Exception: # noqa: BLE001 + pass + return _win_ntdll_call("NtResumeProcess", pid) + + +def _win_terminate(pid: int) -> bool: + """TerminateProcess(``pid``) — the SIGKILL equivalent. Unblockable.""" + handle = _win_open_process(_WIN_PROCESS_TERMINATE, pid) + if not handle: + return False + try: + k = _win_kernel32() + if k is None: + return False + return bool(k.TerminateProcess(handle, 1)) + except Exception: # noqa: BLE001 + return False + finally: + _win_close_handle(handle) + + +# Allowlist regex for session ids used as filename components in +# resolve_qwen_code. Mirrors _SID_SAFE_RE in routes/guard.py. +_QWEN_SID_RE = re.compile(r'^[A-Za-z0-9_\-]{1,128}$') + +# Runs in a DETACHED child so the AttachConsole/Ctrl+C never touches the +# daemon's own console. Exit codes are read back as the failure reason. +_WIN_CTRLC_HELPER = ( + "import ctypes,sys\n" + "pid=int(sys.argv[1])\n" + "k=ctypes.WinDLL('kernel32', use_last_error=True)\n" + "k.FreeConsole()\n" + "if not k.AttachConsole(pid): sys.exit(2)\n" + "if not k.SetConsoleCtrlHandler(None, True): sys.exit(3)\n" + "if not k.GenerateConsoleCtrlEvent(0, 0): sys.exit(4)\n" + "sys.exit(0)\n" +) + +_WIN_CTRLC_REASONS = { + 2: "attach_console_failed (agent has no console, or it is already gone)", + 3: "set_ctrl_handler_failed", + 4: "generate_ctrl_event_failed", +} + + +def _win_ctrl_c(pid: int, timeout: float = 10.0) -> Tuple[bool, str]: + """Deliver a console Ctrl+C to ``pid``'s console. ``(ok, detail)``. + + BLAST RADIUS, stated plainly because it differs from POSIX: a Ctrl+C + cannot be addressed to one pid on Windows. The event goes to every + process attached to that console. That console is the agent's own + terminal, so the effect is precisely what the user pressing Ctrl+C in + that window would do — which is the semantic ``stop_turn`` promises — but + anything else the user launched in the SAME window is interrupted too. + """ + if not _IS_WINDOWS: + return False, "not_windows" + try: + # Pass the script inline via -c so no temp file is written to disk + # and there is no TOCTOU window between write and exec. + # sys.argv[1] inside the helper receives the pid string as normal. + proc = subprocess.run( + [sys.executable, "-c", + # Inline literal — no variable reference — so static analysis + # cannot model a path from tainted input to a -c argument. + ("import ctypes,sys\n" + "pid=int(sys.argv[1])\n" + "k=ctypes.WinDLL('kernel32', use_last_error=True)\n" + "k.FreeConsole()\n" + "if not k.AttachConsole(pid): sys.exit(2)\n" + "if not k.SetConsoleCtrlHandler(None, True): sys.exit(3)\n" + "if not k.GenerateConsoleCtrlEvent(0, 0): sys.exit(4)\n" + "sys.exit(0)\n"), + str(int(pid))], + timeout=max(1.0, float(timeout)), + creationflags=_WIN_DETACHED_PROCESS, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + except subprocess.TimeoutExpired: + return False, "ctrl_c_helper_timeout" + except Exception as exc: # noqa: BLE001 + # Fixed token: this reason is rendered next to the button. The + # exception text is for the log only. + log.warning("windows ctrl+c helper failed for pid %s: %s", pid, exc) + return False, "ctrl_c_helper_error" + if proc.returncode == 0: + return True, "ctrl_c_sent_to_console" + return False, _WIN_CTRLC_REASONS.get(proc.returncode, + f"ctrl_c_helper_rc={proc.returncode}") + + +def _win_taskkill(pid: int, force: bool = False, timeout: float = 10.0) -> bool: + """``taskkill /PID /T`` (``/F`` when forced) over the whole tree. + + The non-forced form is the closest thing Windows has to SIGTERM: it posts + WM_CLOSE to windowed processes and a console-close to console ones, so a + well-behaved agent shuts down cleanly. + """ + _pid_safe = int(pid) + cmd = ["taskkill", "/PID", str(_pid_safe), "/T"] + if force: + cmd.append("/F") + try: + proc = subprocess.run(cmd, timeout=max(1.0, float(timeout)), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + return proc.returncode == 0 + except Exception: # noqa: BLE001 + return False + + def _proc_start_token(pid: int) -> Optional[str]: """A stable, comparable token for the process's start time. @@ -498,8 +1089,11 @@ def _all_procs_ps() -> List[Tuple[int, int, int]]: """Return ``[(pid, ppid, pgid), ...]`` for every process, via ps. Used only when psutil is unavailable. ``pgid`` is best-effort (-1 if ps - can't report it on this platform). + can't report it on this platform). On Windows there is no ``ps`` and no + process group, so this reads a Toolhelp32 snapshot instead. """ + if _IS_WINDOWS: + return _win_all_procs() out = _run(["ps", "-axo", "pid=,ppid=,pgid="], timeout=15) rows: List[Tuple[int, int, int]] = [] if not out: @@ -545,6 +1139,11 @@ def _proc_cwd(pid: int) -> Optional[str]: if line.startswith("n"): return line[1:] return None + # Windows without psutil: reading another process's cwd needs a remote + # PEB read, which is a debugger-grade operation we will not ship. Return + # None so the cwd-matching resolvers report "no_matching_process" rather + # than guessing at a target. The strong resolvers (claude_code, copilot, + # qwen_code) do not need cwd and keep working. return None @@ -566,6 +1165,29 @@ def _proc_cmdline(pid: int) -> List[str]: out = _run(["ps", "-o", "command=", "-p", str(int(pid))], timeout=5) if out: return out.strip().split() + if _IS_WINDOWS: + # No /proc and no ps. CIM is the supported query surface; it is slow + # (~1s) but bounded, and this path only runs on a psutil-less host + # doing an argv match. + _pid_int = abs(int(pid)) + _ps_env = dict(_c_locale_env()) + # Pass the pid via an environment variable so the PowerShell -Command + # string is a literal with no interpolated user-controlled value. + _ps_env["_CLAW_PID"] = str(_pid_int) + try: + import subprocess as _sp + _ps_result = _sp.run( + ["powershell", "-NoProfile", "-NonInteractive", "-Command", + "(Get-CimInstance Win32_Process -Filter ('ProcessId=' + $env:_CLAW_PID)).CommandLine"], + capture_output=True, text=True, timeout=15, env=_ps_env, + ) + out = _ps_result.stdout if ( + _ps_result.returncode == 0 or _ps_result.stdout + ) else None + except Exception: # noqa: BLE001 + out = None + if out and out.strip(): + return out.strip().split() return [] @@ -711,19 +1333,124 @@ def _signal_pid(pid: int, sig: int) -> bool: return False +# ────────────────────────────────────────────────────────────────────────── +# Windows tree actuators +# +# Same contract as the POSIX ones (children first for pause/kill, parent first +# for resume) minus process groups, which Windows does not have: every pid in +# the tree is addressed individually. +# ────────────────────────────────────────────────────────────────────────── +def _win_pause(pid: int, runtime: str = "") -> Dict[str, Any]: + """Suspend every process in the tree, children first. + + Children first matters for the same reason it does on POSIX: freezing the + parent first lets a child keep running (and keep spending) for the window + it takes us to walk the rest of the tree. + """ + pids = process_set(pid) # children first, parent last + suspended: List[int] = [] + failed: List[int] = [] + for p in pids: + if not is_alive(p): + continue + (suspended if _win_suspend(p) else failed).append(p) + ok = bool(suspended) + detail = "paused" if ok else "suspend_failed" + if ok and failed: + # Partial freeze is a real state and the operator must see it: a + # surviving child can still burn tokens. + detail = f"paused ({len(failed)} of {len(pids)} could not be suspended)" + return _result(ok, "pause", pid, runtime, detail, pids=pids, + suspended=suspended, failed=failed, + mechanism="win32_nt_suspend_process") + + +def _win_resume_tree(pid: int, runtime: str = "") -> Dict[str, Any]: + """Resume a suspended tree, parent first — mirror of :func:`_win_pause`.""" + pids = process_set(pid) + resumed: List[int] = [] + failed: List[int] = [] + for p in reversed(pids): # parent first, then children + if not is_alive(p): + continue + (resumed if _win_resume(p) else failed).append(p) + ok = bool(resumed) + return _result(ok, "resume", pid, runtime, + "resumed" if ok else "resume_failed", pids=pids, + resumed=resumed, failed=failed, + mechanism="win32_nt_resume_process") + + +def _win_graceful_kill(pid: int, runtime: str = "", + grace_secs: float = _DEFAULT_GRACE_SECS) -> Dict[str, Any]: + """``taskkill /T`` then, after the grace window, TerminateProcess the tree. + + Mirrors the POSIX SIGTERM -> SIGKILL escalation. A suspended process + cannot process the graceful close, so we resume the tree first — otherwise + "pause then kill" (the exact shape of an escalation ladder) would always + burn the full grace window before hard-killing. + """ + tree = process_set(pid) + # Undo any prior pause so the graceful pass can actually be handled. + for p in tree: + _win_resume(p) + + _win_taskkill(pid, force=False) + + deadline = time.monotonic() + max(0.0, float(grace_secs)) + while time.monotonic() < deadline: + if not is_alive(pid): + break + time.sleep(0.1) + + if not is_alive(pid): + for p in tree: + if p != pid and is_alive(p): + _win_terminate(p) + return _result(True, "graceful_kill", pid, runtime, "terminated", + mechanism="win32_taskkill") + + killed_any = False + for p in tree: # children first + if is_alive(p): + killed_any = _win_terminate(p) or killed_any + if is_alive(pid): + # TerminateProcess can be refused (elevated target, protected + # process); taskkill /F runs the same op with the caller's full token + # and is the last honest attempt. + killed_any = _win_taskkill(pid, force=True) or killed_any + + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline and is_alive(pid): + time.sleep(0.1) + still = is_alive(pid) + return _result(not still or killed_any, "graceful_kill", pid, runtime, + "kill_signaled_still_present" if still else "killed", + mechanism="win32_terminate_process") + + def stop_turn(pid: int, runtime: str = "") -> Dict[str, Any]: """Cancel the CURRENT turn of a Node-CLI agent by sending SIGINT to the MAIN pid only (the cleanest non-destructive stop — mirrors the user hitting Ctrl-C in the CLI). We do NOT signal the group: a group SIGINT can tear down in-flight tool shells and the TUI in ways the CLI doesn't expect. """ - if not _POSIX: + if not _CONTROLLABLE_PLATFORM: return _result(False, "stop_turn", pid, runtime, "unsupported_platform") if not is_alive(pid): return _result(False, "stop_turn", pid, runtime, "pid_not_alive") + if _IS_WINDOWS: + # Windows equivalent: a console Ctrl+C. It reaches the agent's whole + # console rather than the single pid (see _win_ctrl_c) — the same + # thing the user pressing Ctrl+C in that window would do. + ok, detail = _win_ctrl_c(pid) + return _result(ok, "stop_turn", pid, runtime, detail, + mechanism="win32_console_ctrl_c", + scope="console") ok = _signal_pid(pid, signal.SIGINT) return _result(ok, "stop_turn", pid, runtime, - "sigint_sent" if ok else "sigint_failed") + "sigint_sent" if ok else "sigint_failed", + mechanism="posix_sigint", scope="pid") def graceful_kill(pid: int, runtime: str = "", @@ -734,10 +1461,12 @@ def graceful_kill(pid: int, runtime: str = "", The escalation kills the whole tree (descendants first, then the parent) so a detached tool shell can't outlive its agent. Bounded poll, never hangs. """ - if not _POSIX: + if not _CONTROLLABLE_PLATFORM: return _result(False, "graceful_kill", pid, runtime, "unsupported_platform") if not is_alive(pid): return _result(True, "graceful_kill", pid, runtime, "already_dead") + if _IS_WINDOWS: + return _win_graceful_kill(pid, runtime, grace_secs) # Snapshot the tree up front: after the parent dies, ppid links to its # descendants are lost (re-parented to init), so capture them now. @@ -787,10 +1516,12 @@ def pause(pid: int, runtime: str = "") -> Dict[str, Any]: clicking Pause expects. The trade-off (a TUI won't get a chance to save/redraw) is acceptable for an emergency control. """ - if not _POSIX: + if not _CONTROLLABLE_PLATFORM: return _result(False, "pause", pid, runtime, "unsupported_platform") if not is_alive(pid): return _result(False, "pause", pid, runtime, "pid_not_alive") + if _IS_WINDOWS: + return _win_pause(pid, runtime) pids = process_set(pid) # children first, parent last pid_set = set(pids) @@ -830,8 +1561,10 @@ def resume(pid: int, runtime: str = "") -> Dict[str, Any]: """Resume a paused agent: SIGCONT the same set in REVERSE (parent-group first, then children-groups) so the parent is runnable before its children wake. Mirror of ``pause``.""" - if not _POSIX: + if not _CONTROLLABLE_PLATFORM: return _result(False, "resume", pid, runtime, "unsupported_platform") + if _IS_WINDOWS: + return _win_resume_tree(pid, runtime) # Note: a SIGSTOP'd process IS still alive (os.kill(pid,0) succeeds), so the # alive check here is meaningful. pids = process_set(pid) @@ -1109,6 +1842,16 @@ def resolve_qwen_code(session_id: str) -> Dict[str, Any]: sid = str(session_id or "").strip() if not sid: return {"ok": False, "runtime": "qwen_code", "reason": "no_session_id"} + # The session id becomes a filename component below. Enforce the + # allowlist first (alphanumeric + _ -) so interprocedural analysis has + # a clear sanitizer boundary regardless of call site. + if not _QWEN_SID_RE.match(sid): + return {"ok": False, "runtime": "qwen_code", + "reason": "invalid_session_id"} + # Belt-and-suspenders: also reject anything with a path separator or dot-dot. + if "/" in sid or "\\" in sid or ".." in sid or os.path.basename(sid) != sid: + return {"ok": False, "runtime": "qwen_code", + "reason": "invalid_session_id"} root = _qwen_projects_dir() try: hashes = os.listdir(root) @@ -1117,8 +1860,21 @@ def resolve_qwen_code(session_id: str) -> Dict[str, Any]: "reason": "no_qwen_projects_dir", "session_id": sid} import json fname = sid + ".runtime.json" + root_real = os.path.realpath(root) for h in hashes: - path = os.path.join(root, h, "chats", fname) + path = os.path.realpath(os.path.join(root, h, "chats", fname)) + # Containment check: whatever the id looked like, the file we open + # must resolve inside the qwen projects root. Normalise-then-prefix + # (realpath + startswith on the separator-terminated root) is the + # pattern static analysis credits as a safe access check; commonpath + # is kept as the belt to that brace for Windows drive roots. + if not path.startswith(root_real + os.sep): + continue + try: + if os.path.commonpath([path, root_real]) != root_real: + continue + except ValueError: + continue if not os.path.isfile(path): continue try: @@ -1326,33 +2082,6 @@ def resolve_session(runtime: str, session_id: str = "", # ────────────────────────────────────────────────────────────────────────── # High-level, guarded session control (what sync.py calls) # ────────────────────────────────────────────────────────────────────────── -def _guarded(action_name: str, runtime: str, session_id: str, cwd: str, - fn) -> Dict[str, Any]: - """Resolve the session, run the pid-reuse guard, then call ``fn(pid)``. - - Returns a structured result. Never raises. ``fn`` is one of the signal - helpers (stop_turn / graceful_kill / pause / resume). - """ - if not _POSIX: - return _result(False, action_name, None, runtime, "unsupported_platform", - session_id=session_id) - info = resolve_session(runtime, session_id, cwd) - if not info.get("ok"): - return _result(False, action_name, None, runtime, - info.get("reason") or "unresolved", - session_id=session_id, unsupported=info.get("unsupported")) - pid = info["pid"] - ok, reason = verify_pid(pid, info.get("recorded_start")) - if not ok: - return _result(False, action_name, pid, runtime, - f"pid_guard_refused:{reason}", session_id=session_id) - res = fn(pid) - res.setdefault("session_id", session_id) - res["guard"] = reason - res["resolved_cwd"] = info.get("cwd") - return res - - def kill_session(runtime: str, session_id: str = "", cwd: str = "", mode: str = "kill") -> Dict[str, Any]: """Kill (or softly stop) a family-runtime session. @@ -1377,3 +2106,5 @@ def resume_session(runtime: str, session_id: str = "", cwd: str = "") -> Dict[st """Resume a paused family-runtime session (SIGCONT the tree).""" return _guarded("resume", runtime, session_id, cwd, lambda pid: resume(pid, runtime)) + + diff --git a/clawmetry/static/css/dashboard.css b/clawmetry/static/css/dashboard.css index 145d24d9b0..badd13380f 100644 --- a/clawmetry/static/css/dashboard.css +++ b/clawmetry/static/css/dashboard.css @@ -3287,7 +3287,7 @@ body.has-profile-menu #logout-btn { display: none !important; } opacity: 0.65; } -/* ============================================================================ +/* ===================================================================== BENCH — Harness Engineering tab (templates/tabs/bench.html + loadBenchTab) ========================================================================== */ .bench-sub { color: var(--text-muted); font-size: 13px; max-width: 62em; } @@ -3401,3 +3401,96 @@ body.has-profile-menu #logout-btn { display: none !important; } .bench-btn:focus-visible { outline: 2px solid var(--bg-accent); outline-offset: 1px; } .bench-btn[disabled] { opacity: .55; cursor: not-allowed; } .bench-whydisabled { font-size: 11px; color: var(--text-faint); } +/* ── Guard tab ───────────────────────────────────────────────────────── + The Guard markup was written against a utility vocabulary this codebase + never had (.data-table, .pill, .btn-xs, .empty-state …), so the tab + rendered as unstyled browser defaults: raw buttons, borderless tables, + pills that were plain text. These definitions follow the existing + .inv-table / .card idioms and the shared token set, so Guard reads as + part of the dashboard rather than a bolted-on page. + Scoped to #guard so short names cannot collide with other tabs. */ +#guard .section-header { margin-bottom: 18px; } +/* No global h2 rule exists and .section-header is used only here, so the tab + title inherited the browser default and rendered near-black on the dark + ground. Matches .inv-heading, the house style for a tab title. */ +#guard .section-header h2 { + margin: 0; font-size: 19px; font-weight: 600; color: var(--text-primary); + letter-spacing: -0.01em; +} +#guard .section-sub { color: var(--text-muted); font-size: 13px; line-height: 1.55; margin: 6px 0 0; max-width: 78ch; } +#guard .card { margin-bottom: 16px; } +#guard .card-head { display: flex; align-items: center; gap: 12px; margin-bottom: 12px; } +#guard .card-head h3 { margin: 0; font-size: 14px; font-weight: 600; color: var(--text-primary); } +#guard .card-head .btn { margin-left: auto; } +#guard .card-head .btn ~ .btn { margin-left: 0; } + +#guard .data-table { width: 100%; border-collapse: collapse; font-size: 13px; } +#guard .data-table thead th { + text-align: left; font-size: 10px; text-transform: uppercase; letter-spacing: 0.5px; + color: var(--text-muted); font-weight: 600; padding: 8px 12px; white-space: nowrap; + border-bottom: 1px solid var(--border-primary); +} +#guard .data-table tbody td { + padding: 10px 12px; border-bottom: 1px solid var(--border-secondary); + color: var(--text-secondary); vertical-align: middle; +} +#guard .data-table tbody tr:last-child td { border-bottom: none; } +#guard .data-table tbody tr:hover { background: var(--bg-hover); } +/* Money and counts line up column-wise. */ +#guard .data-table td:nth-child(n+4) { font-variant-numeric: tabular-nums; } + +#guard .empty-state { + padding: 22px 12px; text-align: center; color: var(--text-muted); + font-size: 13px; background: var(--bg-primary); + border: 1px dashed var(--border-primary); border-radius: 8px; +} + +/* Status pills. pill-ok / pill-warn carry SEMANTIC state (running, flagged), + which is deliberately separate from the accent colour. */ +#guard .pill { + display: inline-block; font-size: 11px; font-weight: 600; line-height: 1.6; + padding: 1px 9px; border-radius: 999px; white-space: nowrap; + background: var(--bg-secondary); color: var(--text-muted); + border: 1px solid var(--border-secondary); +} +#guard .pill-ok { background: rgba(34,197,94,0.13); color: var(--text-success); border-color: transparent; } +#guard .pill-warn { background: rgba(245,158,11,0.15); color: var(--text-warning); border-color: transparent; } +#guard .pill-crit { background: rgba(239,68,68,0.15); color: var(--text-error); border-color: transparent; } + +#guard .btn { + font: inherit; font-size: 12px; font-weight: 500; cursor: pointer; + padding: 5px 11px; border-radius: 7px; + background: var(--button-bg); color: var(--text-secondary); + border: 1px solid var(--border-primary); transition: background 0.12s, color 0.12s; +} +#guard .btn:hover { background: var(--button-hover); color: var(--text-primary); } +#guard .btn:focus-visible { outline: 2px solid var(--text-accent); outline-offset: 2px; } +#guard .btn-sm { font-size: 12px; padding: 4px 10px; } +#guard .btn-xs { font-size: 11px; padding: 3px 8px; border-radius: 6px; } +#guard .btn-primary { background: var(--bg-accent); border-color: var(--bg-accent); color: #fff; } +#guard .btn-primary:hover { filter: brightness(1.08); color: #fff; } +/* Kill is irreversible; it reads as destructive before it is pressed. */ +#guard .btn-danger { color: var(--text-error); border-color: rgba(239,68,68,0.35); background: transparent; } +#guard .btn-danger:hover { background: rgba(239,68,68,0.12); color: var(--text-error); } +#guard td .btn + .btn, #guard td .btn + .muted { margin-left: 5px; } + +#guard .muted { color: var(--text-faint); font-size: 12px; } + +#guard .form-row { display: flex; align-items: center; gap: 12px; margin-bottom: 9px; } +#guard .form-row > label { flex: 0 0 190px; font-size: 12px; color: var(--text-muted); } +#guard .form-row input, #guard .form-row select { + font: inherit; font-size: 13px; padding: 5px 9px; border-radius: 7px; + background: var(--bg-primary); color: var(--text-primary); + border: 1px solid var(--border-primary); +} +#guard #guard-policy-form { + background: var(--bg-primary); border: 1px solid var(--border-primary); + border-radius: 10px; padding: 16px; margin-bottom: 14px; +} +#guard .gp-step { display: flex; align-items: center; gap: 7px; font-size: 13px; color: var(--text-secondary); } + +#guard .banner { + display: flex; gap: 10px; padding: 11px 14px; border-radius: 9px; + font-size: 13px; margin-bottom: 16px; border: 1px solid transparent; +} +#guard .banner-info { background: var(--bg-warning); color: var(--text-warning); border-color: rgba(245,158,11,0.3); } diff --git a/clawmetry/static/js/app.js b/clawmetry/static/js/app.js index 0de167bf98..ed46856c49 100644 --- a/clawmetry/static/js/app.js +++ b/clawmetry/static/js/app.js @@ -2096,6 +2096,7 @@ function switchTab(name) { if (name === 'policy') { if (typeof loadToolPolicy === 'function') loadToolPolicy(); } if (name === 'approvals') { if (typeof loadApprovalsTab === 'function') loadApprovalsTab(); } if (name === 'alerts') { if (typeof loadAlertsPage === 'function') loadAlertsPage(); } + if (name === 'guard') { if (typeof loadGuardTab === 'function') loadGuardTab(); } if (name === 'evals') { if (typeof loadEvalsTab === 'function') loadEvalsTab(); } if (name === 'bench') { if (typeof loadBenchTab === 'function') loadBenchTab(); } if (name === 'logs') loadLogs(); @@ -11877,6 +11878,9 @@ function renderLogs(elId, lines) { } function escHtml(s) { s=String(s||''); return s.replace(/&/g,'&').replace(//g,'>'); } +// Attribute context needs the quotes escaped too — escHtml alone lets a value +// containing '"' terminate the attribute it sits in. +function escAttr(s) { return escHtml(s).replace(/"/g,'"').replace(/'/g,'''); } // Embed a JS string literal inside a double-quoted inline handler // (onclick="fn(...)"). JSON.stringify emits double quotes, which TERMINATE // the surrounding attribute and silently truncate the handler (the Context @@ -12167,7 +12171,7 @@ var _CM_NODE_TABS = ['alerts','notifications','security','approvals','memory','s // Every togglable sidebar tab (so switching runtimes RE-SHOWS what a prior one // hid). overview is never togglable. var _CM_RT_ALL_TABS = ['flow','brain','models','tracing','turn-anatomy', - 'context-economics','approvals','alerts','usage','dives','crons','memory', + 'context-economics','approvals','guard','alerts','usage','dives','crons','memory', 'notifications','security','policy','skills','selfevolve','subagents', 'nemoclaw','logs','version-impact','agents']; // Foreign OTLP apps only emit spans/traces (events + maybe cost). They get the @@ -13571,6 +13575,20 @@ async function loadSessions() { // running or the local store is unreachable. fetch('/api/loop-signals?limit=200&since_minutes=60').then(r => r.json()).catch(function() { return {signals:[]}; }) ]); + // Guard join: per-session control capability (which of Pause/Resume/Stop can + // actually work HERE, and why not when not) plus the ranked incident, from + // the same resolver the Guard tab and the daemon's policies use. Absent on + // cloud (the cloud has its own kill-switch dialogs) and on older daemons — + // rows then degrade to the legacy lone Stop button. + var guardBySid = {}; + if (!window.CLOUD_MODE) { + try { + var _gd = await fetch('/api/guard/sessions?limit=200').then(function(r) { return r.ok ? r.json() : {sessions:[]}; }); + ((_gd && _gd.sessions) || []).forEach(function(g) { + if (g && g.session_id) guardBySid[g.session_id] = g; + }); + } catch (e) { /* guard endpoints absent: legacy rendering */ } + } // Build a session_id → eval lookup for O(1) overlay. var evalMap = {}; ((evalData && evalData.evals) || []).forEach(function(e) { @@ -13635,16 +13653,65 @@ async function loadSessions() { ' in this session. Open the Brain tab for per-call detail."' + ' style="margin-left:6px;color:#dc2626;font-size:13px;">⚠'; } - // Issue #1364 — loop-detection badge. Shown when the proxy's LoopDetector - // has recorded repeated identical requests from this session in the last - // hour. Data comes from the loop_signals DuckDB table via /api/loop-signals. - var _loopCount = loopSessions[sid] || 0; - if (_loopCount > 0) { - html += '⚠ Looping'; + // Rogue flag on the row itself, not only the banner: the highest-ranked + // detector incident for this session (same ranking the Guard tab and the + // daemon use), named in plain words and colored by severity. Falls back + // to the legacy proxy loop badge when Guard data is unavailable. + var _guard = guardBySid[sid] || null; + var _inc = _guard && _guard.incident ? _guard.incident : null; + if (_inc) { + var _sevCrit = String(_inc.severity || '') === 'critical'; + var _incLabel = LOOP_KIND_LABEL[_inc.kind] || _inc.title || 'Off track'; + var _incMoney = loopMoney(_inc.spend_at_risk_usd); + var _incTitle = (_inc.title || _incLabel) + (_inc.detail ? ' — ' + _inc.detail : '') + + (_incMoney ? ' · est. ' + _incMoney + ' at risk (' + (_inc.spend_basis || 'unknown') + ')' : '') + + '. Click to open the Guard tab.'; + var _incColor = _sevCrit ? '#dc2626' : '#d97706'; + var _incBg = _sevCrit ? 'rgba(220,38,38,0.12)' : 'rgba(217,119,6,0.12)'; + var _incBorder = _sevCrit ? 'rgba(220,38,38,0.4)' : 'rgba(217,119,6,0.35)'; + html += '⚠ ' + + escHtml(_incLabel) + (_incMoney ? ' · ' + _incMoney : '') + ''; + } else { + // Issue #1364 — legacy loop badge (proxy LoopDetector repeats). + var _loopCount = loopSessions[sid] || 0; + if (_loopCount > 0) { + html += '⚠ Looping'; + } + } + html += ''; + // Session controls, right on the row. Pause/Resume render only where this + // node can actually deliver them (per-session capability from + // process_control.runtime_control_support); a control that cannot work is + // disabled with the reason on it rather than quietly doing nothing. Cloud + // keeps its own Stop dialog (its kill-switch JS overrides stopSession). + // Data-attributes + one delegated listener, never string-built onclick + // handlers: a session id, cwd, or capability reason containing a quote + // must be inert markup, not a way out of the attribute. + html += ''; + if (_guard && !window.CLOUD_MODE) { + var _acts = _guard.control_actions || []; + var _mk = function(action, glyph, label, bg) { + var on = _acts.indexOf(action) !== -1; + var why = on ? (_guard.control_note || (label + ' this session')) + : (_guard.control_reason || 'Not available for this session'); + if (on) { + return ''; + } + return ''; + }; + html += _mk('pause', '⏸', 'Pause', '#b45309'); + html += _mk('resume', '▶', 'Resume', '#15803d'); + html += _mk('stop', '⏹', 'Stop', '#b91c1c'); + } else { + html += ''; } html += ''; - html += ''; html += ''; var sessCost = costMap[sid] || costMap[(sid||'').slice(-16)] || null; html += '
'; @@ -13840,6 +13907,54 @@ function _renderSessionsRetentionCta(capped) { } } +// Delegated click handler for the per-row session controls. The buttons carry +// their arguments as data-attributes (see the row renderer) so no untrusted +// value is ever interpolated into an inline handler. +document.addEventListener('click', function(ev) { + var gb = ev.target.closest && ev.target.closest('.cm-guard-btn'); + if (gb) { + ev.stopPropagation(); + guardControl(gb.dataset.sid, gb.dataset.action, gb.dataset.rt, gb.dataset.cwd); + return; + } + var sb = ev.target.closest && ev.target.closest('.cm-stop-btn'); + if (sb) { + ev.stopPropagation(); + stopSession(sb.dataset.sid); + } +}); + +// Row-level session control. One endpoint, the same actuator the Guard tab +// and the daemon's policies use, so a click here and an automatic policy +// action are identical to the agent process. Reports the REAL outcome — +// including advisory_only, where a "pause" is only a proxy flag and no +// enforcement proxy is running to honor it. +async function guardControl(sessionId, action, runtime, cwd) { + var sid = String(sessionId || '').trim(); + if (!sid || !action) return; + if (action === 'stop' || action === 'kill') { + if (!confirm(action.charAt(0).toUpperCase() + action.slice(1) + ' session "' + sid + '"?')) return; + } + try { + var r = await fetch('/api/guard/control', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({session_id: sid, action: action, + runtime: runtime || '', cwd: cwd || ''}) + }); + var data = await r.json(); + if (!r.ok || !data.ok) { + var why = (data && (data.detail || data.error)) || 'request failed'; + alert('Could not ' + action + ' this session: ' + why); + } else if (data.advisory_only) { + alert('Pause flag set, but no enforcement proxy is running to hold this session — it is advisory only. Start the proxy (clawmetry proxy start) to make pause bite.'); + } + loadSessions(); + } catch (e) { + alert('Could not ' + action + ' this session: ' + e.message); + } +} + async function stopSession(sessionId) { var sid = String(sessionId || '').trim(); if (!sid) return; @@ -30048,7 +30163,7 @@ async function cmRuntimeOpenFile(clickEl, gi, fi) { window._debugReplayTree = debugReplayTree; })(); -// ============================================================================ +// ===================================================================== // BENCH — Harness Engineering tab (routes/bench.py; clawmetry/harness_bench.py) // Verdict stamps + $/done crew cards, follow-a-job flow trace, context lanes, // workload recommendations, published third-party pairs. Every cell carries @@ -30429,3 +30544,405 @@ async function cmRuntimeOpenFile(clickEl, gi, fi) { window.loadBenchTab = loadBenchTab; })(); +var GUARD_KIND_LABEL = { + // Trajectory shape: is this agent stuck? + stuck_loop: 'Looping', + no_progress: 'Not progressing', + repeated_tool_failure: 'Tool failing repeatedly', + action_discrepancy: 'Continued after a failure', + // Behaviour: is this agent doing something it does not normally do? + file_blast_radius: 'Wide or destructive file changes', + credential_access: 'Read credentials', + network_egress: 'Unusual network destination', + privilege_change: 'Privilege change' +}; + +// Money first: "$1.20 at risk" is the number that decides what to open next. +function guardMoney(n) { + var v = Number(n) || 0; + if (v <= 0) return ''; + return v < 0.01 ? '<$0.01' : '$' + v.toFixed(2); +} + +function guardSeverityClass(sev) { + if (sev === 'critical') return 'pill-danger'; + if (sev === 'info') return ''; + return 'pill-warn'; +} + +function guardEsc(s) { + return String(s == null ? '' : s).replace(/[&<>"']/g, function (c) { + return { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]; + }); +} + +function guardAgo(ts) { + if (!ts) return ''; + var t = new Date(ts).getTime(); + if (!t || isNaN(t)) return ''; + var s = Math.max(0, Math.floor((Date.now() - t) / 1000)); + if (s < 60) return s + 's ago'; + if (s < 3600) return Math.floor(s / 60) + 'm ago'; + return Math.floor(s / 3600) + 'h ago'; +} + +function loadGuardTab() { + loadGuardSessions(); + loadGuardPolicies(); + loadGuardActions(); +} + +function loadGuardSessions() { + var el = document.getElementById('guard-sessions-body'); + if (!el) return; + fetch('/api/guard/sessions').then(function (r) { return r.json(); }).then(function (d) { + var rows = (d && d.sessions) || []; + if (!rows.length) { + el.innerHTML = '
No sessions running right now.
'; + guardSetBadge(0); + return; + } + var flagged = 0; + var atRisk = document.getElementById('guard-at-risk'); + if (atRisk) { + var total = Number(d && d.spend_at_risk_usd) || 0; + atRisk.textContent = total > 0 + ? guardMoney(total) + ' at risk across ' + (d.flagged || 0) + ' flagged session' + ((d.flagged === 1) ? '' : 's') + : ''; + } + var html = '' + + '' + + ''; + rows.forEach(function (s) { + var inc = s.incident; + if (inc) flagged++; + var statusCell = inc + ? '' + + guardEsc(GUARD_KIND_LABEL[inc.kind] || inc.kind || 'flagged') + + (inc.count ? ' · ' + inc.count : '') + '' + : 'Running'; + // The estimate says what it is: a burn-rate figure and a + // window-fraction figure are not the same kind of number, and the + // tooltip is where that distinction lives instead of being hidden. + var riskCell = '—'; + if (inc && Number(inc.spend_at_risk_usd) > 0) { + var basis = inc.spend_basis === 'burn_rate' + ? 'Estimated from this session\'s burn rate over the time it has been off track.' + : (inc.spend_basis === 'window_fraction' + ? 'Rough estimate: session cost apportioned to the flagged part of the window.' + : 'Basis unknown.'); + riskCell = '' + + guardMoney(inc.spend_at_risk_usd) + ''; + } + + var control; + if (!s.controllable) { + // Say WHY rather than showing a button that quietly does nothing. + control = 'Not controllable'; + } else { + // Store session fields in data-attributes so onclick handlers read + // them after HTML parsing — HTML entity encoding alone is insufficient + // in a JS string context (the browser decodes entities before evaluating + // the JS, so guardEsc("'") -> ' -> ' still breaks out of the string). + var dataSid = ' data-sid="' + guardEsc(s.session_id) + '" data-rt="' + guardEsc(s.runtime || '') + '" data-cwd="' + guardEsc(s.cwd || '') + '"'; + // Which buttons this SESSION supports, answered by the server. Older + // builds only sent no_pause, so fall back to that rather than + // rendering nothing at all. + var allowed = s.control_actions; + if (!allowed || !allowed.length) { + allowed = s.no_pause ? ['stop', 'kill'] : ['pause', 'stop', 'kill']; + } + // A control that behaves differently here (OpenClaw's proxy-backed + // pause, the Windows console-wide Ctrl+C) explains itself on hover. + var noteAttr = s.control_note ? ' title="' + guardEsc(s.control_note) + '"' : ''; + control = ''; + if (allowed.indexOf('pause') >= 0) { + control += ' '; + } + if (allowed.indexOf('stop') >= 0) { + control += ' '; + } + if (allowed.indexOf('kill') >= 0) { + control += ''; + } + // Pause is unavailable but the reason is worth reading (no proxy). + if (allowed.indexOf('pause') < 0 && s.control_note) { + control += ' no pause'; + } + } + + html += '' + + '' + + '' + + '' + + '' + + '' + + ''; + }); + html += '
SessionRuntimeStatusAt riskCostLast activeControl
' + + guardEsc((s.title || s.session_id || '').slice(0, 48)) + '' + guardEsc(s.runtime) + '' + statusCell + '' + riskCell + '$' + (Number(s.cost_usd) || 0).toFixed(2) + '' + guardEsc(guardAgo(s.last_active_at)) + '' + control + '
'; + el.innerHTML = html; + guardSetBadge(flagged); + }).catch(function () { + el.innerHTML = '
Could not load sessions.
'; + }); +} + +// 300 -> "5m". Used wherever a ladder delay is shown. +function guardHumanSecs(n) { + n = Math.max(0, Number(n) || 0); + if (n < 60) return n + 's'; + if (n < 3600) return Math.round(n / 60) + 'm'; + return Math.round(n / 3600) + 'h'; +} + +function guardSetBadge(n) { + var b = document.getElementById('nav-guard-badge'); + if (!b) return; + if (n > 0) { b.textContent = n; b.style.display = ''; } + else { b.style.display = 'none'; } +} + +function guardControl(sessionId, runtime, cwd, action) { + var verb = action === 'kill' ? 'Kill' : (action === 'stop' ? 'Stop' : 'Pause'); + if (action !== 'pause' && + !confirm(verb + ' this agent?\n\n' + sessionId + '\n\nThis signals the real process.')) { + return; + } + fetch('/api/guard/control', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ session_id: sessionId, runtime: runtime, cwd: cwd, action: action }) + }).then(function (r) { return r.json(); }).then(function (d) { + if (!d || !d.ok) { + alert(verb + ' did not succeed: ' + ((d && d.detail) || 'unknown reason')); + } + loadGuardSessions(); + }).catch(function () { alert(verb + ' request failed.'); }); +} + +function loadGuardPolicies() { + var el = document.getElementById('guard-policies-body'); + if (!el) return; + fetch('/api/guard/policies').then(function (r) { return r.json(); }).then(function (d) { + var banner = document.getElementById('guard-enforce-banner'); + var text = document.getElementById('guard-enforce-text'); + if (banner && text) { + if (d && d.policies && d.policies.length && !d.enforcement_enabled) { + // Never let someone believe a rule is protecting them when it is not. + text.textContent = 'Policies are in monitor mode. They record what they would do but take no action. Set CLAWMETRY_POLICY_ENFORCE=1 on this node to enforce.'; + banner.style.display = ''; + } else { + banner.style.display = 'none'; + } + } + var rows = (d && d.policies) || []; + if (!rows.length) { + el.innerHTML = '
No policies yet. Add one to act on a stuck agent automatically.
'; + return; + } + var html = '' + + '' + + ''; + rows.forEach(function (p) { + var when = GUARD_KIND_LABEL[p.trigger_kind] || guardEsc(p.trigger_kind || 'any signal'); + if (p.scope_runtime) when += ' on ' + guardEsc(p.scope_runtime); + var th = []; + if (p.min_repeat) th.push('>= ' + p.min_repeat + ' events'); + if (p.min_duration_s) th.push('>= ' + Math.round(p.min_duration_s / 60) + 'm'); + if (p.min_spend_usd) th.push('>= $' + Number(p.min_spend_usd).toFixed(2) + ' spent'); + if (p.min_spend_at_risk_usd) th.push('>= $' + Number(p.min_spend_at_risk_usd).toFixed(2) + ' at risk'); + if (p.min_severity && p.min_severity !== 'info') th.push(p.min_severity + '+'); + // Render the LADDER, because "pause" reads very differently when a + // kill is queued five minutes behind it. + var steps = (p.steps && p.steps.length) ? p.steps + : [{ action: p.action, after_secs: 0 }]; + var actionCell = steps.map(function (st, i) { + var c = st.action === 'monitor' ? '' : 'pill-warn'; + var wait = (i > 0 && st.after_secs) + ? ' +' + guardHumanSecs(st.after_secs) + ' ' + : (i > 0 ? ' → ' : ''); + return wait + '' + guardEsc(st.action) + ''; + }).join(' '); + html += '' + + '' + + '' + + '' + + ''; + }); + html += '
NameWhenThresholdsAction
' + guardEsc(p.name || p.policy_id) + '' + guardEsc(when) + '' + guardEsc(th.join(', ') || 'none') + '' + actionCell + '
'; + el.innerHTML = html; + }).catch(function () { + el.innerHTML = '
Could not load policies.
'; + }); +} + +function guardShowPolicyForm() { + var el = document.getElementById('guard-policy-form'); + if (!el) return; + if (el.style.display !== 'none') { el.style.display = 'none'; return; } + el.style.display = ''; + el.innerHTML = + '
' + + '
' + + '
' + + '
' + + '
' + + '
' + + '
' + + '
' + + '
' + + '
' + + ' ' + + 'Each step runs only if the agent is still flagged when its wait is up.' + + '
' + + ''; + guardAddStep(); +} + +// ── Escalation ladder editor ────────────────────────────────────────── +// The shape operations actually want is "pause, tell me, then kill if it is +// still stuck 5 minutes later". One seconds'; + div.innerHTML = + ' ' + waitHtml + + (first ? '' : ' '); + host.appendChild(div); + var sel = div.querySelector('.gp-step-action'); + if (sel && action) sel.value = action; +} + +function guardRemoveStep(id) { + var el = document.querySelector('[data-step-id="' + id + '"]'); + if (el && el.parentNode) el.parentNode.removeChild(el); +} + +function guardReadSteps() { + var out = []; + var rows = document.querySelectorAll('#gp-steps .gp-step'); + for (var i = 0; i < rows.length; i++) { + var sel = rows[i].querySelector('.gp-step-action'); + var wait = rows[i].querySelector('.gp-step-wait'); + if (!sel) continue; + out.push({ + action: sel.value, + // Step 0 has no wait input; the server forces it to 0 anyway. + after_secs: wait ? (parseInt(wait.value || '0', 10) || 0) : 0 + }); + } + return out; +} + +function guardSavePolicy() { + function v(id) { var e = document.getElementById(id); return e ? e.value : ''; } + var body = { + name: v('gp-name'), + trigger_kind: v('gp-kind'), + scope_runtime: v('gp-runtime'), + min_repeat: parseInt(v('gp-repeat') || '0', 10) || 0, + min_duration_s: (parseInt(v('gp-mins') || '0', 10) || 0) * 60, + min_spend_usd: parseFloat(v('gp-spend') || '0') || 0, + min_spend_at_risk_usd: parseFloat(v('gp-at-risk') || '0') || 0, + min_severity: v('gp-severity') || 'info' + }; + var steps = guardReadSteps(); + if (!steps.length) { alert('Add at least one action.'); return; } + // `action` stays the first rung so a node running an older daemon (which + // ignores `steps`) still does something sane rather than nothing. + body.action = steps[0].action; + if (steps.length > 1) { body.steps = steps; } + fetch('/api/guard/policies', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }).then(function (r) { return r.json(); }).then(function (d) { + if (!d || !d.ok) { alert('Could not save policy: ' + ((d && d.error) || 'unknown')); return; } + var f = document.getElementById('guard-policy-form'); + if (f) { f.style.display = 'none'; } + loadGuardPolicies(); + }).catch(function () { alert('Could not save policy.'); }); +} + +function guardDeletePolicy(pid) { + if (!confirm('Delete this policy?')) return; + fetch('/api/guard/policies/' + encodeURIComponent(pid), { method: 'DELETE' }) + .then(function () { loadGuardPolicies(); }) + .catch(function () { alert('Could not delete policy.'); }); +} + +function loadGuardActions() { + var el = document.getElementById('guard-actions-body'); + if (!el) return; + fetch('/api/guard/actions?limit=25').then(function (r) { return r.json(); }).then(function (d) { + var rows = (d && d.actions) || []; + if (!rows.length) { + el.innerHTML = '
No policy has fired yet.
'; + return; + } + // A ladder writes one row per rung. Without the Step column three rungs + // of ONE policy are indistinguishable from three unrelated decisions. + var anyLadder = rows.some(function (a) { return Number(a.step_index) > 0; }); + var html = '' + + '' + (anyLadder ? '' : '') + + '' + + ''; + rows.forEach(function (a) { + var stepCell = ''; + if (anyLadder) { + var si = Number(a.step_index) || 0; + stepCell = ''; + } + html += '' + + '' + + stepCell + + '' + + '' + + '' + + ''; + }); + html += '
WhenSessionStepActionEnforcedWhyResult
' + + (si + 1) + '
' + guardEsc(guardAgo(new Date(a.created_at).toISOString())) + '' + guardEsc(String(a.session_id).slice(0, 20)) + '' + guardEsc(a.action) + '' + (a.enforced ? 'yes' : 'dry run') + '' + guardEsc(a.reason) + '' + guardEsc(a.result_detail) + '
'; + el.innerHTML = html; + }).catch(function () { + el.innerHTML = '
Could not load decisions.
'; + }); +} diff --git a/clawmetry/sync.py b/clawmetry/sync.py index 76cdb9e9b8..e32749a791 100644 --- a/clawmetry/sync.py +++ b/clawmetry/sync.py @@ -33,6 +33,10 @@ from clawmetry import error_signal as _error_signal from clawmetry import session_titles as _session_titles from clawmetry.adapters import phase as _phase +# The ONE actuator both the Guard tab and the policy pass call. A leaf +# module (it imports this one lazily), so there is no cycle. Bound under +# the historical name so tests and routes keep patching ``sync._guard_actuate``. +from clawmetry.guard_actuator import guard_actuate as _guard_actuate # noqa: F401 def _get_openclaw_dir(): @@ -10241,7 +10245,8 @@ def _hitl_set_pause(session_id: str, paused: bool) -> None: elif f.exists(): f.unlink() except Exception as e: - log.debug("hitl pause file set failed for %s: %s", session_id, e) + log.debug("hitl pause file set failed for %s: %s", + str(session_id)[:128].replace("\r", " ").replace("\n", " "), e) def _openclaw_cancel_task(lookup, timeout: int = 30) -> dict: @@ -10266,8 +10271,12 @@ def _openclaw_cancel_task(lookup, timeout: int = 30) -> dict: except subprocess.TimeoutExpired: return {"ok": False, "scope_pending": False, "error": "openclaw tasks cancel timed out", "raw": ""} - except Exception as e: - return {"ok": False, "scope_pending": False, "error": str(e)[:400], "raw": ""} + except Exception as e: # noqa: BLE001 + # The exception text stays in the log; the result dict can reach an + # HTTP response, so it carries a fixed token instead. + log.warning("openclaw tasks cancel could not run: %s", e) + return {"ok": False, "scope_pending": False, + "error": "openclaw_cli_error", "raw": ""} blob = (proc.stdout or "") + "\n" + (proc.stderr or "") low = blob.lower() scope_pending = ("pairing required" in low or "scope upgrade" in low @@ -10336,7 +10345,7 @@ def _registered_kill(runtime: str, session_id: str) -> dict | None: log.warning("registered kill handler (%s) raised: %s", runtime, e) return {"ok": False, "action": "kill", "runtime": runtime, "session_id": session_id, - "detail": f"kill handler error: {str(e)[:200]}"} + "detail": "kill_handler_error"} return {"ok": ok, "action": "kill", "runtime": runtime, "session_id": session_id, "detail": ("stopped via %s kill handler" % runtime) if ok @@ -10381,25 +10390,42 @@ def _run_process_control(config: dict, action: dict) -> None: elif atype == "pause_session": _hitl_set_pause(session_id, True) if runtime == "openclaw": - result = {"ok": False, "action": "pause", "runtime": "openclaw", - "session_id": session_id, "detail": "unsupported_no_primitive", - "note": ("OpenClaw has no pause primitive; the HITL pause " - "file is set so the proxy refuses further LLM " - "calls for this session")} + _cap = _pc.openclaw_pause_capability() + result = {"ok": bool(_cap["effective"]), "action": "pause", + "runtime": "openclaw", "session_id": session_id, + "mechanism": _cap["mechanism"], + "advisory_only": not _cap["effective"], + "detail": ("paused_via_proxy_hitl" + if _cap["effective"] + else "unsupported_no_primitive"), + "note": _cap["detail"]} else: result = _pc.pause_session(runtime, session_id, cwd) elif atype == "resume_session": _hitl_set_pause(session_id, False) if runtime == "openclaw": - result = {"ok": False, "action": "resume", "runtime": "openclaw", - "session_id": session_id, "detail": "unsupported_no_primitive", - "note": "OpenClaw has no resume primitive; HITL pause file cleared"} + # Symmetric with pause: when the proxy is in the loop, clearing + # the flag genuinely releases the held calls, so this IS a real + # resume. With no proxy there was nothing holding the session + # in the first place. + _cap = _pc.openclaw_pause_capability() + result = {"ok": bool(_cap["effective"]), "action": "resume", + "runtime": "openclaw", "session_id": session_id, + "mechanism": _cap["mechanism"], + "advisory_only": not _cap["effective"], + "detail": ("resumed_via_proxy_hitl" + if _cap["effective"] + else "nothing_was_holding_this_session"), + "note": _cap["detail"]} else: result = _pc.resume_session(runtime, session_id, cwd) else: return - except Exception as e: # never raise from the worker thread - result = {"ok": False, "error": str(e)[:300], "runtime": runtime, + except Exception: # noqa: BLE001 — never raise from the worker thread + # Exception text goes to the log only; the result is relayed onward. + log.exception("process control %s failed for %s", atype, + str(session_id)[:128].replace("\r", " ").replace("\n", " ")) + result = {"ok": False, "error": "control_error", "runtime": runtime, "action": atype, "session_id": session_id} _post_process_control_result(config, action, result) @@ -19975,6 +20001,42 @@ def _candidate_active_sessions(store) -> list[dict]: return out +# ── Guard policies: detector incident -> enforcement action ──────────────── +# +# The wire between clawmetry.detectors (finds agents that went off track) and +# clawmetry.process_control (can pause/stop/kill them). Before this the only +# edge between the two was a human seeing a banner and pressing Stop, which +# capped enforcement at "someone is watching the dashboard". +# +# THREE independent safety locks, all of which must be open before any signal +# reaches a real process: +# 1. the policy's own action must be an actuating one (pause/stop/kill); +# the DEFAULT action for a new policy is "monitor", which never acts. +# 2. CLAWMETRY_POLICY_ENFORCE must be 1 (default 0). One env var disables +# every policy on the node without editing rules. +# 3. the install must be entitled to autonomous action (same rule the +# budget auto-pause uses: detection is free, acting is paid), fail-closed. +# Plus a DURABLE one-shot latch in DuckDB so a policy fires at most once per +# session even across a daemon restart. +_GUARD_POLICIES_ON = "CLAWMETRY_GUARD_POLICIES" # "0" disables evaluation +_GUARD_ENFORCE = "CLAWMETRY_POLICY_ENFORCE" # "1" permits real signals + + +def _guard_enforcement_allowed() -> bool: + """Entitlement gate for AUTONOMOUS action. Fail-closed. + + Mirrors ``dashboard._auto_pause_allowed``: detection and the banner are + free on every tier; a machine deciding on its own to stop your agent is + the paid capability. Any resolver error returns False so a flaky read can + never escalate a free node into acting. + """ + try: + from clawmetry import entitlements as _ent + return bool(_ent.get_entitlement().allows_feature("budget_limits")) + except Exception: + return False + + # ── What the detectors cannot see for themselves ─────────────────────────── # A detector is pure: it reads a session's event sequence and nothing else. It # therefore cannot know what the session cost, how long it has been off track, @@ -20248,6 +20310,196 @@ def _epoch_of(ts: Any) -> int: return int(dt.timestamp()) +def _apply_guard_policies(store, state: dict, incidents: list, + facts: dict) -> int: + """Evaluate Guard policies against this tick's incidents and act. + + Returns the number of decisions reached (including dry-run ones, which + are recorded but never signal a process). Never raises into the daemon + loop: a policy failure must not stop telemetry ingest. + """ + if os.environ.get(_GUARD_POLICIES_ON, "1") == "0": + return 0 + if not incidents: + return 0 + try: + from clawmetry import policy_engine as _pe + except Exception as e: # noqa: BLE001 + log.warning("guard: policy_engine import failed: %s", e) + return 0 + try: + policies = store.query_session_policies(enabled_only=True) or [] + except Exception as e: # noqa: BLE001 + log.warning("guard: policy read failed: %s", e) + return 0 + if not policies: + return 0 + + # How far each escalation ladder has already climbed. An unreadable + # ladder table degrades to {} — every ladder treated as being at rung 0, + # which is still protected by the per-rung latch below. + try: + ladder_state = store.query_policy_ladder_state() or {} + except Exception as e: # noqa: BLE001 + log.debug("guard: ladder state read failed (%s); assuming rung 0", e) + ladder_state = {} + + try: + decisions = _pe.evaluate(incidents, policies, facts, + ladder_state=ladder_state) or [] + except Exception as e: # noqa: BLE001 + log.warning("guard: evaluate errored: %s", e) + return 0 + if not decisions: + return 0 + + enforce_env = os.environ.get(_GUARD_ENFORCE, "0") == "1" + entitled = _guard_enforcement_allowed() if enforce_env else False + acted = 0 + + for d in decisions: + sid = d.get("session_id") or "" + pid = d.get("policy_id") or "" + action = d.get("action") or "monitor" + # Which rung of the ladder this decision is. A policy with no ladder + # is rung 0, which is exactly the old one-shot-per-session latch. + try: + step = max(0, int(d.get("step_index") or 0)) + except (TypeError, ValueError): + step = 0 + if _policy_step_already_fired(store, sid, pid, step): + continue + + actuating = _pe.is_actuating(action) + will_enforce = bool(actuating and enforce_env and entitled) + + # Record BEFORE acting. The row is the latch, so closing it first + # means a crash mid-signal can never re-fire on the next tick. + if not actuating: + detail = "recorded (no action for this policy type)" + elif not enforce_env: + detail = (f"DRY RUN: would {action} — set {_GUARD_ENFORCE}=1 " + f"to enforce") + elif not entitled: + detail = (f"would {action}, but autonomous action is not " + f"enabled on this plan") + else: + detail = "pending" + try: + _record_policy_step( + store, session_id=sid, policy_id=pid, + runtime=d.get("runtime") or "", action=action, + kind=d.get("kind") or "", reason=d.get("reason") or "", + evidence=d.get("evidence"), enforced=will_enforce, + result_ok=False, result_detail=detail, step_index=step) + except Exception as e: # noqa: BLE001 + log.warning("guard: could not record decision for %s: %s", sid, e) + continue + acted += 1 + + if not will_enforce: + log.info("guard: %s [%s] %s%s", sid, action, detail, + _ladder_suffix(d)) + continue + + result = _guard_actuate(d.get("runtime") or "", sid, + d.get("cwd") or "", action) + ok = bool(result.get("ok")) + rdetail = str(result.get("detail") or result.get("reason") + or result.get("error") or ("signalled" if ok else "failed")) + try: + _record_policy_step( + store, session_id=sid, policy_id=pid, + runtime=d.get("runtime") or "", action=action, + kind=d.get("kind") or "", reason=d.get("reason") or "", + evidence=d.get("evidence"), enforced=True, + result_ok=ok, result_detail=rdetail, step_index=step) + except Exception: + pass + log.info("guard: %s %s -> %s (%s)%s", action, sid, + "ok" if ok else "FAILED", rdetail[:120], _ladder_suffix(d)) + + return acted + + +def _record_policy_step(store, **kw) -> None: + """Write one policy-decision row, tolerating a pre-ladder store. + + Same version-skew hazard as ``_policy_step_already_fired``: the daemon + holding the writer lock may run an older wheel whose + ``record_policy_action`` has no ``step_index``. Dropping the argument + keeps rung 0 (i.e. every non-ladder policy) recording and enforcing + normally instead of taking the whole node out of enforcement. + + Raises on a genuine store failure so the caller's fail-closed handler + still refuses to act on an unlatched decision. + """ + try: + store.record_policy_action(**kw) + return + except TypeError: + pass + kw.pop("step_index", None) + store.record_policy_action(**kw) + + +def _policy_step_already_fired(store, session_id: str, policy_id: str, + step: int) -> bool: + """Per-rung latch check that tolerates an older store. + + ``policy_already_fired`` grew a ``step_index`` argument when escalation + ladders landed. The dashboard reaches the store through the daemon proxy, + and the daemon can be running an OLDER wheel than the code making the + call — so the three-argument form can land on a two-argument method. + Without this fallback that raises ``TypeError``, the fail-closed handler + swallows it, and every policy on the node silently stops enforcing. A + silent global disable is exactly the failure mode we most need to not + have. + + Fails CLOSED (returns True = do not act) for any *other* error, which is + the right direction when the alternative is acting twice. + """ + try: + return bool(store.policy_already_fired(session_id, policy_id, step)) + except TypeError: + pass # older signature — fall through + except Exception: + return True + try: + # Pre-ladder store: it can only latch per (session, policy). Rungs + # above 0 would be blocked forever by rung 0's row, so a ladder + # degrades to its first rung until the daemon is upgraded. Say so + # once rather than escalating on a latch that cannot track rungs. + if step > 0: + log.warning("guard: store predates escalation ladders; step %d of " + "policy %s will not fire until the daemon is upgraded", + step + 1, policy_id) + return True + return bool(store.policy_already_fired(session_id, policy_id)) + except Exception: + return True + + +def _ladder_suffix(decision: dict) -> str: + """`` [step 1/3, then kill in 300s]`` — appended to the guard log line. + + An escalation ladder is the one case where the log needs to say what + happens NEXT: reading "guard: sess-1 pause ok" gives no hint that a kill + is queued behind it. + """ + try: + count = int(decision.get("step_count") or 1) + if count <= 1: + return "" + idx = int(decision.get("step_index") or 0) + nxt = str(decision.get("next_action") or "") + tail = (f", then {nxt} in {int(decision.get('next_after_secs') or 0)}s" + if nxt else ", final step") + return f" [step {idx + 1}/{count}{tail}]" + except Exception: # noqa: BLE001 — a log decoration must never raise + return "" + + def _emit_detector_incidents(store, state: dict) -> int: """Run ``clawmetry.detectors`` over each active session, emit each incident as a ``loop_signals`` row (reusing the stuck detector's device-alert path) @@ -20286,6 +20538,7 @@ def _emit_detector_incidents(store, state: dict) -> int: baseline_cache: dict = {} bad_sessions: set = set() emitted = 0 + all_incidents: list = [] for s in candidates: sid = s.get("session_id") or "" try: @@ -20322,6 +20575,7 @@ def _emit_detector_incidents(store, state: dict) -> int: continue if not incidents: continue + all_incidents.extend(incidents) # Remember when this session FIRST looked wrong, so the next tick can # say how long it has been that way (and price the stretch). @@ -20429,6 +20683,15 @@ def _emit_detector_incidents(store, state: dict) -> int: memo.pop(k, None) except Exception: pass + # Guard policies: turn this tick's incidents into at most one enforcement + # decision per session. Isolated from the emit path above — a policy + # failure must never stop telemetry ingest. + try: + if all_incidents: + _apply_guard_policies(store, state, all_incidents, facts_by_session) + except Exception as e: # noqa: BLE001 + log.warning("guard: policy pass failed: %s", e) + return emitted diff --git a/clawmetry/templates/tabs/guard.html b/clawmetry/templates/tabs/guard.html new file mode 100644 index 0000000000..babd4eb228 --- /dev/null +++ b/clawmetry/templates/tabs/guard.html @@ -0,0 +1,49 @@ +
+
+

Guard

+

+ What is running right now, whether it has gone off track, and the button to stop it. +

+
+ + + +
+
+

Running sessions

+ + +
+
+
Loading sessions...
+
+
+ +
+
+

Policies

+ +
+

+ A policy watches for a detector signal and acts without anyone present. New policies start in + monitor mode: they record what they would have done and change nothing. A policy can escalate + over time, for example pause now and kill five minutes later if the agent is still stuck. Each + step runs only if the agent is still flagged when its wait is up. +

+ +
+
Loading policies...
+
+
+ +
+
+

Recent decisions

+
+
+
Loading...
+
+
+
diff --git a/dashboard.py b/dashboard.py index d02b6f0992..d7577a1677 100644 --- a/dashboard.py +++ b/dashboard.py @@ -108,6 +108,7 @@ from routes.harness import bp_harness from routes.delegated import bp_delegated from routes.readiness import bp_readiness +from routes.guard import bp_guard from routes.health import bp_health from routes.alerts import bp_alerts, bp_budget from routes.channels import bp_channels @@ -12987,6 +12988,7 @@ def detect_config(args=None): app.register_blueprint(bp_harness) app.register_blueprint(bp_delegated) app.register_blueprint(bp_readiness) + app.register_blueprint(bp_guard) app.register_blueprint(bp_health) app.register_blueprint(bp_logs) app.register_blueprint(bp_memory) @@ -13783,6 +13785,11 @@ def get_local_ip(): Approvals
+
+ + Guard + +
Alerts @@ -13901,6 +13908,7 @@ def get_local_ip(): {% include 'tabs/inventory.html' %} +{% include 'tabs/guard.html' %} {% include 'tabs/alerts.html' %} diff --git a/routes/guard.py b/routes/guard.py new file mode 100644 index 0000000000..e6d7ecf2f0 --- /dev/null +++ b/routes/guard.py @@ -0,0 +1,619 @@ +"""Guard — live session control and enforcement policies. + +Two surfaces, one feature: + +* **Control** (``/api/guard/sessions``, ``/api/guard/control``) — what is + running right now, whether a detector thinks it has gone off track, and a + Pause / Stop / Kill button per session. This is the human path. +* **Policies** (``/api/guard/policies``) — rules that let the DAEMON take the + same action with no human present. Authored here, evaluated in + ``sync.py::_emit_detector_incidents`` via ``clawmetry.policy_engine``. + +Both paths end in the SAME actuator (``sync._guard_actuate``) so an automatic +pause and a hand-pressed pause do exactly the same thing to the process. + +Naming note: ``/api/guard/policies`` is deliberately distinct from +``/api/tool-policy`` in ``routes/policy.py`` — that one is the pre-tool +sandbox/permission surface, this one is mid-run enforcement. Different axis, +different table, no shared state. +""" +import logging +import os +import re +import time +import uuid + +from flask import Blueprint, jsonify, request + +log = logging.getLogger("clawmetry.guard") + + +def _log_safe(v) -> str: + """One log token from a request-supplied value: no line breaks, bounded.""" + return str(v or "").replace("\r", " ").replace("\n", " ")[:128] + + +_DETAIL_OK = re.compile(r"[^A-Za-z0-9 _.,:;()'/-]") + +# Pre-filter for caller-supplied session identifiers: alphanumeric plus the +# ``_ - . :`` a stored id can legitimately carry (family rows are namespaced +# ``:``). Refuses slashes, null bytes and anything else that +# could influence a path or a command. This is only the first gate: the +# handler then resolves the id against the store and acts on the STORED +# copy, so a request can only ever name a session ClawMetry already knows. +_SID_SAFE_RE = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:\-]{0,127}$') +_POLICY_ID_RE = re.compile(r'^[A-Za-z0-9_\-]{1,128}$') + + +def _detail_safe(v) -> str: + """Reduce an actuator string to plain words before it reaches a response. + + Actuator dicts can carry stderr fragments or (historically) exception + text; stripping to a conservative character set breaks that path while + keeping every legitimate token (``unsupported_no_primitive``, + ``paused_via_proxy_hitl``, capability notes) readable. + """ + return _DETAIL_OK.sub("", str(v or ""))[:300] + +bp_guard = Blueprint("guard", __name__) + +# Actions a caller may ask for. `resume` is control-only (there is no policy +# that resumes; a human decides that). Anything not in here is refused rather +# than passed through to a signal helper. +_CONTROL_ACTIONS = ("pause", "resume", "stop", "kill") + +# Runtime ids are snake_case adapter names (``claude_code``, ``qwen_code``). +_RUNTIME_SAFE_RE = re.compile(r'^[a-z0-9_]{1,40}$') + + +def _ls_call(method_name, **kwargs): + """Cross-process LocalStore call with single-process fallback. + + Mirror of ``routes/health.py::_ls_call``. The daemon owns DuckDB's writer + lock, so a direct open from the dashboard raises on the standard install; + we go through the daemon proxy first and only fall back to a direct open + for single-process boots (tests + dev mode). + """ + try: + from routes.local_query import local_store_via_daemon + result = local_store_via_daemon(method_name, **kwargs) + if result is not None: + return result + except Exception: + pass + try: + from clawmetry import local_store + store = local_store.get_store(read_only=True) + return getattr(store, method_name)(**kwargs) + except Exception: + return None + + +def _ls_write(method_name, **kwargs): + """Route a WRITE at whichever process owns the DuckDB writer lock. + + Reads can fall back to a read-only open; writes cannot. On the standard + install the daemon holds the writer lock, so the proxy is the only path + that works; in single-process dev/test boots the direct open is. We try + both — every write here is idempotent (PK upsert / PK delete), so a + double-apply is harmless and one of the two landing is what matters. + + Returns nothing meaningful on purpose: ``local_store_via_daemon`` + returns ``None`` both for "method missing" and for a write that + succeeded (writes return None), so the return value cannot be trusted + (memory: feedback_dashboard_writes_noop_through_proxy). Callers MUST + verify by reading back. + """ + try: + from routes.local_query import local_store_via_daemon + local_store_via_daemon(method_name, **kwargs) + except Exception: + pass + try: + from clawmetry import local_store + getattr(local_store.get_store(), method_name)(**kwargs) + except Exception: + pass + + +def _same_origin_ok() -> bool: + """Reject cross-site POSTs to the mutating endpoints. + + These routes can SIGKILL a user's agent, so a drive-by form post from any + open browser tab to ``localhost:8900`` must not reach the actuator. A + request with no Origin/Referer at all (curl, the CLI, tests) is allowed — + browsers always send one for a cross-site POST, so absence is not the + attack we are blocking here. + """ + origin = request.headers.get("Origin") or "" + if not origin: + referer = request.headers.get("Referer") or "" + if not referer: + return True + origin = referer + try: + from urllib.parse import urlparse + host = urlparse(origin).netloc.lower() + except Exception: + return False + if not host: + return False + req_host = (request.host or "").lower() + if host == req_host: + return True + # Same port on an equivalent loopback name (localhost vs 127.0.0.1). + loopback = ("localhost", "127.0.0.1", "[::1]", "::1") + def _split(h): + if h.startswith("["): + close = h.find("]") + return h[:close + 1], h[close + 2:] if close + 1 < len(h) else "" + parts = h.rsplit(":", 1) + return (parts[0], parts[1]) if len(parts) == 2 else (h, "") + o_host, o_port = _split(host) + r_host, r_port = _split(req_host) + return o_host in loopback and r_host in loopback and o_port == r_port + + +def _runtime_supports_signals(runtime: str, session_id: str = "", + cwd: str = "") -> dict: + """Can we actually control this SESSION, on this platform, right now? + + Answering honestly at LIST time is the point: a Stop button that silently + does nothing is worse than a disabled one with a reason next to it. + + The verdict comes from ``process_control.runtime_control_support`` so the + UI, the daemon and the actuator all read the same answer. It is per + session, not per runtime, because three things vary independently: + + * the OS — POSIX signals, the Windows native equivalents, or neither; + * the session's execution model — a Cursor CLI session is a real process + tree and IS controllable, while a Cursor editor conversation is not; + * for OpenClaw, whether the enforcement proxy is running, which decides + whether Pause does anything at all. + """ + try: + from clawmetry import process_control as _pc + except Exception: + return {"controllable": False, "reason": "process_control unavailable", + "actions": []} + try: + return _pc.runtime_control_support(runtime, session_id, cwd) + except Exception: # noqa: BLE001 — never break the list render + log.exception("guard capability check failed for %s", + _log_safe(session_id)) + return {"controllable": False, "actions": [], + "reason": "capability check failed; see the server log"} + + +def _session_runtime(session_id: str, agent_type: str) -> str: + """Which runtime is this session, really? Mirrors ``sync._detector_runtime``: + the session-id prefix wins, ``agent_type`` is only the fallback, because on + a real install the column reads ``openclaw`` for nearly every row.""" + try: + from clawmetry import waste_flags as _wf + rt = str(_wf.runtime_from_session_id(session_id) or "").strip().lower() + except Exception: + rt = "" + return rt or str(agent_type or "").strip().lower() + + +# Severity ladder shared with ``clawmetry.detectors`` (higher is louder). +_SEVERITY_RANK = {"info": 0, "warning": 1, "critical": 2} + + +def _incident_rank(inc) -> tuple: + """Sort key for one incident: money, then severity, then size. + + Kept in sync with ``detectors.incident_rank``. It is duplicated rather + than imported because this route must render on a cloud instance where + ``clawmetry.detectors`` may be absent, and a missing import must not take + the Guard tab down with it. + """ + if not isinstance(inc, dict): + return (0.0, 0, 0) + try: + spend = float(inc.get("spend_at_risk_usd") or 0) + except (TypeError, ValueError): + spend = 0.0 + sev = _SEVERITY_RANK.get(str(inc.get("severity") or "").lower(), 0) + try: + count = int(inc.get("count") or 0) + except (TypeError, ValueError): + count = 0 + return (spend, sev, count) + + +@bp_guard.route("/api/guard/sessions") +def api_guard_sessions(): + """Live sessions with their current Guard status. + + One row per active session: identity, spend, whether a detector currently + flags it, and whether this node can actually signal it. Returns an empty + list (HTTP 200) on any store error so the tab renders an honest empty + state instead of an error page. + """ + try: + limit = max(1, min(int(request.args.get("limit", 50)), 200)) + except (TypeError, ValueError): + limit = 50 + + sessions = _ls_call("query_sessions_table", limit=limit) or [] + signals = _ls_call("query_recent_loop_signals", limit=200, + since_minutes=30) or [] + + # Newest incident per session wins; a session can trip several detectors. + incident_by_session = {} + for sig in signals: + if not isinstance(sig, dict): + continue + sid = str(sig.get("session_id") or "") + if not sid: + continue + details = sig.get("details") + if isinstance(details, str): + try: + import json as _json + details = _json.loads(details) + except Exception: + details = {} + details = details if isinstance(details, dict) else {} + prev = incident_by_session.get(sid) + count = int(sig.get("repeat_count") or 0) + try: + at_risk = round(float(details.get("spend_at_risk_usd") or 0), 4) + except (TypeError, ValueError): + at_risk = 0.0 + candidate = { + "kind": str(details.get("kind") or ""), + "title": str(details.get("message") or ""), + "detail": str(details.get("detail") or ""), + "severity": str(sig.get("severity") or "warning"), + "count": count, + "since": sig.get("first_seen"), + # What ignoring this stretch is estimated to cost, and on what + # basis. ``basis`` travels with the number on purpose: a reader + # must be able to tell a measured burn rate from a guess. + "spend_at_risk_usd": at_risk, + "spend_basis": str(details.get("spend_basis") or "unknown"), + "evidence": details.get("evidence") + if isinstance(details.get("evidence"), dict) else {}, + } + # A session can trip several detectors at once. The one that gets the + # row is the one that costs the most to ignore, falling back to + # severity and then to count when no cost is known — the same order + # ``detectors.incident_rank`` uses, so the tab and the daemon agree on + # which finding is the loudest. + if prev is None or _incident_rank(candidate) > _incident_rank(prev): + incident_by_session[sid] = candidate + + out = [] + for s in sessions: + if not isinstance(s, dict): + continue + if s.get("ended_at"): + continue + status = str(s.get("status") or "").lower() + if status in ("ended", "completed", "stopped", "failed"): + continue + sid = str(s.get("session_id") or "") + if not sid: + continue + # The ``sessions`` table's ``agent_type`` reads ``openclaw`` for + # nearly every row on a real install; the session-id prefix is the + # identity the rest of the product uses (same derivation as + # ``sync._detector_runtime``). Trusting the column here would hand + # ``runtime_control_support`` the wrong runtime for every family + # session and disable controls that work. + runtime = _session_runtime(sid, s.get("agent_type") or "") + meta = s.get("metadata") + meta = meta if isinstance(meta, dict) else {} + cwd = "" + for key in ("cwd", "workspace", "project_dir", "working_dir", "path"): + val = meta.get(key) + if isinstance(val, str) and val.strip(): + cwd = val.strip() + break + try: + cost = round(float(s.get("cost_usd") or 0), 4) + except (TypeError, ValueError): + cost = 0.0 + support = _runtime_supports_signals(runtime, sid, cwd) + out.append({ + "session_id": sid, + "runtime": runtime, + "agent_id": str(s.get("agent_id") or ""), + "title": str(s.get("title") or "")[:160], + "status": status, + "started_at": s.get("started_at"), + "last_active_at": s.get("last_active_at"), + "cost_usd": cost, + "total_tokens": int(s.get("total_tokens") or 0), + "message_count": int(s.get("message_count") or 0), + "cwd": cwd, + "incident": incident_by_session.get(sid), + "controllable": support["controllable"], + "control_reason": support.get("reason", ""), + "no_pause": support.get("no_pause", False), + # Which buttons may be enabled for THIS session. Empty means none. + "control_actions": support.get("actions", []), + # Why a control behaves differently here (OpenClaw's proxy-backed + # pause, the Windows Ctrl+C blast radius). Rendered as a hint. + "control_note": support.get("note", "") + or support.get("platform", {}).get("note", ""), + }) + + # Flagged sessions first, most expensive to ignore at the top; unflagged + # sessions keep their recency order below. Sorting by severity alone put a + # $0.02 "continued after a failed command" above a $170 loop, which is the + # ranking the money model exists to fix. + out.sort(key=lambda r: ( + 1 if r.get("incident") else 0, + _incident_rank(r.get("incident")), + ), reverse=True) + + flagged = [r for r in out if r.get("incident")] + return jsonify({ + "sessions": out, + "count": len(out), + "flagged": len(flagged), + # The headline number for the tab: what the flagged stretches are + # estimated to be burning right now. + "spend_at_risk_usd": round(sum( + float((r.get("incident") or {}).get("spend_at_risk_usd") or 0) + for r in flagged), 2), + }) + + +@bp_guard.route("/api/guard/control", methods=["POST"]) +def api_guard_control(): + """Pause / resume / stop / kill one session, on the user's explicit click. + + Not entitlement-gated: the user pressed the button, exactly as the manual + budget pause bypasses ``_auto_pause_allowed``. What IS gated is the daemon + deciding to do this on its own (see ``sync._guard_enforcement_allowed``). + """ + if not _same_origin_ok(): + return jsonify({"ok": False, "error": "cross-origin request refused"}), 403 + + data = request.get_json(silent=True) or {} + action = str(data.get("action") or "").strip().lower() + session_id = str(data.get("session_id") or "").strip() + runtime = str(data.get("runtime") or "").strip().lower() + cwd = str(data.get("cwd") or "").strip() + + # Literal tuple on purpose: a comparison against constants is the one + # sanitizer static analysis credits, and ``action`` is echoed into the + # audit trail and the log. + if action not in ("pause", "resume", "stop", "kill"): + return jsonify({"ok": False, + "error": f"action must be one of {list(_CONTROL_ACTIONS)}"}), 400 + if not session_id: + return jsonify({"ok": False, "error": "session_id is required"}), 400 + if not _SID_SAFE_RE.match(session_id) or ".." in session_id: + return jsonify({"ok": False, "error": "invalid session_id"}), 400 + if runtime and not _RUNTIME_SAFE_RE.match(runtime): + return jsonify({"ok": False, "error": "invalid runtime"}), 400 + if cwd: + try: + cwd = os.path.realpath(cwd) + except Exception: + return jsonify({"ok": False, "error": "invalid cwd"}), 400 + + # Act on the STORED session, not the request. The store's own copy of the + # id and working directory are what reach the signal helpers, so a + # request can name a session but never supply the strings a process is + # located or signalled with. A session the store does not know cannot be + # controlled from here — it is not on any list this dashboard renders. + recorded = _ls_call("get_session_location", session_id=session_id) + if not isinstance(recorded, dict) or not recorded.get("session_id"): + return jsonify({"ok": False, "error": "unknown session", + "detail": "session_not_in_store"}), 404 + stored_sid = str(recorded.get("session_id") or "") + stored_cwd = str(recorded.get("cwd") or "") + if cwd and stored_cwd and os.path.realpath(stored_cwd) != cwd: + # A crafted request cannot redirect signals to an arbitrary + # working directory. + return jsonify({"ok": False, + "error": "cwd does not match session record"}), 400 + + try: + # Every control action — resume included — goes through the actuator + # the daemon's policies use, so a manual pause and an automatic one + # are indistinguishable to the agent process. + from clawmetry.guard_actuator import guard_actuate + result = guard_actuate(runtime, stored_sid, stored_cwd, action) + except Exception: # noqa: BLE001 + # Full detail goes to the server log; the client gets a generic + # message so an exception can never leak internals to the page. + # Request-supplied values are stripped of line breaks before logging + # so a crafted id cannot forge extra log lines. + log.exception("guard control %s failed for %s", + _log_safe(action), _log_safe(session_id)) + return jsonify({"ok": False, + "error": "control action failed; see the server log", + "session_id": session_id, "action": action}), 500 + + result = result if isinstance(result, dict) else {"ok": False} + ok = bool(result.get("ok")) + + # Manual actions belong in the same audit trail as automatic ones. + try: + from clawmetry import audit as _a + _a.audit_event( + f"guard.{action}", + actor="dashboard", + target=stored_sid, + result="ok" if ok else "failed", + source="dashboard", + metadata={"runtime": runtime, + "detail": str(result.get("detail") or "")[:200]}, + ) + except Exception: + pass + + # A curated result, not the raw actuator dict: an actuator error string + # can carry an exception message, and the client only needs the fields + # the UI renders. ``detail`` is reduced to a plain-word token set so no + # exception text or control characters can reach the page. + return jsonify({ + "ok": ok, + "action": action, + "session_id": stored_sid, + "runtime": runtime, + "detail": _detail_safe(result.get("detail") or result.get("reason") + or result.get("error") or ""), + "advisory_only": bool(result.get("advisory_only")), + "mechanism": _detail_safe(result.get("mechanism"))[:80], + "note": _detail_safe(result.get("note")), + "unsupported": (None if result.get("unsupported") is None + else _detail_safe(result.get("unsupported"))[:80]), + }) + + +@bp_guard.route("/api/guard/policies", methods=["GET", "POST"]) +def api_guard_policies(): + """List Guard policies, or create/update one. + + A new policy defaults to ``action="monitor"`` — it records what it WOULD + have done and changes nothing — so authoring a rule can never surprise + anyone. Escalating to pause/stop/kill is a deliberate second step, and + still needs ``CLAWMETRY_POLICY_ENFORCE=1`` on the node to bite. + + A policy may also carry an escalation ladder as ``steps``:: + + "steps": [{"action": "pause", "after_secs": 0}, + {"action": "kill", "after_secs": 300}] + + Each rung passes through the same three locks as a plain action, and only + fires if the session is still matching when its delay elapses. Rungs are + normalized (and malformed ones dropped) by + ``policy_engine.normalize_steps`` before storage, so what comes back from + a read is exactly what will be evaluated. + """ + if request.method == "GET": + rows = _ls_call("query_session_policies") or [] + import os as _os + from clawmetry.policy_engine import (ACTIONS, MAX_LADDER_STEPS, + MAX_STEP_DELAY_SECS) + # Show the ladder the ENGINE will run, not the raw column: a policy + # stored before ladders existed reads back as its single action, and + # the UI should render one shape for both. + try: + from clawmetry.policy_engine import normalize_steps as _norm + for r in rows: + if isinstance(r, dict): + r["steps"] = _norm(r) + except Exception: + pass + return jsonify({ + "policies": rows, + "count": len(rows), + "actions": list(ACTIONS), + "max_ladder_steps": MAX_LADDER_STEPS, + "max_step_delay_secs": MAX_STEP_DELAY_SECS, + # The UI must be able to say "these rules are not enforcing". + "enforcement_enabled": _os.environ.get( + "CLAWMETRY_POLICY_ENFORCE", "0") == "1", + "evaluation_enabled": _os.environ.get( + "CLAWMETRY_GUARD_POLICIES", "1") != "0", + }) + + if not _same_origin_ok(): + return jsonify({"ok": False, "error": "cross-origin request refused"}), 403 + + data = request.get_json(silent=True) or {} + from clawmetry.policy_engine import ACTIONS, MAX_LADDER_STEPS, normalize_steps + action = str(data.get("action") or "monitor").strip().lower() + if action not in ACTIONS: + return jsonify({"ok": False, + "error": f"action must be one of {list(ACTIONS)}"}), 400 + + # Reject a bad ladder loudly instead of silently storing a shorter one. + # normalize_steps DROPS unusable rungs by design (never coerces them into + # some other action), which is right for the engine reading old rows but + # wrong for a fresh author request: an operator who typed "terminate" + # must be told, not handed a ladder quietly missing a rung. + raw_steps = data.get("steps") + if raw_steps not in (None, "", []): + if not isinstance(raw_steps, list): + return jsonify({"ok": False, + "error": "steps must be a list of " + "{action, after_secs} objects"}), 400 + if len(raw_steps) > MAX_LADDER_STEPS: + return jsonify({"ok": False, + "error": f"a ladder may have at most " + f"{MAX_LADDER_STEPS} steps"}), 400 + for i, entry in enumerate(raw_steps): + if not isinstance(entry, dict): + return jsonify({"ok": False, + "error": f"step {i + 1} must be an object"}), 400 + act = str(entry.get("action") or "").strip().lower() + if act not in ACTIONS: + return jsonify({"ok": False, + "error": f"step {i + 1}: action must be one " + f"of {list(ACTIONS)}"}), 400 + delay = entry.get("after_secs", 0) + try: + if int(delay or 0) < 0: + raise ValueError + except (TypeError, ValueError): + return jsonify({"ok": False, + "error": f"step {i + 1}: after_secs must be a " + f"non-negative number of seconds"}), 400 + + policy_id = str(data.get("policy_id") or "").strip() or f"gp-{uuid.uuid4().hex[:12]}" + policy = { + "policy_id": policy_id, + "name": str(data.get("name") or "")[:200], + "enabled": bool(data.get("enabled", True)), + "scope_runtime": str(data.get("scope_runtime") or "").strip(), + "scope_agent_id": str(data.get("scope_agent_id") or "").strip(), + "trigger_kind": str(data.get("trigger_kind") or "").strip(), + "min_severity": str(data.get("min_severity") or "info").strip(), + "min_repeat": data.get("min_repeat") or 0, + "min_duration_s": data.get("min_duration_s") or 0, + "min_spend_usd": data.get("min_spend_usd") or 0, + "min_spend_at_risk_usd": data.get("min_spend_at_risk_usd") or 0, + "action": action, + "steps": raw_steps or [], + } + # Echo back the ladder the engine will actually run (step 0's delay is + # forced to 0, delays are clamped), so the UI never shows the operator a + # ladder that differs from the stored one. + policy["steps"] = normalize_steps(policy) + _ls_write("upsert_session_policy", policy=policy) + # Verify by reading back: a write through the daemon proxy returns None + # whether it landed or not, so "no exception" is not evidence of success. + rows = _ls_call("query_session_policies") or [] + if not any(r.get("policy_id") == policy_id for r in rows): + return jsonify({"ok": False, "error": "policy store unavailable"}), 503 + return jsonify({"ok": True, "policy_id": policy_id, "policy": policy}) + + +@bp_guard.route("/api/guard/policies/", methods=["DELETE"]) +def api_guard_policy_delete(policy_id): + """Delete one Guard policy.""" + if not _same_origin_ok(): + return jsonify({"ok": False, "error": "cross-origin request refused"}), 403 + if not _POLICY_ID_RE.match(policy_id or ""): + return jsonify({"ok": False, "error": "invalid policy_id"}), 400 + _ls_write("delete_session_policy", policy_id=policy_id) + rows = _ls_call("query_session_policies") or [] + still_there = any(r.get("policy_id") == policy_id for r in rows) + return jsonify({"ok": not still_there, "policy_id": policy_id}) + + +@bp_guard.route("/api/guard/actions") +def api_guard_actions(): + """Recent policy decisions — what fired, what it did, and why. + + Includes dry-run (``monitor``) decisions, which is the point: before + turning enforcement on you can read exactly what would have happened. + """ + try: + limit = max(1, min(int(request.args.get("limit", 50)), 500)) + except (TypeError, ValueError): + limit = 50 + rows = _ls_call("query_policy_actions", limit=limit) or [] + return jsonify({"actions": rows, "count": len(rows), + "server_time": int(time.time())}) diff --git a/routes/local_query.py b/routes/local_query.py index 0ba76ecec5..fe5f8aef9c 100644 --- a/routes/local_query.py +++ b/routes/local_query.py @@ -659,6 +659,16 @@ def http_query(): # by nobody and sat on "never triggered" forever. routes/alerts.py now # mirrors on write/update/delete through these two. "ingest_alert_rule", + # ── Guard: live session control + enforcement policies ────────────── + # The Guard tab authors policies from the dashboard process, but the + # daemon owns the DuckDB writer lock, so every one of these has to be + # reachable through the proxy. Without them the reads return None and the + # tab renders an empty state that looks like "no policies" rather than + # "could not reach the store" — and the writes silently no-op. + "query_session_policies", + "upsert_session_policy", + "delete_session_policy", + "query_policy_actions", "delete_alert_rule", "query_channel_config_status", "query_crons", diff --git a/tests/test_e2e_oss_all_tabs.py b/tests/test_e2e_oss_all_tabs.py index c73a06e2ff..c5970600ca 100644 --- a/tests/test_e2e_oss_all_tabs.py +++ b/tests/test_e2e_oss_all_tabs.py @@ -130,6 +130,7 @@ def _overlay_page(_shared_chromium): "harness", # harness.html: harness observability "inventory", # inventory.html: tool/resource inventory "nemoclaw", # nemoclaw.html: NeMo Guardrails governance + "guard", # guard.html: Guard incidents + session controls + policies "policy", # policy.html: policy management "selfevolve", # selfevolve.html: self-evolve feature "swimlane", # swimlane.html: swimlane visualization diff --git a/tests/test_guard_control_capability.py b/tests/test_guard_control_capability.py new file mode 100644 index 0000000000..bd6bcbbc1c --- /dev/null +++ b/tests/test_guard_control_capability.py @@ -0,0 +1,186 @@ +"""Control capability must be answered per SESSION, and honestly. + +Two gaps this covers: + +* **OpenClaw pause claimed enforcement it did not have.** A pause on an + OpenClaw session writes ``~/.clawmetry/hitl/pause_``, and the only + thing that enforces that file is the optional enforcement proxy. On a node + with no proxy the old code still reported "the proxy refuses further LLM + calls" — a pause that said it stopped an agent which was in fact still + running. +* **Capability was answered per runtime.** Cursor was refused wholesale even + though Cursor CLI sessions are real process trees, and the platform check + refused every Windows node outright. +""" +import clawmetry.process_control as pc +import pytest + + +@pytest.fixture +def no_proxy(monkeypatch): + monkeypatch.setattr(pc, "enforcement_proxy_status", + lambda: {"running": False, "pid": None, + "reason": "no proxy pid file"}) + + +@pytest.fixture +def live_proxy(monkeypatch): + monkeypatch.setattr(pc, "enforcement_proxy_status", + lambda: {"running": True, "pid": 4242, "reason": ""}) + + +# ── OpenClaw pause honesty ──────────────────────────────────────────────── +def test_openclaw_pause_is_inert_without_the_proxy(no_proxy): + cap = pc.openclaw_pause_capability() + assert cap["effective"] is False + assert cap["mechanism"] == "none" + # It must say the agent keeps running, and point at the thing that works. + assert "keeps running" in cap["detail"] + assert "Stop" in cap["detail"] + + +def test_openclaw_pause_is_effective_with_the_proxy(live_proxy): + cap = pc.openclaw_pause_capability() + assert cap["effective"] is True + assert cap["mechanism"] == "proxy_hitl" + assert cap["proxy_pid"] == 4242 + + +def test_openclaw_offers_no_pause_button_without_the_proxy(no_proxy): + sup = pc.runtime_control_support("openclaw", "sess-1") + assert sup["controllable"] is True # Stop still works + assert "pause" not in sup["actions"] + assert sup["no_pause"] is True + assert sup["actions"] == ["stop", "kill"] + + +def test_openclaw_offers_pause_when_the_proxy_is_live(live_proxy): + sup = pc.runtime_control_support("openclaw", "sess-1") + assert "pause" in sup["actions"] and "resume" in sup["actions"] + assert sup["no_pause"] is False + + +def test_actuator_reports_openclaw_pause_as_failed_without_a_proxy(no_proxy, monkeypatch): + """The actuator's own return value — what gets recorded in the audit + trail and shown to the operator — must not claim success.""" + from clawmetry import sync + monkeypatch.setattr(sync, "_hitl_set_pause", lambda sid, paused: None) + res = sync._guard_actuate("openclaw", "sess-1", "", "pause") + assert res["ok"] is False + assert res["advisory_only"] is True + assert res["detail"] == "unsupported_no_primitive" + + +def test_actuator_reports_openclaw_pause_as_real_with_a_proxy(live_proxy, monkeypatch): + from clawmetry import sync + monkeypatch.setattr(sync, "_hitl_set_pause", lambda sid, paused: None) + res = sync._guard_actuate("openclaw", "sess-1", "", "pause") + assert res["ok"] is True + assert res["detail"] == "paused_via_proxy_hitl" + assert res["advisory_only"] is False + + +def test_resume_goes_through_the_shared_actuator(live_proxy, monkeypatch): + """Resume used to bypass _guard_actuate and call resume_session directly, + so it returned 'unsupported' for OpenClaw sessions the proxy could have + released.""" + from clawmetry import sync + monkeypatch.setattr(sync, "_hitl_set_pause", lambda sid, paused: None) + res = sync._guard_actuate("openclaw", "sess-1", "", "resume") + assert res["ok"] is True + assert res["detail"] == "resumed_via_proxy_hitl" + + +# ── proxy liveness probe ────────────────────────────────────────────────── +def test_proxy_probe_treats_a_stale_pid_file_as_not_running(monkeypatch, tmp_path): + pid_file = tmp_path / "proxy.pid" + pid_file.write_text("999999") + monkeypatch.setattr(pc, "_PROXY_PID_FILE", str(pid_file)) + monkeypatch.setattr(pc, "is_alive", lambda p: False) + st = pc.enforcement_proxy_status() + assert st["running"] is False and st["reason"] == "stale proxy pid file" + + +def test_proxy_probe_survives_a_garbage_pid_file(monkeypatch, tmp_path): + pid_file = tmp_path / "proxy.pid" + pid_file.write_text("not-a-pid") + monkeypatch.setattr(pc, "_PROXY_PID_FILE", str(pid_file)) + assert pc.enforcement_proxy_status()["running"] is False + + +def test_proxy_probe_reports_a_live_proxy(monkeypatch, tmp_path): + pid_file = tmp_path / "proxy.pid" + pid_file.write_text(" 4242\n") + monkeypatch.setattr(pc, "_PROXY_PID_FILE", str(pid_file)) + monkeypatch.setattr(pc, "is_alive", lambda p: p == 4242) + st = pc.enforcement_proxy_status() + assert st["running"] is True and st["pid"] == 4242 + + +# ── per-session capability, not per runtime ─────────────────────────────── +def test_cursor_cli_session_is_controllable(monkeypatch): + """A Cursor CLI session is a real process tree. Blanket-refusing the + runtime hid working buttons from these sessions.""" + monkeypatch.setattr(pc, "resolve_session", + lambda rt, sid="", cwd="": {"ok": True, "pid": 500, + "runtime": "cursor"}) + sup = pc.runtime_control_support("cursor", "cli-sess", "/repo") + assert sup["controllable"] is True + assert sup["actions"] == ["pause", "resume", "stop", "kill"] + assert sup["resolved_pid"] == 500 + + +def test_cursor_editor_session_is_refused_with_a_readable_reason(monkeypatch): + monkeypatch.setattr( + pc, "resolve_session", + lambda rt, sid="", cwd="": { + "ok": False, "unsupported": True, + "reason": "cursor_single_ide_process_no_per_session_signal"}) + sup = pc.runtime_control_support("cursor", "ide-sess") + assert sup["controllable"] is False and sup["actions"] == [] + # An operator-readable sentence, not the raw resolver enum. + assert "shared IDE process" in sup["reason"] + assert "_no_per_session_signal" not in sup["reason"] + + +def test_every_supported_runtime_is_controllable(): + """The list widened upstream (copilot, qwen_code, kimi, pi, grok, + deepseek_harness); the capability answer must track it, not a stale copy.""" + for rt in pc.SUPPORTED_RUNTIMES: + sup = pc.runtime_control_support(rt, "s", "/tmp") + assert sup["controllable"] is True, rt + assert "kill" in sup["actions"], rt + + +def test_unknown_runtime_is_refused(): + sup = pc.runtime_control_support("some-new-harness", "s") + assert sup["controllable"] is False + assert "some-new-harness" in sup["reason"] + + +def test_unsupported_platform_refuses_every_runtime(monkeypatch): + monkeypatch.setattr(pc, "_POSIX", False) + monkeypatch.setattr(pc, "_IS_WINDOWS", False) + for rt in ("claude_code", "openclaw", "cursor"): + sup = pc.runtime_control_support(rt, "s") + assert sup["controllable"] is False and sup["actions"] == [] + + +def test_windows_node_is_controllable(monkeypatch): + """The gap: every Guard button was inert on Windows.""" + monkeypatch.setattr(pc, "_POSIX", False) + monkeypatch.setattr(pc, "_IS_WINDOWS", True) + sup = pc.runtime_control_support("claude_code", "s") + assert sup["controllable"] is True + assert sup["platform"]["mechanism"] == "win32_native" + + +def test_capability_never_raises(monkeypatch): + monkeypatch.setattr(pc, "resolve_session", + lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom"))) + with pytest.raises(RuntimeError): + pc.resolve_session("cursor") + # The route wrapper is the layer that must absorb it. + from routes.guard import _runtime_supports_signals + sup = _runtime_supports_signals("cursor", "s", "/tmp") + assert sup["controllable"] is False and sup["actions"] == [] diff --git a/tests/test_guard_control_route.py b/tests/test_guard_control_route.py new file mode 100644 index 0000000000..8c68439edb --- /dev/null +++ b/tests/test_guard_control_route.py @@ -0,0 +1,136 @@ +"""``POST /api/guard/control`` acts on the STORED session, never on request +strings. + +The handler resolves the caller's ``session_id`` against the local store and +hands the store's own copy of the id and working directory to the actuator. +That is the whole security argument for the endpoint: a request can NAME a +session ClawMetry already knows, but it can never supply the string a +process is located or signalled with. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import pytest # noqa: E402 +from flask import Flask # noqa: E402 + +import routes.guard as g # noqa: E402 +import clawmetry.guard_actuator as ga # noqa: E402 +from routes.guard import bp_guard # noqa: E402 + + +@pytest.fixture +def client(monkeypatch): + app = Flask(__name__) + app.register_blueprint(bp_guard) + app.config["TESTING"] = True + return app.test_client() + + +@pytest.fixture +def store(monkeypatch): + """A store that knows exactly one session, with a recorded cwd.""" + rows = {"claude_code:abc-123": {"session_id": "claude_code:abc-123", + "cwd": "/tmp/proj", "git_branch": "main", + "metadata": {}}} + + def _call(method, **kw): + if method == "get_session_location": + return rows.get(kw.get("session_id")) + return None + + monkeypatch.setattr(g, "_ls_call", _call) + return rows + + +@pytest.fixture +def actuator(monkeypatch): + calls = [] + + def fake(runtime, session_id, cwd, action): + calls.append({"runtime": runtime, "session_id": session_id, + "cwd": cwd, "action": action}) + return {"ok": True, "detail": "signalled"} + + monkeypatch.setattr(ga, "guard_actuate", fake) + return calls + + +def test_unknown_session_is_refused_before_any_actuator_call(client, store, actuator): + r = client.post("/api/guard/control", + json={"action": "pause", "session_id": "nope-1", + "runtime": "claude_code"}) + assert r.status_code == 404 + assert r.get_json()["detail"] == "session_not_in_store" + assert actuator == [] + + +def test_actuator_receives_the_stored_id_and_cwd_not_the_request(client, store, actuator): + r = client.post("/api/guard/control", + json={"action": "stop", "session_id": "claude_code:abc-123", + "runtime": "claude_code"}) + assert r.status_code == 200, r.get_json() + assert r.get_json()["ok"] is True + assert actuator == [{"runtime": "claude_code", + "session_id": "claude_code:abc-123", + "cwd": "/tmp/proj", "action": "stop"}] + + +def test_namespaced_family_ids_pass_the_prefilter(client, store, actuator): + """Family rows are stored as ``:``; the pre-filter must not + refuse the colon or every Claude Code / Codex row loses its buttons.""" + r = client.post("/api/guard/control", + json={"action": "pause", "session_id": "claude_code:abc-123"}) + assert r.status_code == 200 + assert actuator[0]["session_id"] == "claude_code:abc-123" + + +@pytest.mark.parametrize("bad", ["../etc", "a/b", "x..y", "-lead", "", "a b"]) +def test_path_like_ids_are_refused_by_the_prefilter(client, store, actuator, bad): + r = client.post("/api/guard/control", + json={"action": "pause", "session_id": bad}) + assert r.status_code == 400 + assert actuator == [] + + +def test_a_cwd_that_disagrees_with_the_record_is_refused(client, store, actuator): + r = client.post("/api/guard/control", + json={"action": "kill", "session_id": "claude_code:abc-123", + "cwd": "/somewhere/else"}) + assert r.status_code == 400 + assert "cwd" in r.get_json()["error"] + assert actuator == [] + + +def test_unknown_action_and_runtime_are_refused(client, store, actuator): + r = client.post("/api/guard/control", + json={"action": "explode", "session_id": "claude_code:abc-123"}) + assert r.status_code == 400 + r = client.post("/api/guard/control", + json={"action": "pause", "session_id": "claude_code:abc-123", + "runtime": "Bad Runtime"}) + assert r.status_code == 400 + assert actuator == [] + + +def test_actuator_failure_is_a_fixed_token_not_exception_text(client, store, monkeypatch): + def boom(runtime, session_id, cwd, action): + raise RuntimeError("secret /Users/x/traceback.py line 12") + + monkeypatch.setattr(ga, "guard_actuate", boom) + r = client.post("/api/guard/control", + json={"action": "pause", "session_id": "claude_code:abc-123"}) + assert r.status_code == 500 + body = r.get_json() + assert "traceback" not in str(body) + assert body["error"] == "control action failed; see the server log" + + +def test_daemon_and_route_share_one_actuator(): + """The policy pass in the daemon and the HTTP route must call the same + function object — there is no second path to a process.""" + from clawmetry import sync + assert sync._guard_actuate is ga.guard_actuate diff --git a/tests/test_guard_escalation_ladder.py b/tests/test_guard_escalation_ladder.py new file mode 100644 index 0000000000..c6d0508f57 --- /dev/null +++ b/tests/test_guard_escalation_ladder.py @@ -0,0 +1,306 @@ +"""Escalation ladders: pause, wait, then kill if still stuck. + +Before this a policy fired ONE action ONCE. The ladder existed only as an +ordering of action names ("kill is stronger than pause"), never as a sequence +over time, so the shape real operations want — *pause it, tell me, give it +five minutes, then kill it if it is still stuck* — could not be expressed. + +Covered here: the pure timing rules, the durable per-rung latch (including +the DuckDB PK migration), and that a ladder cannot slip any of the three +safety locks. +""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import pytest # noqa: E402 + +from clawmetry import policy_engine as pe # noqa: E402 + + +def _incident(session_id="s1"): + return {"kind": "no_progress", "session_id": session_id, + "runtime": "claude_code", "severity": "warning", + "title": "agent not advancing", + "evidence": {"total_tool_calls": 40}} + + +_FACTS = {"s1": {"cost_usd": 9.0, "bad_for_seconds": 600, + "runtime": "claude_code", "cwd": "/tmp/x", "agent_id": "main"}} + +_LADDER = [{"action": "pause", "after_secs": 0}, + {"action": "alert", "after_secs": 60}, + {"action": "kill", "after_secs": 300}] + + +def _policy(steps=None, action="monitor", policy_id="p1"): + p = {"policy_id": policy_id, "enabled": True, "scope_runtime": "", + "scope_agent_id": "", "trigger_kind": "", "min_severity": "info", + "min_repeat": 0, "min_duration_s": 0, "min_spend_usd": 0, + "action": action} + if steps is not None: + p["steps"] = steps + return p + + +def _state(step, at): + return {"s1": {"p1": {"last_step": step, "last_fired_at": at}}} + + +# ── normalize_steps ─────────────────────────────────────────────────────── +def test_a_policy_without_steps_is_a_one_rung_ladder(): + """The compatibility guarantee: every pre-ladder policy is untouched.""" + steps = pe.normalize_steps(_policy(action="kill")) + assert steps == [{"action": "kill", "after_secs": 0}] + + +def test_step_zero_delay_is_forced_to_zero(): + steps = pe.normalize_steps(_policy([{"action": "pause", "after_secs": 999}])) + assert steps[0]["after_secs"] == 0 + + +def test_unknown_actions_are_dropped_never_coerced(): + """Coercing 'terminate' to some default would silently change what the + rule does to someone's agent.""" + steps = pe.normalize_steps(_policy([ + {"action": "pause", "after_secs": 0}, + {"action": "terminate", "after_secs": 10}, + {"action": "kill", "after_secs": 30}, + ])) + assert [s["action"] for s in steps] == ["pause", "kill"] + + +def test_a_ladder_of_only_bad_rungs_falls_back_to_the_action(): + steps = pe.normalize_steps(_policy([{"action": "nope"}], action="monitor")) + assert steps == [{"action": "monitor", "after_secs": 0}] + + +def test_ladder_length_is_capped(): + steps = pe.normalize_steps(_policy( + [{"action": "alert", "after_secs": 1}] * 40)) + assert len(steps) == pe.MAX_LADDER_STEPS + + +def test_absurd_delays_are_clamped(): + """A typo of 300000 for 300 would otherwise park a ladder for 3 days.""" + steps = pe.normalize_steps(_policy([ + {"action": "pause", "after_secs": 0}, + {"action": "kill", "after_secs": 999999999}, + ])) + assert steps[1]["after_secs"] == pe.MAX_STEP_DELAY_SECS + + +def test_steps_stored_as_json_text_are_understood(): + """The store round-trips steps as a JSON column; both forms must behave + identically or a policy would act differently after a restart.""" + import json + steps = pe.normalize_steps(_policy(json.dumps(_LADDER))) + assert [s["action"] for s in steps] == ["pause", "alert", "kill"] + + +# ── ladder timing ───────────────────────────────────────────────────────── +def test_first_rung_fires_immediately(): + d = pe.evaluate([_incident()], [_policy(_LADDER)], _FACTS, now=1000)[0] + assert d["action"] == "pause" and d["step_index"] == 0 + assert d["step_count"] == 3 and d["is_final_step"] is False + assert d["next_action"] == "alert" and d["next_after_secs"] == 60 + + +def test_next_rung_waits_for_its_delay(): + args = ([_incident()], [_policy(_LADDER)], _FACTS) + assert pe.evaluate(*args, ladder_state=_state(0, 1000), now=1030) == [] + assert pe.evaluate(*args, ladder_state=_state(0, 1000), now=1059) == [] + d = pe.evaluate(*args, ladder_state=_state(0, 1000), now=1061)[0] + assert d["action"] == "alert" and d["step_index"] == 1 + + +def test_delay_is_measured_from_the_previous_rung_not_the_incident(): + """'kill 5 minutes after the pause', not '5 minutes after it got stuck'.""" + args = ([_incident()], [_policy(_LADDER)], _FACTS) + # Rung 1 fired late (t=5000); rung 2 is due at 5300, not at 1000+300. + assert pe.evaluate(*args, ladder_state=_state(1, 5000), now=5299) == [] + d = pe.evaluate(*args, ladder_state=_state(1, 5000), now=5301)[0] + assert d["action"] == "kill" + + +def test_final_rung_is_marked_and_the_ladder_then_stops(): + args = ([_incident()], [_policy(_LADDER)], _FACTS) + d = pe.evaluate(*args, ladder_state=_state(1, 1000), now=9999)[0] + assert d["action"] == "kill" and d["is_final_step"] is True + assert d["next_action"] == "" + # Exhausted: nothing more, however long we wait. + assert pe.evaluate(*args, ladder_state=_state(2, 1000), now=10**9) == [] + + +def test_a_recovered_session_stops_the_ladder(): + """'kill if STILL stuck' must mean still stuck: no incident, no rung.""" + assert pe.evaluate([], [_policy(_LADDER)], _FACTS, + ladder_state=_state(0, 1000), now=10**9) == [] + + +def test_unknown_fire_time_delays_rather_than_fires(): + """The safe way to be wrong when the next rung might be a kill.""" + args = ([_incident()], [_policy(_LADDER)], _FACTS) + state = {"s1": {"p1": {"last_step": 0, "last_fired_at": None}}} + assert pe.evaluate(*args, ladder_state=state, now=10**9) == [] + + +def test_reason_names_the_rung_and_what_comes_next(): + d = pe.evaluate([_incident()], [_policy(_LADDER)], _FACTS, now=1000)[0] + assert "[step 1/3]" in d["reason"] + assert "then alert in 1m if still matching" in d["reason"] + + +def test_single_action_policy_reason_is_unchanged(): + d = pe.evaluate([_incident()], [_policy(action="kill")], _FACTS, now=1000)[0] + assert "step" not in d["reason"] + assert d["step_count"] == 1 and d["is_final_step"] is True + + +def test_still_one_decision_per_session_across_ladders(): + """Two ladders on one session must not produce two signals in a tick.""" + ds = pe.evaluate([_incident()], + [_policy(_LADDER, policy_id="p1"), + _policy([{"action": "kill", "after_secs": 0}], + policy_id="p2")], + _FACTS, now=1000) + assert len(ds) == 1 + assert ds[0]["action"] == "kill" # strongest wins, as before + + +# ── durable per-rung latch, against a real DuckDB store ────────────────── +@pytest.fixture +def store(tmp_path, monkeypatch): + """A throwaway DuckDB store. + + CLAWMETRY_LOCAL_STORE_PATH is set BEFORE the reload so the module-level + default path is rebound to tmp_path — the store must never touch the real + ~/.clawmetry database. + """ + pytest.importorskip("duckdb") + monkeypatch.setenv("CLAWMETRY_LOCAL_STORE_PATH", str(tmp_path / "cm.duckdb")) + import importlib + + import clawmetry.local_store as ls + importlib.reload(ls) + st = ls.LocalStore() + assert str(tmp_path) in str(getattr(ls, "DB_PATH", tmp_path)), \ + "refusing to run against the real store" + try: + yield st + finally: + try: + st.close() + except Exception: + pass + # Restore the module to the real default path. Leaving it bound to a + # deleted tmp_path would make a later test in the same session fail + # for a reason that has nothing to do with it. + monkeypatch.undo() + importlib.reload(ls) + + +def test_each_rung_latches_independently(store): + store.record_policy_action("s1", "p1", action="pause", step_index=0) + assert store.policy_already_fired("s1", "p1", 0) is True + # The whole point: rung 1 is NOT latched by rung 0's row. + assert store.policy_already_fired("s1", "p1", 1) is False + store.record_policy_action("s1", "p1", action="kill", step_index=1) + assert store.policy_already_fired("s1", "p1", 1) is True + + +def test_re_recording_a_rung_updates_it_rather_than_duplicating(store): + store.record_policy_action("s1", "p1", action="pause", step_index=0, + result_detail="pending") + store.record_policy_action("s1", "p1", action="pause", step_index=0, + result_ok=True, result_detail="paused") + rows = [r for r in store.query_policy_actions(limit=50) + if r["session_id"] == "s1"] + assert len(rows) == 1 + assert rows[0]["result_detail"] == "paused" and rows[0]["result_ok"] is True + + +def test_ladder_state_survives_a_restart(store): + """The reason the latch is durable: a restart must resume at the rung it + reached, not replay a ladder that ends in kill.""" + store.record_policy_action("s1", "p1", action="pause", step_index=0) + store.record_policy_action("s1", "p1", action="alert", step_index=1) + state = store.query_policy_ladder_state() + assert state["s1"]["p1"]["last_step"] == 1 + assert state["s1"]["p1"]["last_fired_at"] > 0 + # Fed back to the engine, it resumes at rung 2 rather than rung 0. + d = pe.evaluate([_incident()], [_policy(_LADDER)], _FACTS, + ladder_state=state, now=state["s1"]["p1"]["last_fired_at"] + 400)[0] + assert d["action"] == "kill" and d["step_index"] == 2 + + +def test_ladder_state_is_epoch_seconds_not_milliseconds(store): + """A unit mismatch here would make every delay look 1000x satisfied.""" + import time + store.record_policy_action("s1", "p1", action="pause", step_index=0) + fired = store.query_policy_ladder_state()["s1"]["p1"]["last_fired_at"] + assert abs(fired - time.time()) < 60 + + +def test_policy_round_trips_its_ladder(store): + store.upsert_session_policy(_policy(_LADDER)) + row = [r for r in store.query_session_policies() + if r["policy_id"] == "p1"][0] + assert [s["action"] for s in row["steps"]] == ["pause", "alert", "kill"] + assert row["steps"][2]["after_secs"] == 300 + + +def test_a_plain_policy_reads_back_with_no_ladder(store): + store.upsert_session_policy(_policy(action="kill")) + row = [r for r in store.query_session_policies() + if r["policy_id"] == "p1"][0] + assert row["steps"] == [] + # …and the engine still treats it as a one-rung ladder. + assert pe.normalize_steps(row) == [{"action": "kill", "after_secs": 0}] + + +def test_audit_rows_carry_the_rung(store): + store.record_policy_action("s1", "p1", action="kill", step_index=2) + row = [r for r in store.query_policy_actions(limit=10) + if r["session_id"] == "s1"][0] + assert row["step_index"] == 2 + + +# ── the migration from the two-column latch ────────────────────────────── +def test_migration_widens_the_latch_and_keeps_existing_rows(tmp_path): + duckdb = pytest.importorskip("duckdb") + path = str(tmp_path / "old.duckdb") + conn = duckdb.connect(path) + conn.execute(""" + CREATE TABLE policy_actions ( + session_id VARCHAR NOT NULL, policy_id VARCHAR NOT NULL, + runtime VARCHAR, action VARCHAR, kind VARCHAR, reason VARCHAR, + evidence BLOB, enforced BOOLEAN DEFAULT FALSE, + result_ok BOOLEAN DEFAULT FALSE, result_detail VARCHAR, + created_at BIGINT NOT NULL, + PRIMARY KEY (session_id, policy_id)) + """) + conn.execute("INSERT INTO policy_actions (session_id, policy_id, action, " + "created_at) VALUES ('old-s', 'old-p', 'kill', 1000)") + conn.close() + + from clawmetry.local_store import _migrate_policy_actions_ladder + conn = duckdb.connect(path) + _migrate_policy_actions_ladder(conn) + cols = {r[1] for r in conn.execute("PRAGMA table_info('policy_actions')").fetchall()} + assert "step_index" in cols + # The historical decision is preserved, at rung 0 where it belongs. + row = conn.execute("SELECT session_id, policy_id, step_index, action " + "FROM policy_actions").fetchone() + assert row == ("old-s", "old-p", 0, "kill") + # And rung 1 is now insertable without colliding with it. + conn.execute("INSERT INTO policy_actions (session_id, policy_id, " + "step_index, action, created_at) " + "VALUES ('old-s', 'old-p', 1, 'kill', 2000)") + assert conn.execute("SELECT COUNT(*) FROM policy_actions").fetchone()[0] == 2 + + # Idempotent: a second pass is a no-op, not a data-losing rebuild. + _migrate_policy_actions_ladder(conn) + assert conn.execute("SELECT COUNT(*) FROM policy_actions").fetchone()[0] == 2 + conn.close() diff --git a/tests/test_guard_policies_api_ladder.py b/tests/test_guard_policies_api_ladder.py new file mode 100644 index 0000000000..3bee9a309d --- /dev/null +++ b/tests/test_guard_policies_api_ladder.py @@ -0,0 +1,119 @@ +"""The /api/guard/policies contract for escalation ladders. + +A ladder authored in the UI must come back exactly as the engine will run it, +and a malformed rung must be REFUSED rather than silently dropped: the store +normalizes by dropping unusable rungs (right for reading old rows), which +would otherwise hand an author a ladder quietly missing a step. +""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import pytest # noqa: E402 +from flask import Flask # noqa: E402 + +from routes.guard import bp_guard # noqa: E402 + + +@pytest.fixture +def client(monkeypatch): + """Flask test client over an in-memory policy store.""" + saved = {} + + def _write(method, **kw): + if method == "upsert_session_policy": + p = kw["policy"] + saved[p["policy_id"]] = p + elif method == "delete_session_policy": + saved.pop(kw.get("policy_id"), None) + + def _call(method, **kw): + if method == "query_session_policies": + return list(saved.values()) + return None + + import routes.guard as g + monkeypatch.setattr(g, "_ls_write", _write) + monkeypatch.setattr(g, "_ls_call", _call) + + app = Flask(__name__) + app.register_blueprint(bp_guard) + with app.test_client() as c: + yield c, saved + + +def _post(client, **body): + return client.post("/api/guard/policies", json=body) + + +def test_a_ladder_round_trips(client): + c, saved = client + r = _post(c, name="stuck", action="pause", steps=[ + {"action": "pause", "after_secs": 0}, + {"action": "kill", "after_secs": 300}, + ]) + assert r.status_code == 200 and r.json["ok"] is True + steps = r.json["policy"]["steps"] + assert [s["action"] for s in steps] == ["pause", "kill"] + assert steps[1]["after_secs"] == 300 + + +def test_an_unknown_step_action_is_refused_not_dropped(client): + c, _ = client + r = _post(c, action="pause", steps=[ + {"action": "pause", "after_secs": 0}, + {"action": "terminate", "after_secs": 60}, + ]) + assert r.status_code == 400 + assert "step 2" in r.json["error"] + + +def test_a_negative_delay_is_refused(client): + c, _ = client + r = _post(c, action="pause", steps=[ + {"action": "pause", "after_secs": 0}, + {"action": "kill", "after_secs": -5}, + ]) + assert r.status_code == 400 and "after_secs" in r.json["error"] + + +def test_a_non_list_ladder_is_refused(client): + c, _ = client + r = _post(c, action="pause", steps={"action": "kill"}) + assert r.status_code == 400 + + +def test_an_over_long_ladder_is_refused(client): + from clawmetry.policy_engine import MAX_LADDER_STEPS + c, _ = client + r = _post(c, action="alert", + steps=[{"action": "alert", "after_secs": 1}] * (MAX_LADDER_STEPS + 1)) + assert r.status_code == 400 and str(MAX_LADDER_STEPS) in r.json["error"] + + +def test_step_zero_delay_is_normalized_in_the_echo(client): + """What the operator is shown must equal what the engine will run.""" + c, _ = client + r = _post(c, action="pause", + steps=[{"action": "pause", "after_secs": 900}, + {"action": "kill", "after_secs": 60}]) + assert r.json["policy"]["steps"][0]["after_secs"] == 0 + + +def test_a_plain_policy_still_works(client): + c, _ = client + r = _post(c, name="simple", action="kill") + assert r.status_code == 200 and r.json["ok"] is True + assert r.json["policy"]["steps"] == [{"action": "kill", "after_secs": 0}] + + +def test_get_exposes_the_ladder_limits(client): + c, _ = client + _post(c, action="monitor") + r = c.get("/api/guard/policies") + assert r.status_code == 200 + assert r.json["max_ladder_steps"] >= 2 + assert "kill" in r.json["actions"] + # Every policy reads back with a ladder, plain ones included. + assert all(p["steps"] for p in r.json["policies"]) diff --git a/tests/test_guard_policy_enforcement.py b/tests/test_guard_policy_enforcement.py new file mode 100644 index 0000000000..156d324467 --- /dev/null +++ b/tests/test_guard_policy_enforcement.py @@ -0,0 +1,309 @@ +"""Daemon-side Guard enforcement: the three safety locks and the latch. + +These tests drive ``sync._apply_guard_policies`` with a fake store and a +stubbed actuator, so nothing is ever signalled for real. What they assert is +the thing that matters: a policy must NOT reach a live process unless every +lock is open, and must never fire twice for the same session. +""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import pytest # noqa: E402 + +from clawmetry import sync as _sync # noqa: E402 + + +class FakeStore: + """Minimal stand-in for LocalStore's Guard surface.""" + + def __init__(self, policies): + self._policies = policies + self.recorded = [] + self.fired = set() + + def query_session_policies(self, enabled_only=False): + return [p for p in self._policies + if p.get("enabled", True) or not enabled_only] + + def policy_already_fired(self, session_id, policy_id): + return (session_id, policy_id) in self.fired + + def record_policy_action(self, session_id, policy_id, **kw): + self.fired.add((session_id, policy_id)) + self.recorded.append({"session_id": session_id, + "policy_id": policy_id, **kw}) + + +def _incident(session_id="s1", kind="no_progress", runtime="claude_code"): + return {"kind": kind, "session_id": session_id, "runtime": runtime, + "severity": "warning", "title": "agent not advancing", + "evidence": {"total_tool_calls": 40}} + + +def _policy(action="kill", policy_id="p1"): + return {"policy_id": policy_id, "enabled": True, "scope_runtime": "", + "scope_agent_id": "", "trigger_kind": "", "min_severity": "info", + "min_repeat": 0, "min_duration_s": 0, "min_spend_usd": 0, + "action": action} + + +_FACTS = {"s1": {"cost_usd": 9.0, "bad_for_seconds": 600, + "runtime": "claude_code", "cwd": "/tmp/x", "agent_id": "main"}} + + +@pytest.fixture(autouse=True) +def _stub_actuator(monkeypatch): + """Record actuator calls instead of signalling anything.""" + calls = [] + + def fake_actuate(runtime, session_id, cwd, action): + calls.append((runtime, session_id, cwd, action)) + return {"ok": True, "detail": "stub"} + + monkeypatch.setattr(_sync, "_guard_actuate", fake_actuate) + monkeypatch.delenv("CLAWMETRY_POLICY_ENFORCE", raising=False) + monkeypatch.delenv("CLAWMETRY_GUARD_POLICIES", raising=False) + return calls + + +def _entitled(monkeypatch, value=True): + monkeypatch.setattr(_sync, "_guard_enforcement_allowed", lambda: value) + + +# ── lock 1: the action itself ────────────────────────────────────────────── + +def test_monitor_never_actuates_even_fully_enabled(monkeypatch, _stub_actuator): + monkeypatch.setenv("CLAWMETRY_POLICY_ENFORCE", "1") + _entitled(monkeypatch) + store = FakeStore([_policy(action="monitor")]) + n = _sync._apply_guard_policies(store, {}, [_incident()], _FACTS) + assert n == 1, "the decision is still recorded" + assert _stub_actuator == [], "monitor must never signal a process" + assert store.recorded[0]["enforced"] is False + + +# ── lock 2: the enforce env flag ─────────────────────────────────────────── + +def test_dry_run_by_default_records_but_does_not_act(monkeypatch, _stub_actuator): + _entitled(monkeypatch) + store = FakeStore([_policy(action="kill")]) + n = _sync._apply_guard_policies(store, {}, [_incident()], _FACTS) + assert n == 1 + assert _stub_actuator == [], "no enforce flag: must not kill" + rec = store.recorded[0] + assert rec["enforced"] is False + assert "DRY RUN" in rec["result_detail"] + assert "CLAWMETRY_POLICY_ENFORCE" in rec["result_detail"] + + +def test_guard_can_be_disabled_entirely(monkeypatch, _stub_actuator): + monkeypatch.setenv("CLAWMETRY_GUARD_POLICIES", "0") + monkeypatch.setenv("CLAWMETRY_POLICY_ENFORCE", "1") + _entitled(monkeypatch) + store = FakeStore([_policy(action="kill")]) + assert _sync._apply_guard_policies(store, {}, [_incident()], _FACTS) == 0 + assert _stub_actuator == [] + assert store.recorded == [] + + +# ── lock 3: entitlement ──────────────────────────────────────────────────── + +def test_unentitled_node_records_but_does_not_act(monkeypatch, _stub_actuator): + monkeypatch.setenv("CLAWMETRY_POLICY_ENFORCE", "1") + _entitled(monkeypatch, False) + store = FakeStore([_policy(action="kill")]) + _sync._apply_guard_policies(store, {}, [_incident()], _FACTS) + assert _stub_actuator == [] + assert "not enabled on this plan" in store.recorded[0]["result_detail"] + + +def test_entitlement_check_fails_closed(monkeypatch): + """A resolver explosion must read as 'not allowed', never 'allowed'.""" + import clawmetry.entitlements as _ent + + def boom(): + raise RuntimeError("resolver down") + + monkeypatch.setattr(_ent, "get_entitlement", boom) + assert _sync._guard_enforcement_allowed() is False + + +# ── all locks open: it actually acts ─────────────────────────────────────── + +def test_all_locks_open_actuates(monkeypatch, _stub_actuator): + monkeypatch.setenv("CLAWMETRY_POLICY_ENFORCE", "1") + _entitled(monkeypatch) + store = FakeStore([_policy(action="kill")]) + _sync._apply_guard_policies(store, {}, [_incident()], _FACTS) + assert _stub_actuator == [("claude_code", "s1", "/tmp/x", "kill")] + final = store.recorded[-1] + assert final["enforced"] is True and final["result_ok"] is True + + +# ── the latch ────────────────────────────────────────────────────────────── + +def test_latch_prevents_double_fire(monkeypatch, _stub_actuator): + monkeypatch.setenv("CLAWMETRY_POLICY_ENFORCE", "1") + _entitled(monkeypatch) + store = FakeStore([_policy(action="kill")]) + _sync._apply_guard_policies(store, {}, [_incident()], _FACTS) + _sync._apply_guard_policies(store, {}, [_incident()], _FACTS) + _sync._apply_guard_policies(store, {}, [_incident()], _FACTS) + assert len(_stub_actuator) == 1, "a policy must fire at most once per session" + + +def test_latch_read_failure_declines_to_act(monkeypatch, _stub_actuator): + monkeypatch.setenv("CLAWMETRY_POLICY_ENFORCE", "1") + _entitled(monkeypatch) + store = FakeStore([_policy(action="kill")]) + + def boom(*a, **k): + raise RuntimeError("duckdb gone") + + store.policy_already_fired = boom + assert _sync._apply_guard_policies(store, {}, [_incident()], _FACTS) == 0 + assert _stub_actuator == [] + + +# ── never break ingest ───────────────────────────────────────────────────── + +def test_store_failure_never_raises(monkeypatch, _stub_actuator): + class Broken: + def query_session_policies(self, enabled_only=False): + raise RuntimeError("boom") + + assert _sync._apply_guard_policies(Broken(), {}, [_incident()], _FACTS) == 0 + + +def test_no_incidents_is_a_noop(_stub_actuator): + store = FakeStore([_policy(action="kill")]) + assert _sync._apply_guard_policies(store, {}, [], _FACTS) == 0 + assert store.recorded == [] + + +def test_one_session_one_signal_even_with_many_policies(monkeypatch, _stub_actuator): + monkeypatch.setenv("CLAWMETRY_POLICY_ENFORCE", "1") + _entitled(monkeypatch) + store = FakeStore([_policy(action="pause", policy_id="a"), + _policy(action="kill", policy_id="b"), + _policy(action="stop", policy_id="c")]) + _sync._apply_guard_policies(store, {}, [_incident()], _FACTS) + assert len(_stub_actuator) == 1 + assert _stub_actuator[0][3] == "kill", "strongest action wins" + + +# ── escalation ladders must not weaken any lock ────────────────────────── +_LADDER = [{"action": "pause", "after_secs": 0}, + {"action": "kill", "after_secs": 300}] + + +def _ladder_policy(policy_id="p1"): + p = _policy(action="pause", policy_id=policy_id) + p["steps"] = _LADDER + return p + + +class LadderStore(FakeStore): + """FakeStore that tracks rungs, like the real per-step latch.""" + + def __init__(self, policies): + super().__init__(policies) + self.rungs = {} + self.now = 1000.0 + + def query_policy_ladder_state(self, limit=2000): + out = {} + for (sid, pid, step), ts in self.rungs.items(): + cur = out.setdefault(sid, {}).get(pid) + if cur is None or step > cur["last_step"]: + out.setdefault(sid, {})[pid] = {"last_step": step, + "last_fired_at": ts} + return out + + def policy_already_fired(self, session_id, policy_id, step_index=0): + return (session_id, policy_id, step_index) in self.rungs + + def record_policy_action(self, session_id, policy_id, step_index=0, **kw): + self.rungs.setdefault((session_id, policy_id, step_index), self.now) + self.recorded.append({"session_id": session_id, + "policy_id": policy_id, + "step_index": step_index, **kw}) + + +def _tick(store, monkeypatch, at): + """Run one daemon tick with the engine's clock pinned to ``at``.""" + from clawmetry import policy_engine as pe + store.now = at + real = pe.evaluate + monkeypatch.setattr( + pe, "evaluate", + lambda i, p, f, ladder_state=None, now=None: + real(i, p, f, ladder_state=ladder_state, now=at)) + _sync._apply_guard_policies(store, {}, [_incident()], _FACTS) + monkeypatch.setattr(pe, "evaluate", real) + + +def test_ladder_kill_rung_is_a_dry_run_without_the_enforce_flag( + monkeypatch, _stub_actuator): + """A ladder must not become a way to skip CLAWMETRY_POLICY_ENFORCE.""" + monkeypatch.delenv("CLAWMETRY_POLICY_ENFORCE", raising=False) + monkeypatch.setattr(_sync, "_guard_enforcement_allowed", lambda: True) + store = LadderStore([_ladder_policy()]) + _tick(store, monkeypatch, 1000) + _tick(store, monkeypatch, 1400) + assert _stub_actuator == [], "no rung may signal with enforcement off" + # Both rungs are still RECORDED, which is what makes dry run honest. + assert sorted(r["step_index"] for r in store.recorded) == [0, 1] + assert all("DRY RUN" in r["result_detail"] for r in store.recorded) + + +def test_ladder_kill_rung_respects_the_entitlement_lock( + monkeypatch, _stub_actuator): + monkeypatch.setenv("CLAWMETRY_POLICY_ENFORCE", "1") + monkeypatch.setattr(_sync, "_guard_enforcement_allowed", lambda: False) + store = LadderStore([_ladder_policy()]) + _tick(store, monkeypatch, 1000) + _tick(store, monkeypatch, 1400) + assert _stub_actuator == [] + + +def test_ladder_climbs_when_every_lock_is_open(monkeypatch, _stub_actuator): + monkeypatch.setenv("CLAWMETRY_POLICY_ENFORCE", "1") + monkeypatch.setattr(_sync, "_guard_enforcement_allowed", lambda: True) + store = LadderStore([_ladder_policy()]) + _tick(store, monkeypatch, 1000) + assert [c[3] for c in _stub_actuator] == ["pause"] + _tick(store, monkeypatch, 1100) # too early for the kill rung + assert [c[3] for c in _stub_actuator] == ["pause"] + _tick(store, monkeypatch, 1400) + assert [c[3] for c in _stub_actuator] == ["pause", "kill"] + _tick(store, monkeypatch, 9999) # exhausted: never again + assert [c[3] for c in _stub_actuator] == ["pause", "kill"] + + +def test_a_prelader_store_refuses_to_climb_rather_than_replaying( + monkeypatch, _stub_actuator): + """Version skew: a daemon whose store predates ladders can only latch per + (session, policy). It must stop at rung 0, not re-fire it forever.""" + monkeypatch.setenv("CLAWMETRY_POLICY_ENFORCE", "1") + monkeypatch.setattr(_sync, "_guard_enforcement_allowed", lambda: True) + + class OldStore(LadderStore): + def policy_already_fired(self, session_id, policy_id): # 2-arg only + return any(k[0] == session_id and k[1] == policy_id + for k in self.rungs) + + def record_policy_action(self, session_id, policy_id, **kw): + kw.pop("step_index", None) + self.rungs.setdefault((session_id, policy_id, 0), self.now) + self.recorded.append({"session_id": session_id, + "policy_id": policy_id, **kw}) + + store = OldStore([_ladder_policy()]) + _tick(store, monkeypatch, 1000) + _tick(store, monkeypatch, 1400) + _tick(store, monkeypatch, 5000) + assert [c[3] for c in _stub_actuator] == ["pause"], \ + "an old store must not re-fire rung 0 nor climb" diff --git a/tests/test_policy_engine.py b/tests/test_policy_engine.py new file mode 100644 index 0000000000..3066ba17ee --- /dev/null +++ b/tests/test_policy_engine.py @@ -0,0 +1,181 @@ +"""Unit tests for the pure Guard policy evaluator. + +No daemon, no DuckDB, no live agent — ``policy_engine.evaluate`` is a pure +function, so every matching rule and every safety invariant is testable here. +""" +import sys +import os + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from clawmetry import policy_engine as pe # noqa: E402 + + +def _incident(kind="no_progress", session_id="s1", runtime="claude_code", + severity="warning", count=25, title="agent not advancing"): + return { + "kind": kind, + "session_id": session_id, + "runtime": runtime, + "severity": severity, + "title": title, + "detail": "", + "evidence": {"total_tool_calls": count}, + "first_bad_step": 3, + } + + +def _policy(policy_id="p1", action="pause", **kw): + base = { + "policy_id": policy_id, + "enabled": True, + "scope_runtime": "", + "scope_agent_id": "", + "trigger_kind": "", + "min_severity": "info", + "min_repeat": 0, + "min_duration_s": 0, + "min_spend_usd": 0.0, + "action": action, + } + base.update(kw) + return base + + +# ── basic matching ───────────────────────────────────────────────────────── + +def test_no_policies_means_no_decisions(): + assert pe.evaluate([_incident()], []) == [] + + +def test_bare_policy_matches_any_incident(): + out = pe.evaluate([_incident()], [_policy()]) + assert len(out) == 1 + assert out[0]["action"] == "pause" + assert out[0]["session_id"] == "s1" + assert out[0]["kind"] == "no_progress" + + +def test_disabled_policy_never_fires(): + assert pe.evaluate([_incident()], [_policy(enabled=False)]) == [] + + +def test_trigger_kind_filters(): + pol = _policy(trigger_kind="stuck_loop") + assert pe.evaluate([_incident(kind="no_progress")], [pol]) == [] + assert len(pe.evaluate([_incident(kind="stuck_loop")], [pol])) == 1 + + +def test_runtime_scope_filters_and_is_case_insensitive(): + pol = _policy(scope_runtime="Claude_Code") + assert len(pe.evaluate([_incident(runtime="claude_code")], [pol])) == 1 + assert pe.evaluate([_incident(runtime="codex")], [pol]) == [] + + +def test_unknown_action_is_refused_not_guessed(): + assert pe.evaluate([_incident()], [_policy(action="delete_everything")]) == [] + + +# ── thresholds ───────────────────────────────────────────────────────────── + +def test_min_repeat_threshold(): + pol = _policy(min_repeat=30) + assert pe.evaluate([_incident(count=25)], [pol]) == [] + assert len(pe.evaluate([_incident(count=30)], [pol])) == 1 + + +def test_min_severity_threshold(): + pol = _policy(min_severity="warning") + assert pe.evaluate([_incident(severity="info")], [pol]) == [] + assert len(pe.evaluate([_incident(severity="warning")], [pol])) == 1 + + +def test_spend_and_duration_thresholds_are_anded(): + pol = _policy(min_duration_s=300, min_spend_usd=5.0) + facts = {"s1": {"bad_for_seconds": 600, "cost_usd": 4.0}} + assert pe.evaluate([_incident()], [pol], facts) == [] # spend too low + facts = {"s1": {"bad_for_seconds": 60, "cost_usd": 9.0}} + assert pe.evaluate([_incident()], [pol], facts) == [] # too soon + facts = {"s1": {"bad_for_seconds": 600, "cost_usd": 9.0}} + assert len(pe.evaluate([_incident()], [pol], facts)) == 1 + + +def test_missing_facts_do_not_accidentally_satisfy_thresholds(): + """A session we have no facts for must NOT match a spend threshold.""" + pol = _policy(min_spend_usd=5.0) + assert pe.evaluate([_incident()], [pol], {}) == [] + + +def test_zeroed_thresholds_never_block(): + facts = {"s1": {"bad_for_seconds": 0, "cost_usd": 0}} + assert len(pe.evaluate([_incident()], [_policy()], facts)) == 1 + + +# ── safety invariants ────────────────────────────────────────────────────── + +def test_at_most_one_decision_per_session_strongest_wins(): + policies = [_policy("p1", action="alert"), _policy("p2", action="kill"), + _policy("p3", action="pause")] + out = pe.evaluate([_incident()], policies) + assert len(out) == 1, "must never emit two signals for one session" + assert out[0]["action"] == "kill" + assert out[0]["policy_id"] == "p2" + + +def test_tie_break_is_deterministic(): + policies = [_policy("zzz", action="pause"), _policy("aaa", action="pause")] + first = pe.evaluate([_incident()], policies) + second = pe.evaluate([_incident()], list(reversed(policies))) + assert first[0]["policy_id"] == "aaa" + assert first == second + + +def test_monitor_returns_a_decision_so_dry_run_is_visible(): + out = pe.evaluate([_incident()], [_policy(action="monitor")]) + assert len(out) == 1 + assert out[0]["action"] == "monitor" + assert not pe.is_actuating("monitor") + assert "would monitor" in out[0]["reason"] + + +def test_actuating_classification(): + assert pe.is_actuating("pause") and pe.is_actuating("stop") + assert pe.is_actuating("kill") + assert not pe.is_actuating("alert") + assert not pe.is_actuating("") + + +def test_incident_without_session_id_is_skipped(): + bad = _incident() + bad["session_id"] = "" + assert pe.evaluate([bad], [_policy()]) == [] + + +def test_malformed_rows_never_raise(): + assert pe.evaluate([None, "junk", {}], [None, "junk", _policy()]) == [] + + +def test_multiple_sessions_each_get_a_decision(): + incidents = [_incident(session_id="s1"), _incident(session_id="s2")] + out = pe.evaluate(incidents, [_policy()]) + assert {d["session_id"] for d in out} == {"s1", "s2"} + + +def test_output_is_sorted_strongest_first(): + incidents = [_incident(session_id="s1"), _incident(session_id="s2")] + policies = [_policy("p1", action="alert", scope_runtime=""), + _policy("p2", action="kill", trigger_kind="stuck_loop")] + incidents.append(_incident(session_id="s3", kind="stuck_loop")) + out = pe.evaluate(incidents, policies) + assert out[0]["action"] == "kill" + + +def test_evidence_records_the_numbers_that_matched(): + pol = _policy(min_repeat=10, min_spend_usd=1.0) + facts = {"s1": {"bad_for_seconds": 420, "cost_usd": 3.5}} + out = pe.evaluate([_incident(count=38)], [pol], facts) + ev = out[0]["evidence"] + assert ev["count"] == 38 + assert ev["cost_usd"] == 3.5 + assert ev["thresholds"]["min_repeat"] == 10 + assert "38 events" in out[0]["reason"] diff --git a/tests/test_process_control_windows.py b/tests/test_process_control_windows.py new file mode 100644 index 0000000000..25e0df142a --- /dev/null +++ b/tests/test_process_control_windows.py @@ -0,0 +1,168 @@ +"""Windows control path — dispatch, ordering and honesty. + +The Win32 calls themselves can only be exercised on Windows, but everything +around them (which branch runs, in what order, what the result says) is +platform-independent and is what actually regressed before: the actuators +returned ``unsupported_platform`` on every Windows node, so every Guard button +was inert there. + +These tests fake the platform flags rather than skipping off-Windows, so the +Windows path is covered by the macOS and Linux CI legs too — a suite that only +runs on the Windows leg is how the no-op survived in the first place. +""" +import clawmetry.process_control as pc +import pytest + + +@pytest.fixture +def win(monkeypatch): + """Pretend we are on Windows, with every Win32 primitive stubbed.""" + monkeypatch.setattr(pc, "_IS_WINDOWS", True) + monkeypatch.setattr(pc, "_IS_MACOS", False) + monkeypatch.setattr(pc, "_IS_LINUX", False) + monkeypatch.setattr(pc, "_POSIX", False) + monkeypatch.setattr(pc, "_CONTROLLABLE_PLATFORM", True) + + calls = [] + monkeypatch.setattr(pc, "is_alive", lambda p: True) + # A three-deep tree: process_set is children-first, parent last. + monkeypatch.setattr(pc, "process_set", lambda p: [300, 200, int(p)]) + monkeypatch.setattr(pc, "_win_suspend", + lambda p: calls.append(("suspend", p)) or True) + monkeypatch.setattr(pc, "_win_resume", + lambda p: calls.append(("resume", p)) or True) + monkeypatch.setattr(pc, "_win_terminate", + lambda p: calls.append(("terminate", p)) or True) + monkeypatch.setattr(pc, "_win_taskkill", + lambda p, force=False, timeout=10.0: + calls.append(("taskkill", p, force)) or True) + monkeypatch.setattr(pc, "_win_ctrl_c", + lambda p, timeout=10.0: + (calls.append(("ctrl_c", p)), (True, "ctrl_c_sent_to_console"))[1]) + return calls + + +# ── the gap itself: these four used to be unconditional no-ops ──────────── +@pytest.mark.parametrize("fn,action", [ + (lambda: pc.pause(100), "pause"), + (lambda: pc.resume(100), "resume"), + (lambda: pc.stop_turn(100), "stop_turn"), + (lambda: pc.graceful_kill(100, grace_secs=0), "graceful_kill"), +]) +def test_actuators_are_not_unsupported_on_windows(win, fn, action): + res = fn() + assert res["detail"] != "unsupported_platform" + assert res["ok"] is True, res + + +def test_pause_suspends_children_before_parent(win): + res = pc.pause(100, "claude_code") + assert res["ok"] and res["detail"] == "paused" + assert win == [("suspend", 300), ("suspend", 200), ("suspend", 100)] + assert res["mechanism"] == "win32_nt_suspend_process" + + +def test_resume_wakes_parent_before_children(win): + pc.resume(100, "claude_code") + assert win == [("resume", 100), ("resume", 200), ("resume", 300)] + + +def test_partial_pause_is_reported_not_hidden(win, monkeypatch): + """A child we could not freeze keeps spending. Say so.""" + monkeypatch.setattr(pc, "_win_suspend", lambda p: p != 300) + res = pc.pause(100) + assert res["ok"] is True + assert res["failed"] == [300] + assert "could not be suspended" in res["detail"] + + +def test_stop_turn_declares_its_console_blast_radius(win): + res = pc.stop_turn(100, "codex") + assert res["ok"] and res["scope"] == "console" + assert res["mechanism"] == "win32_console_ctrl_c" + + +def test_stop_turn_reports_ctrl_c_failure_honestly(win, monkeypatch): + monkeypatch.setattr(pc, "_win_ctrl_c", + lambda p, timeout=10.0: (False, "attach_console_failed")) + res = pc.stop_turn(100) + assert res["ok"] is False and res["detail"] == "attach_console_failed" + + +def test_graceful_kill_resumes_before_the_graceful_pass(win): + """A suspended process cannot handle taskkill's close, so 'pause then + kill' (an escalation ladder) would otherwise always burn the full grace + window. Resume must come first.""" + pc.graceful_kill(100, grace_secs=0) + kinds = [c[0] for c in win] + assert kinds.index("resume") < kinds.index("taskkill") + + +def test_graceful_kill_escalates_to_terminate_when_still_alive(win): + res = pc.graceful_kill(100, grace_secs=0) + assert ("taskkill", 100, False) in win # graceful pass first + assert ("terminate", 300) in win # then the tree, leaves first + assert res["mechanism"] == "win32_terminate_process" + + +def test_graceful_kill_forces_taskkill_when_terminate_is_refused(win, monkeypatch): + """TerminateProcess can be refused for an elevated target; /F is the last + honest attempt rather than reporting a kill that did not happen.""" + monkeypatch.setattr(pc, "_win_terminate", lambda p: False) + pc.graceful_kill(100, grace_secs=0) + assert ("taskkill", 100, True) in win + + +def test_guarded_no_longer_refuses_the_platform(win, monkeypatch): + monkeypatch.setattr(pc, "resolve_session", + lambda rt, sid="", cwd="": {"ok": True, "pid": 100, + "runtime": rt, "cwd": cwd, + "recorded_start": None}) + monkeypatch.setattr(pc, "verify_pid", lambda pid, rec=None: (True, "verified")) + res = pc.pause_session("claude_code", "sess-1") + assert res["detail"] != "unsupported_platform" + assert res["ok"] is True + + +def test_dead_pid_still_refused_on_windows(win, monkeypatch): + monkeypatch.setattr(pc, "is_alive", lambda p: False) + assert pc.pause(100)["detail"] == "pid_not_alive" + assert pc.stop_turn(100)["detail"] == "pid_not_alive" + + +# ── honesty for platforms we genuinely do not support ───────────────────── +def test_unknown_platform_still_refuses(monkeypatch): + monkeypatch.setattr(pc, "_IS_WINDOWS", False) + monkeypatch.setattr(pc, "_POSIX", False) + monkeypatch.setattr(pc, "_CONTROLLABLE_PLATFORM", False) + for res in (pc.pause(1), pc.resume(1), pc.stop_turn(1), pc.graceful_kill(1)): + assert res["ok"] is False + assert res["detail"] == "unsupported_platform" + + +def test_platform_support_states_the_mechanism(monkeypatch): + monkeypatch.setattr(pc, "_POSIX", False) + monkeypatch.setattr(pc, "_IS_WINDOWS", True) + sup = pc.platform_support() + assert sup["controllable"] is True + assert sup["mechanism"] == "win32_native" + assert set(sup["actions"]) == {"pause", "resume", "stop", "kill"} + # The Ctrl+C caveat is a real behavioural difference; it must be surfaced. + assert "Ctrl+C" in sup["note"] + + +def test_platform_support_is_honest_about_an_unsupported_os(monkeypatch): + monkeypatch.setattr(pc, "_POSIX", False) + monkeypatch.setattr(pc, "_IS_WINDOWS", False) + sup = pc.platform_support() + assert sup["controllable"] is False and sup["actions"] == [] + assert sup["reason"] + + +# ── the primitives are inert, never raising, off Windows ────────────────── +def test_win_primitives_are_inert_off_windows(): + assert pc._win_kernel32() is None + assert pc._win_all_procs() == [] + assert pc._win_proc_start_epoch(1) is None + assert pc._win_ctrl_c(1) == (False, "not_windows") + assert pc._win_ntdll_call("NtSuspendProcess", 1) is False