diff --git a/frontend-dist/index.html b/frontend-dist/index.html index d111c269..490dc4fa 100644 --- a/frontend-dist/index.html +++ b/frontend-dist/index.html @@ -5,7 +5,7 @@ Pinky -- Personal AI Companion - + diff --git a/src/pinky_daemon/agent_registry.py b/src/pinky_daemon/agent_registry.py index f2dbd006..3d1514b2 100644 --- a/src/pinky_daemon/agent_registry.py +++ b/src/pinky_daemon/agent_registry.py @@ -25,6 +25,7 @@ import json import sqlite3 import sys +import threading import time from dataclasses import dataclass, field from pathlib import Path @@ -34,6 +35,159 @@ def _log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) +def _verify_effort_hook_source() -> str: + """Return the source for ``.claude/hook_verify_effort.py``. + + The hook compares the runtime ``$CLAUDE_EFFORT`` (Claude Code v2.1.133+) + against ``$PINKY_EXPECTED_EFFORT`` (injected by the daemon at session + start). On drift, it POSTs to ``/agents/{name}/effort-drift`` and, in + strict mode (``$PINKY_STRICT_EFFORT=1``), emits a block decision so + Claude Code refuses the tool call. + + No-ops silently when expected is empty/auto, when actual is unset (older + CLI), or when ``$PINKY_AGENT_NAME`` is missing. + """ + return '''\ +#!/usr/bin/env python3 +"""PinkyBot effort-drift verification hook (#429). + +Compares the runtime thinking effort surfaced by Claude Code (v2.1.133+) to +the expected value injected by the daemon. On mismatch, reports drift and +optionally blocks the tool call. + +Hooks must never crash the tool call — failures are swallowed. +""" +from __future__ import annotations + +import json +import os +import sys + +# Tools the hook MUST NOT block even in strict mode — these are how the +# agent self-remediates drift. Blocking set_thinking_effort would trap +# the agent: the only fix becomes unavailable. Match is substring against +# the tool name, so it covers raw `set_thinking_effort` and any MCP-qualified +# variant (e.g. `mcp__pinky-self__set_thinking_effort`). +REMEDIATION_TOOLS = ( + "set_thinking_effort", +) + +# Levels set_thinking_effort MCP tool accepts. Kept in sync with the tool's +# validator (pinky_self/server.py). If a future registry adds a level +# outside this set, suggesting `set_thinking_effort(expected)` would fail +# at the MCP layer — surface a clear "not self-remediable" reason instead +# of an unreachable suggestion. +SET_EFFORT_ACCEPTED = ("low", "medium", "high", "xhigh", "max", "auto") + + +def _post_drift(agent: str, expected: str, actual: str, tool_name: str, + strict: bool, daemon_url: str) -> None: + import urllib.request + + path = f"/agents/{agent}/effort-drift" + body = json.dumps({ + "expected": expected, + "actual": actual, + "tool_name": tool_name, + "strict": bool(strict), + }).encode() + req = urllib.request.Request( + f"{daemon_url}{path}", data=body, method="POST", + ) + req.add_header("Content-Type", "application/json") + req.add_header("x-pinky-agent", agent) + try: + urllib.request.urlopen(req, timeout=2) + except Exception: + pass + + +def _is_remediation_tool(tool_name: str) -> bool: + """True if tool_name is on the strict-mode allowlist.""" + if not tool_name: + return False + needle = tool_name.lower() + return any(t in needle for t in REMEDIATION_TOOLS) + + +def _remediation_suggestion(expected: str) -> str: + """Return a remediation call the agent can actually make. + + When expected is in the MCP tool's accepted set (the common case), + suggest the direct call. Otherwise be honest: tell the agent the + expected level isn't reachable from inside the session, so it knows + to escalate to the owner rather than spinning on a suggestion that + won't resolve the drift. + """ + if expected in SET_EFFORT_ACCEPTED: + return f"set_thinking_effort({expected!r})" + return ( + f"" + ) + + +def main() -> None: + actual = os.environ.get("CLAUDE_EFFORT", "").strip().lower() + expected = os.environ.get("PINKY_EXPECTED_EFFORT", "").strip().lower() + agent = os.environ.get("PINKY_AGENT_NAME", "").strip() + strict = os.environ.get("PINKY_STRICT_EFFORT", "").strip() == "1" + daemon_url = os.environ.get( + "PINKY_DAEMON_URL", "http://localhost:8888" + ).rstrip("/") + + # No-op cases — these are not drift events: + # - no expected configured + # - expected is "auto" (effort is intentionally adaptive) + # - actual is unset (older Claude Code without v2.1.133 effort.level) + # - no agent name (hook misconfigured) + if not expected or expected == "auto" or not actual or not agent: + return + + if actual == expected: + return # match — no action + + # Try to read the tool name from the JSON event piped to stdin, if any. + tool_name = "" + try: + stdin_data = sys.stdin.read() if not sys.stdin.isatty() else "" + if stdin_data: + payload = json.loads(stdin_data) + tool_name = ( + payload.get("tool_name") + or (payload.get("tool") or {}).get("name") + or "" + ) + except Exception: + pass + + # Always record the drift — even when we're about to let it through. + _post_drift(agent, expected, actual, tool_name, strict, daemon_url) + + # Strict path: emit a block decision EXCEPT when the tool being called + # is the very thing that can fix the drift. Blocking the remediation + # tool would trap the agent in an unbreakable strict-mode loop. + if strict and not _is_remediation_tool(tool_name): + print(json.dumps({ + "decision": "block", + "reason": ( + f"Effort drift detected: expected={expected} actual={actual}. " + f"Call {_remediation_suggestion(expected)} and retry the tool, " + "or contact your owner to relax strict_effort_enforcement." + ), + })) + + +if __name__ == "__main__": + try: + main() + except Exception: + # Hooks must never crash the tool call. + pass +''' + + def _cron_next_run(cron: str, timezone: str = "UTC") -> float | None: """Compute the next run timestamp for a cron expression using stdlib only. @@ -191,6 +345,10 @@ class Agent: provider_model: str = "" # model name override (e.g. "llama3.2"), empty = use agent.model provider_ref: str = "" # ID of a global provider from the providers table thinking_effort: str = "medium" # low, medium, high, xhigh, max — default thinking depth + # When True, the verify_effort CLI hook blocks tool calls if the runtime + # effort drifts from thinking_effort. Default False (warn-only): drift is + # surfaced to /agents/{name}/effort-drift + heartbeat but does not block. + strict_effort_enforcement: bool = False watchdog_config: dict = field(default_factory=dict) # Per-agent watchdog overrides (JSON blob) # watchdog_config schema: { # "enabled": true, # Enable/disable watchdog for this agent @@ -250,6 +408,7 @@ def to_dict(self) -> dict: "provider_model": self.provider_model, "provider_ref": self.provider_ref, "thinking_effort": self.thinking_effort, + "strict_effort_enforcement": self.strict_effort_enforcement, "watchdog_config": self.watchdog_config, "created_at": self.created_at, "updated_at": self.updated_at, @@ -452,6 +611,13 @@ def __init__(self, db_path: str = "data/agents.db") -> None: self._db = sqlite3.connect(db_path, check_same_thread=False) self._db.execute("PRAGMA journal_mode=WAL") self._db.execute("PRAGMA foreign_keys=ON") + # Guard read-modify-write sequences (e.g. peer_fleet_acl mutation) + # from concurrent admin-API requests. SQLite connection is shared + # across threads (check_same_thread=False) and Python's default + # isolation_level uses deferred BEGIN — without this lock, two + # admin requests can both read the same baseline ACL, append + # different entries, and one write loses. + self._rmw_lock = threading.RLock() self._init_tables() def _init_tables(self) -> None: @@ -673,6 +839,20 @@ def _init_tables(self) -> None: updated_at REAL NOT NULL DEFAULT 0 ); + CREATE TABLE IF NOT EXISTS effort_drift_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + agent_name TEXT NOT NULL, + session_id TEXT NOT NULL DEFAULT '', + expected TEXT NOT NULL, + actual TEXT NOT NULL, + tool_name TEXT NOT NULL DEFAULT '', + strict INTEGER NOT NULL DEFAULT 0, + timestamp REAL NOT NULL, + FOREIGN KEY (agent_name) REFERENCES agents(name) ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS idx_effort_drift_agent ON effort_drift_events(agent_name, timestamp DESC); + CREATE TABLE IF NOT EXISTS models ( id TEXT PRIMARY KEY, provider TEXT NOT NULL DEFAULT 'anthropic', @@ -730,9 +910,19 @@ def _migrate(self) -> None: ("provider_ref", "TEXT NOT NULL DEFAULT ''"), ("disallowed_tools", "TEXT NOT NULL DEFAULT '[]'"), ("thinking_effort", "TEXT NOT NULL DEFAULT 'medium'"), + # When 1, verify_effort CLI hook blocks tool calls on effort drift + # (vs. warn-only default of 0). See #429. + ("strict_effort_enforcement", "INTEGER NOT NULL DEFAULT 0"), ("watchdog_config", "TEXT NOT NULL DEFAULT '{}'"), ("last_seen_at", "REAL NOT NULL DEFAULT 0"), ("runtime", "TEXT NOT NULL DEFAULT 'claude_sdk'"), + # Ferry peer-fleet ACL — list of AgentCardSelector dicts + # (separate identity primitive from approved_users; default deny-all) + ("peer_fleet_acl", "TEXT NOT NULL DEFAULT '[]'"), + # Ferry outbound mesh allowlist — list of "agent@fleet" patterns + # gating which targets this agent may publish to via + # mesh_remote_send. Default-deny (empty list = no outbound). + ("mesh_outbound_allowlist", "TEXT NOT NULL DEFAULT '[]'"), ] for col, typedef in migrations: if col not in existing: @@ -831,6 +1021,24 @@ def _backfill_runtime_from_provider_url(self) -> None: # ── Workspace Init ───────────────────────────────────── + def ensure_workspace_hooks(self, agent_name: str) -> None: + """Re-run hook setup for an existing agent's workspace. + + Idempotent. Use to install new hooks (e.g. ``hook_verify_effort.py`` + from #429) on agents whose workspace pre-dates them, without nuking + any user customizations to existing scripts. + """ + agent = self.get(agent_name) + if not agent or not agent.working_dir: + return + work_dir = Path(agent.working_dir) + if not work_dir.exists(): + return + try: + self._setup_hooks(work_dir, agent_name) + except Exception as e: # pragma: no cover — defensive + _log(f"agent_registry: ensure_workspace_hooks({agent_name}) failed: {e}") + @staticmethod def _init_workspace(work_dir: Path, agent_name: str = "") -> None: """Create an agent workspace with default directory structure. @@ -857,10 +1065,13 @@ def _init_workspace(work_dir: Path, agent_name: str = "") -> None: @staticmethod def _setup_hooks(work_dir: Path, agent_name: str) -> None: - """Generate Claude Code hooks for agent working/idle status reporting. + """Generate Claude Code hooks for agent working/idle status reporting + and (since #429) effort-drift verification. - Creates .claude/ directory with hook_working.py, hook_idle.py, and - settings.json if they don't already exist. Won't overwrite custom configs. + Creates ``.claude/`` directory with hook scripts and settings.json. + Existing scripts are not overwritten; settings.json is idempotently + merged so the verify_effort hook can be added to agents whose + settings predate #429 without nuking their existing hooks. """ claude_dir = work_dir / ".claude" claude_dir.mkdir(exist_ok=True) @@ -896,6 +1107,7 @@ def _setup_hooks(work_dir: Path, agent_name: str) -> None: ''' working_path = claude_dir / "hook_working.py" idle_path = claude_dir / "hook_idle.py" + verify_effort_path = claude_dir / "hook_verify_effort.py" if not working_path.exists(): working_path.write_text( @@ -909,35 +1121,130 @@ def _setup_hooks(work_dir: Path, agent_name: str) -> None: ) _log(f"agent_registry: created hook_idle.py for {agent_name}") - settings_path = claude_dir / "settings.json" + # #429: verify_effort hook — compares $CLAUDE_EFFORT (v2.1.133+) to + # PINKY_EXPECTED_EFFORT (set by daemon at session start). On drift, + # posts to /agents/{name}/effort-drift; under strict mode also emits + # a block decision so Claude Code refuses the tool call. + # + # We ALWAYS rewrite this script (unlike hook_working / hook_idle + # which are left alone if present) — it's fully PinkyBot-managed + # and getting the latest semantics on disk matters: e.g. the + # remediation-tool allowlist fix shipped after the initial cut. + existing_source = ( + verify_effort_path.read_text() if verify_effort_path.exists() else "" + ) + new_source = _verify_effort_hook_source() + if existing_source != new_source: + verify_effort_path.write_text(new_source) + verb = "updated" if existing_source else "created" + _log(f"agent_registry: {verb} hook_verify_effort.py for {agent_name}") + + AgentRegistry._sync_hooks_settings( + claude_dir / "settings.json", + working_path=working_path.resolve(), + idle_path=idle_path.resolve(), + verify_effort_path=verify_effort_path.resolve(), + agent_name=agent_name, + ) + + @staticmethod + def _sync_hooks_settings( + settings_path: Path, + *, + working_path: Path, + idle_path: Path, + verify_effort_path: Path, + agent_name: str, + ) -> None: + """Idempotently ensure settings.json has all PinkyBot hooks wired up. + + - Creates settings.json with the full hook set if missing. + - If present, adds any missing PinkyBot-managed hook entries to + PreToolUse / Stop, preserving user-added entries. + - Identifies PinkyBot-managed entries by the absolute script path + appearing in the command string. + """ + import json as _json + + verify_cmd = ( + f"python3 {verify_effort_path}" + f' "$CLAUDE_PROJECT_DIR" 2>/dev/null || true' + ) + working_cmd = f"python3 {working_path} 2>/dev/null || true" + idle_cmd = f"python3 {idle_path} 2>/dev/null || true" + if not settings_path.exists(): - abs_working = str(working_path.resolve()) - abs_idle = str(idle_path.resolve()) settings = { "hooks": { - "PreToolUse": [{ - "matcher": ".*", - "hooks": [{ - "type": "command", - "command": ( - f"python3 {abs_working} 2>/dev/null || true" - ), - }], - }], - "Stop": [{ - "matcher": ".*", - "hooks": [{ - "type": "command", - "command": ( - f"python3 {abs_idle} 2>/dev/null || true" - ), - }], - }], + "PreToolUse": [ + { + "matcher": ".*", + "hooks": [ + {"type": "command", "command": working_cmd}, + {"type": "command", "command": verify_cmd}, + ], + } + ], + "Stop": [ + { + "matcher": ".*", + "hooks": [ + {"type": "command", "command": idle_cmd}, + ], + } + ], } } - import json as _json settings_path.write_text(_json.dumps(settings, indent=2) + "\n") _log(f"agent_registry: created settings.json for {agent_name}") + return + + # Merge path — only add the verify_effort hook if it's not already + # present. Match on the absolute script path so we don't dup if the + # user re-pathed it manually. + try: + data = _json.loads(settings_path.read_text()) + except Exception as e: + _log( + f"agent_registry: settings.json parse failed for {agent_name}: {e}; " + "skipping verify_effort merge" + ) + return + + hooks = data.setdefault("hooks", {}) + pre_tool_use = hooks.setdefault("PreToolUse", []) + + needle = str(verify_effort_path) + already_installed = False + for entry in pre_tool_use: + for h in entry.get("hooks", []): + if needle in (h.get("command") or ""): + already_installed = True + break + if already_installed: + break + + if already_installed: + return + + # Append to the first matcher=".*" bucket if one exists, else create. + target_bucket = None + for entry in pre_tool_use: + if entry.get("matcher") == ".*": + target_bucket = entry + break + if target_bucket is None: + target_bucket = {"matcher": ".*", "hooks": []} + pre_tool_use.append(target_bucket) + + target_bucket.setdefault("hooks", []).append( + {"type": "command", "command": verify_cmd} + ) + settings_path.write_text(_json.dumps(data, indent=2) + "\n") + _log( + f"agent_registry: merged hook_verify_effort into settings.json " + f"for {agent_name}" + ) # ── Agent CRUD ────────────────────────────────────────── @@ -958,7 +1265,7 @@ def register(self, name: str, **kwargs) -> Agent: "dream_enabled", "dream_schedule", "dream_timezone", "dream_model", "dream_notify", "librarian_enabled", "librarian_schedule", "runtime", "provider_url", "provider_key", "provider_model", "provider_ref", - "thinking_effort"): + "thinking_effort", "strict_effort_enforcement"): if key in kwargs: updates[key] = kwargs[key] @@ -988,6 +1295,8 @@ def register(self, name: str, **kwargs) -> Agent: updates["dream_notify"] = int(updates["dream_notify"]) if "librarian_enabled" in updates: updates["librarian_enabled"] = int(updates["librarian_enabled"]) + if "strict_effort_enforcement" in updates: + updates["strict_effort_enforcement"] = int(updates["strict_effort_enforcement"]) if updates: updates["updated_at"] = now @@ -1045,6 +1354,7 @@ def register(self, name: str, **kwargs) -> Agent: provider_model=kwargs.get("provider_model", ""), provider_ref=kwargs.get("provider_ref", ""), thinking_effort=kwargs.get("thinking_effort", "medium"), + strict_effort_enforcement=kwargs.get("strict_effort_enforcement", False), watchdog_config=kwargs.get("watchdog_config", {}), created_at=now, updated_at=now, @@ -1060,9 +1370,9 @@ def register(self, name: str, **kwargs) -> Agent: dream_enabled, dream_schedule, dream_timezone, dream_model, dream_notify, librarian_enabled, librarian_schedule, runtime, provider_url, provider_key, provider_model, provider_ref, - thinking_effort, watchdog_config, + thinking_effort, strict_effort_enforcement, watchdog_config, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (agent.name, agent.display_name, agent.model, agent.soul, agent.users, agent.boundaries, agent.system_prompt, agent.working_dir, agent.permission_mode, @@ -1077,7 +1387,8 @@ def register(self, name: str, **kwargs) -> Agent: int(agent.librarian_enabled), agent.librarian_schedule, agent.runtime, agent.provider_url, agent.provider_key, agent.provider_model, agent.provider_ref, - agent.thinking_effort, json.dumps(agent.watchdog_config), + agent.thinking_effort, int(agent.strict_effort_enforcement), + json.dumps(agent.watchdog_config), agent.created_at, agent.updated_at), ) self._db.commit() @@ -1096,7 +1407,8 @@ def register(self, name: str, **kwargs) -> Agent: "librarian_enabled, librarian_schedule, " "working_status, working_status_updated_at, " "runtime, provider_url, provider_key, provider_model, provider_ref, " - "disallowed_tools, thinking_effort, watchdog_config, last_seen_at" + "disallowed_tools, thinking_effort, watchdog_config, last_seen_at, " + "strict_effort_enforcement" ) def get(self, name: str) -> Agent | None: @@ -1647,6 +1959,102 @@ def record_heartbeat( notes=notes, latency_ms=latency_ms, ) + # ── Effort Drift Events (#429) ─────────────────────── + + def record_effort_drift( + self, + agent_name: str, + *, + expected: str, + actual: str, + session_id: str = "", + tool_name: str = "", + strict: bool = False, + ) -> int: + """Record a thinking-effort drift event from the verify_effort CLI hook. + + Emitted when ``$CLAUDE_EFFORT`` (runtime) diverges from the agent's + configured ``thinking_effort``. See #429 / verify_effort hook. + + Also writes a structured note to the heartbeat stream so drift is + visible alongside normal liveness telemetry without a separate + query. + + Returns the inserted event row id. + """ + now = time.time() + cursor = self._db.execute( + """INSERT INTO effort_drift_events + (agent_name, session_id, expected, actual, tool_name, strict, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?)""", + (agent_name, session_id, expected, actual, tool_name, + int(bool(strict)), now), + ) + self._db.commit() + + # Mirror to heartbeat notes so it shows up in normal observability. + # Don't let heartbeat failure break the drift recording. + try: + label = "blocked" if strict else "warn" + note = ( + f"[effort drift / {label}] expected={expected} actual={actual}" + ) + if tool_name: + note += f" tool={tool_name}" + self.record_heartbeat( + agent_name, + session_id=session_id, + status="alive", + notes=note, + ) + except Exception as e: # pragma: no cover — defensive + _log(f"agent_registry: effort-drift heartbeat note failed: {e}") + + return int(cursor.lastrowid or 0) + + def get_effort_drift_events( + self, + agent_name: str = "", + *, + limit: int = 50, + since: float = 0.0, + ) -> list[dict]: + """Query recent effort-drift events. + + Pass ``agent_name=""`` to get fleet-wide events. ``since`` is a unix + timestamp; only events after it are returned. + """ + conditions: list[str] = [] + params: list = [] + if agent_name: + conditions.append("agent_name=?") + params.append(agent_name) + if since: + conditions.append("timestamp>?") + params.append(since) + where = f"WHERE {' AND '.join(conditions)}" if conditions else "" + params.append(limit) + rows = self._db.execute( + f"""SELECT id, agent_name, session_id, expected, actual, + tool_name, strict, timestamp + FROM effort_drift_events {where} + ORDER BY timestamp DESC LIMIT ?""", + params, + ).fetchall() + return [ + { + "id": r[0], + "agent_name": r[1], + "session_id": r[2], + "expected": r[3], + "actual": r[4], + "tool_name": r[5], + "strict": bool(r[6]), + "timestamp": r[7], + } + for r in rows + ] + def get_latest_heartbeat(self, agent_name: str) -> AgentHeartbeat | None: """Get the most recent heartbeat for an agent.""" row = self._db.execute( @@ -2628,6 +3036,7 @@ def _row_to_agent(self, row: tuple) -> Agent: thinking_effort=row[45] if len(row) > 45 and row[45] else "medium", watchdog_config=json.loads(row[46]) if len(row) > 46 and row[46] else {}, last_seen_at=row[47] if len(row) > 47 else 0.0, + strict_effort_enforcement=bool(row[48]) if len(row) > 48 else False, ) # ── Cost Tracking ────────────────────────────────────── @@ -2674,5 +3083,222 @@ def get_total_lifetime_cost(self) -> float: row = self._db.execute("SELECT SUM(cost_usd) FROM agent_costs").fetchone() return round(row[0] or 0, 6) + # ── Ferry peer-fleet ACL ─────────────────────────────── + # + # Separate identity primitive from approved_users (humans on Telegram / + # Discord / etc.). Ferry inbound is *agents* addressing an agent — + # different identity primitive, separate list. Default-deny. + # + # Stored as JSON array of AgentCardSelector dicts on the `agents` row's + # `peer_fleet_acl` column. See `pinky_daemon.ferry.types.AgentCardSelector` + # for the selector shape. + + def has_agent(self, agent_name: str) -> bool: + """Return True if an agent with this name is registered (any status).""" + row = self._db.execute( + "SELECT 1 FROM agents WHERE name = ? LIMIT 1", (agent_name,) + ).fetchone() + return row is not None + + def get_peer_fleet_acl(self, agent_name: str) -> list[dict]: + """Return the list of peer-fleet ACL selector dicts for an agent. + + Each dict has shape `{fleet, agent_id, pinky_type}` (any combination, + with at least one non-null field per AgentCardSelector contract). + Returns [] for unknown agents or empty/missing ACL. + """ + row = self._db.execute( + "SELECT peer_fleet_acl FROM agents WHERE name = ?", (agent_name,) + ).fetchone() + if not row or not row[0]: + return [] + try: + data = json.loads(row[0]) + except (TypeError, ValueError): + return [] + if not isinstance(data, list): + return [] + return [d for d in data if isinstance(d, dict)] + + def set_peer_fleet_acl( + self, + agent_name: str, + selectors: list[dict], + ) -> None: + """Replace the peer-fleet ACL for an agent (full replacement, not merge). + + Each selector must have at least one of fleet/agent_id/pinky_type + non-empty. Selectors that don't validate are silently dropped with + a log line. Empty list = deny all peer-fleet inbound. + """ + clean: list[dict] = [] + for raw in selectors or []: + if not isinstance(raw, dict): + _log(f"peer_fleet_acl: skipping non-dict selector for {agent_name}: {raw!r}") + continue + fleet = (raw.get("fleet") or "").strip() or None + agent_id = (raw.get("agent_id") or "").strip() or None + pinky_type = (raw.get("pinky_type") or "").strip() or None + if not (fleet or agent_id or pinky_type): + _log(f"peer_fleet_acl: skipping empty selector for {agent_name}") + continue + clean.append({ + "fleet": fleet, + "agent_id": agent_id, + "pinky_type": pinky_type, + }) + self._db.execute( + "UPDATE agents SET peer_fleet_acl = ? WHERE name = ?", + (json.dumps(clean), agent_name), + ) + self._db.commit() + + def add_peer_fleet_acl( + self, + agent_name: str, + *, + fleet: str | None = None, + agent_id: str | None = None, + pinky_type: str | None = None, + ) -> bool: + """Append one selector to an agent's peer_fleet_acl. + + Returns True if added, False if the selector was empty (dropped). + Idempotent: a selector matching an existing entry is skipped. + + Thread-safe: read-modify-write is guarded by ``_rmw_lock`` so + concurrent admin-API requests can't lose updates. + """ + entry = { + "fleet": (fleet or "").strip() or None, + "agent_id": (agent_id or "").strip() or None, + "pinky_type": (pinky_type or "").strip() or None, + } + if not (entry["fleet"] or entry["agent_id"] or entry["pinky_type"]): + return False + with self._rmw_lock: + existing = self.get_peer_fleet_acl(agent_name) + if entry in existing: + return True + existing.append(entry) + self.set_peer_fleet_acl(agent_name, existing) + return True + + def remove_peer_fleet_acl( + self, + agent_name: str, + *, + fleet: str | None = None, + agent_id: str | None = None, + pinky_type: str | None = None, + ) -> int: + """Remove all selectors matching the given criteria. Returns count removed. + + Matching is **exact** on every field. A stored selector matches + only when all three of ``fleet`` / ``agent_id`` / ``pinky_type`` + equal the corresponding argument (``None`` and empty string are + normalized to the same value, so omitting an argument is the + same as passing ``""``). + + Wildcard caveat: a stored selector with ``agent_id="*"`` is + removed only by passing ``agent_id="*"`` — calling + ``remove_peer_fleet_acl(agent_name, fleet="sigil")`` will **not** + remove a stored ``{fleet:"sigil", agent_id:"*"}`` and silently + returns 0. Pass the exact stored selector you want to delete. + + Thread-safe: read-modify-write is guarded by ``_rmw_lock`` so + concurrent admin-API requests can't lose updates. + """ + target = { + "fleet": (fleet or "").strip() or None, + "agent_id": (agent_id or "").strip() or None, + "pinky_type": (pinky_type or "").strip() or None, + } + with self._rmw_lock: + existing = self.get_peer_fleet_acl(agent_name) + kept = [s for s in existing if s != target] + removed = len(existing) - len(kept) + if removed: + self.set_peer_fleet_acl(agent_name, kept) + return removed + + # ── Ferry outbound mesh allowlist ────────────────────── + # + # Per-agent allowlist gating which (fleet, agent) targets this agent + # may publish to via the mesh_remote_send tool. Stored as JSON list + # of "agent_slug@fleet" patterns on the `agents` row's + # `mesh_outbound_allowlist` column. Default-deny (empty list = no + # outbound). Patterns support wildcards per + # `pinky_daemon.ferry.outbound.allowlist_matches`. + + def get_mesh_outbound_allowlist(self, agent_name: str) -> list[str]: + """Return the agent's mesh outbound allowlist patterns. + + Returns [] for unknown agents or empty/missing allowlist. + """ + row = self._db.execute( + "SELECT mesh_outbound_allowlist FROM agents WHERE name = ?", + (agent_name,), + ).fetchone() + if not row or not row[0]: + return [] + try: + data = json.loads(row[0]) + except (TypeError, ValueError): + return [] + if not isinstance(data, list): + return [] + return [s for s in data if isinstance(s, str) and s.strip()] + + def set_mesh_outbound_allowlist( + self, + agent_name: str, + patterns: list[str], + ) -> None: + """Replace the mesh outbound allowlist for an agent (full replace). + + Empty/whitespace patterns are silently dropped. Empty list = deny + all outbound. + """ + clean = [p.strip() for p in (patterns or []) if isinstance(p, str) and p.strip()] + self._db.execute( + "UPDATE agents SET mesh_outbound_allowlist = ? WHERE name = ?", + (json.dumps(clean), agent_name), + ) + self._db.commit() + + def add_mesh_outbound_allowlist(self, agent_name: str, pattern: str) -> bool: + """Append one pattern to an agent's mesh outbound allowlist. + + Returns True if added (or already present), False if pattern is empty. + Idempotent. Thread-safe via ``_rmw_lock``. + """ + pat = (pattern or "").strip() + if not pat: + return False + with self._rmw_lock: + existing = self.get_mesh_outbound_allowlist(agent_name) + if pat in existing: + return True + existing.append(pat) + self.set_mesh_outbound_allowlist(agent_name, existing) + return True + + def remove_mesh_outbound_allowlist(self, agent_name: str, pattern: str) -> int: + """Remove all matching patterns. Returns count removed. + + Exact string match only; pass the stored pattern verbatim. + """ + pat = (pattern or "").strip() + if not pat: + return 0 + with self._rmw_lock: + existing = self.get_mesh_outbound_allowlist(agent_name) + kept = [s for s in existing if s != pat] + removed = len(existing) - len(kept) + if removed: + self.set_mesh_outbound_allowlist(agent_name, kept) + return removed + def close(self) -> None: self._db.close() diff --git a/src/pinky_daemon/api.py b/src/pinky_daemon/api.py index 82e22cfd..fa6d3a91 100644 --- a/src/pinky_daemon/api.py +++ b/src/pinky_daemon/api.py @@ -62,11 +62,18 @@ ConversationListResponse, CreateGroupRequest, CreateSessionRequest, + EffortDriftRequest, + FederationPeerUpsertRequest, ForkSessionRequest, HistoryResponse, JoinGroupRequest, + MeshAllowlistEntryRequest, + MeshAllowlistSetRequest, + MeshSendRequest, MessageResponse, OwnerProfileRequest, + PeerFleetAclEntryRequest, + PeerFleetAclSetRequest, PushEventRequest, RecordHeartbeatRequest, RegisterAgentRequest, @@ -115,6 +122,7 @@ ) from pinky_daemon.kb_store import KBStore from pinky_daemon.librarian_runner import LibrarianRunner +from pinky_daemon.mesh_store import MeshStore from pinky_daemon.outreach_config import OutreachConfigStore from pinky_daemon.plugin_manager import PluginManager from pinky_daemon.presentation_store import PresentationStore @@ -201,8 +209,21 @@ def resolve_provider_config( return url, key, model -# Models that support 1M context windows (fallback; dynamically loaded from registry) -_1M_MODELS = {"claude-sonnet-4-6", "claude-opus-4-6", "claude-opus-4-7"} +# ── Agent Comms Models ─────────────────────────────────────── + + +CONTENT_TYPES = Literal["text", "task_request", "task_response", "status", "file_transfer"] +PRIORITY_LEVELS = Literal[0, 1, 2] + + +# ── Skill Models ──────────────────────────────────────────── + + +# ── Outreach Config Models ────────────────────────────────── + + +# ── Agent Models ───────────────────────────────────────────── + def _refresh_1m_models(registry) -> None: """Refresh the 1M model set from the registry.""" @@ -215,6 +236,9 @@ def _refresh_1m_models(registry) -> None: pass # Keep fallback +# ── Research Pipeline Models ────────────────────────────────── + + # ── Core Skill Seeding ────────────────────────────────────── @@ -530,6 +554,66 @@ def _write_mcp_json( mcp_json.write_text(json.dumps(mcp_config, indent=2)) +def _check_installed_deps_drift(repo_dir: str) -> list[dict]: + """Compare installed package versions to pyproject.toml pins. + + Returns a list of drift entries — one per dependency whose installed + version doesn't satisfy the pin (including missing packages). Each entry + is ``{"package": str, "specifier": str, "installed": str | None}``. + + Used by ``admin_update`` to decide whether ``pip install`` is needed even + when ``pyproject.toml`` didn't change in the current pull. Catches the + common failure mode where an earlier pull bumped a pin but its routine + restart skipped the reinstall step, leaving installed packages stale. + + Failures (missing pyproject, unparseable deps) are surfaced via raised + exceptions and treated as non-fatal by callers — drift can't be assessed + so reinstall stays gated on the other triggers. + """ + import tomllib + from importlib.metadata import PackageNotFoundError, version + + from packaging.requirements import Requirement + + pyproject_path = Path(repo_dir) / "pyproject.toml" + with open(pyproject_path, "rb") as f: + meta = tomllib.load(f) + + project = meta.get("project", {}) or {} + deps: list[str] = list(project.get("dependencies", []) or []) + optional = project.get("optional-dependencies", {}) or {} + for group_deps in optional.values(): + if group_deps: + deps.extend(group_deps) + + drifts: list[dict] = [] + for dep_str in deps: + try: + req = Requirement(dep_str) + except Exception: + # Unparseable entry — skip rather than fail the whole check. + continue + # Skip markers that don't apply to the current env (e.g. python_version<"3.10") + if req.marker is not None and not req.marker.evaluate(): + continue + try: + installed = version(req.name) + except PackageNotFoundError: + drifts.append({ + "package": req.name, + "specifier": str(req.specifier), + "installed": None, + }) + continue + if req.specifier and not req.specifier.contains(installed, prereleases=True): + drifts.append({ + "package": req.name, + "specifier": str(req.specifier), + "installed": installed, + }) + return drifts + + # ── API Server ─────────────────────────────────────────────── @@ -1923,6 +2007,14 @@ def runtime_from_legacy_provider(agent_config) -> str: "headers": agent_headers, } + # #429: ensure verify_effort hook is installed in the agent's + # .claude/settings.json before we spin up the session. No-op for + # agents whose workspace already has it. + try: + agents.ensure_workspace_hooks(agent_name) + except Exception as e: + _log(f"streaming-start: ensure_workspace_hooks({agent_name}) — {e}") + config = StreamingSessionConfig( agent_name=agent_name, label=label, @@ -1945,6 +2037,9 @@ def runtime_from_legacy_provider(agent_config) -> str: provider_url=resolved_provider_url, provider_key=resolved_provider_key, thinking_effort=agent.thinking_effort or "medium", + strict_effort_enforcement=bool( + getattr(agent, "strict_effort_enforcement", False) + ), ) callback = await _make_streaming_response_callback() @@ -2088,6 +2183,7 @@ async def _disconnect_streaming_sessions(agent_name: str) -> int: presentations = PresentationStore(db_path=db_path.replace(".db", "_presentations.db")) app_store = AppStore(db_path=db_path.replace(".db", "_apps.db")) trigger_store = TriggerStore(db_path=db_path.replace(".db", "_triggers.db")) + mesh_store = MeshStore(db_path=db_path.replace(".db", "_mesh.db")) # Knowledge Base — project-level, all agents share _data_dir = Path(db_path).parent @@ -3875,6 +3971,7 @@ async def register_agent(req: RegisterAgentRequest): provider_model=req.provider_model, provider_ref=req.provider_ref, thinking_effort=req.thinking_effort, + strict_effort_enforcement=req.strict_effort_enforcement, watchdog_config=req.watchdog_config or {}, ) # Write .mcp.json so the agent gets default MCP servers (memory, self, messaging) @@ -4007,6 +4104,63 @@ async def set_agent_working_status(name: str, req: AgentStatusRequest): "ok": True, } + @app.post("/agents/{name}/effort-drift") + async def report_effort_drift(name: str, req: EffortDriftRequest): + """Record a thinking-effort drift event from hook_verify_effort.py (#429). + + Called when ``$CLAUDE_EFFORT`` (runtime) diverges from + ``$PINKY_EXPECTED_EFFORT`` (agent's configured ``thinking_effort``). + The hook is fire-and-forget — we return quickly and never fail the + tool call. + """ + agent = agents.get(name) + if not agent: + raise HTTPException(404, f"Agent '{name}' not found") + event_id = agents.record_effort_drift( + name, + expected=req.expected, + actual=req.actual, + session_id=req.session_id, + tool_name=req.tool_name, + strict=req.strict, + ) + activity.log( + agent_name=name, + event_type="effort_drift", + title=( + f"effort drift — expected={req.expected} actual={req.actual}" + + (" (blocked)" if req.strict else "") + ), + metadata={ + "agent": name, + "expected": req.expected, + "actual": req.actual, + "tool_name": req.tool_name, + "strict": req.strict, + }, + ) + return { + "ok": True, + "agent": name, + "event_id": event_id, + "expected": req.expected, + "actual": req.actual, + "strict": req.strict, + } + + @app.get("/agents/{name}/effort-drift") + async def list_effort_drift_events( + name: str, limit: int = 50, since: float = 0.0, + ): + """List recent effort-drift events for an agent (#429).""" + agent = agents.get(name) + if not agent: + raise HTTPException(404, f"Agent '{name}' not found") + events = agents.get_effort_drift_events( + agent_name=name, limit=limit, since=since, + ) + return {"agent": name, "events": events, "count": len(events)} + @app.get("/agents/{name}/status") async def get_agent_working_status(name: str): """Get an agent's current working status. @@ -4598,6 +4752,310 @@ async def set_user_timezone(name: str, chat_id: str, timezone: str = ""): raise HTTPException(404, "User not found") return {"updated": True, "chat_id": chat_id, "timezone": timezone} + # ── Ferry Peer-Fleet ACL ─────────────────────────────── + # + # Separate identity primitive from approved_users. Ferry inbound is + # *agents* addressing an agent; approved_users is *humans*. Default-deny. + # See `pinky_daemon.ferry.types.AgentCardSelector` for selector shape. + + @app.get("/agents/{name}/peer-fleet-acl") + async def get_peer_fleet_acl(name: str): + """List the peer-fleet ACL selectors for an agent.""" + if not agents.has_agent(name): + raise HTTPException(404, f"Agent '{name}' not found") + return { + "agent": name, + "selectors": agents.get_peer_fleet_acl(name), + } + + @app.post("/agents/{name}/peer-fleet-acl") + async def add_peer_fleet_acl(name: str, req: PeerFleetAclEntryRequest): + """Add one selector to an agent's peer-fleet ACL. + + Empty selectors (no fleet/agent_id/pinky_type set) are rejected with 400. + Adds are idempotent — already-present selectors return success without + creating duplicates. + """ + if not agents.has_agent(name): + raise HTTPException(404, f"Agent '{name}' not found") + added = agents.add_peer_fleet_acl( + name, + fleet=req.fleet or None, + agent_id=req.agent_id or None, + pinky_type=req.pinky_type or None, + ) + if not added: + raise HTTPException( + 400, + "selector requires at least one of fleet, agent_id, or pinky_type", + ) + return { + "agent": name, + "added": True, + "selectors": agents.get_peer_fleet_acl(name), + } + + @app.put("/agents/{name}/peer-fleet-acl") + async def set_peer_fleet_acl(name: str, req: PeerFleetAclSetRequest): + """Replace the full peer-fleet ACL for an agent (full replacement).""" + if not agents.has_agent(name): + raise HTTPException(404, f"Agent '{name}' not found") + selector_dicts = [ + { + "fleet": s.fleet or None, + "agent_id": s.agent_id or None, + "pinky_type": s.pinky_type or None, + } + for s in req.selectors + ] + agents.set_peer_fleet_acl(name, selector_dicts) + return { + "agent": name, + "selectors": agents.get_peer_fleet_acl(name), + } + + @app.delete("/agents/{name}/peer-fleet-acl") + async def remove_peer_fleet_acl( + name: str, + fleet: str = "", + agent_id: str = "", + pinky_type: str = "", + ): + """Remove all selectors matching the given query. + + Match is exact across all three fields (None == empty == "don't filter"). + Returns the count removed. + """ + if not agents.has_agent(name): + raise HTTPException(404, f"Agent '{name}' not found") + removed = agents.remove_peer_fleet_acl( + name, + fleet=fleet or None, + agent_id=agent_id or None, + pinky_type=pinky_type or None, + ) + return { + "agent": name, + "removed": removed, + "selectors": agents.get_peer_fleet_acl(name), + } + + # ── Ferry Mesh Outbound (mesh_remote_send) ───────────── + # + # Agent-driven outbound: build a ferry envelope and publish to the + # mesh transport (NATS). Default-deny via per-agent allowlist; cred + # never leaves the daemon process. See pinky_daemon.ferry.outbound. + + @app.get("/agents/{name}/mesh/allowlist") + async def get_mesh_allowlist(name: str): + """List the outbound allowlist patterns for an agent.""" + if not agents.has_agent(name): + raise HTTPException(404, f"Agent '{name}' not found") + return { + "agent": name, + "patterns": agents.get_mesh_outbound_allowlist(name), + } + + @app.post("/agents/{name}/mesh/allowlist") + async def add_mesh_allowlist(name: str, req: MeshAllowlistEntryRequest): + """Add one pattern to an agent's outbound allowlist (idempotent).""" + if not agents.has_agent(name): + raise HTTPException(404, f"Agent '{name}' not found") + added = agents.add_mesh_outbound_allowlist(name, req.pattern) + if not added: + raise HTTPException(400, "pattern must be non-empty") + return { + "agent": name, + "added": True, + "patterns": agents.get_mesh_outbound_allowlist(name), + } + + @app.put("/agents/{name}/mesh/allowlist") + async def set_mesh_allowlist(name: str, req: MeshAllowlistSetRequest): + """Replace the full outbound allowlist for an agent.""" + if not agents.has_agent(name): + raise HTTPException(404, f"Agent '{name}' not found") + agents.set_mesh_outbound_allowlist(name, req.patterns) + return { + "agent": name, + "patterns": agents.get_mesh_outbound_allowlist(name), + } + + @app.delete("/agents/{name}/mesh/allowlist") + async def remove_mesh_allowlist(name: str, pattern: str = ""): + """Remove one pattern from an agent's outbound allowlist.""" + if not agents.has_agent(name): + raise HTTPException(404, f"Agent '{name}' not found") + removed = agents.remove_mesh_outbound_allowlist(name, pattern) + return { + "agent": name, + "removed": removed, + "patterns": agents.get_mesh_outbound_allowlist(name), + } + + @app.get("/mesh/diagnostics") + async def mesh_diagnostics(): + """Operator-facing health snapshot for the daemon's mesh sender. + + Reports whether creds + URL + nats CLI are wired without leaking + the password. + """ + from pinky_daemon.ferry.outbound import MeshSender + return MeshSender().diagnostics() + + @app.post("/agents/{name}/mesh/send") + async def mesh_send(name: str, req: MeshSendRequest): + """Publish a ferry envelope to ``req.target`` on behalf of ``name``. + + Permission model: ``req.target`` must match at least one pattern + in the agent's ``mesh_outbound_allowlist``. Default-deny. + + Returns ``{sent, correlation_id, subject, ts}`` on success; + on failure the error is surfaced in ``error``. + """ + from pinky_daemon.ferry.outbound import ( + MeshSender, + allowlist_matches, + build_envelope, + parse_address, + ) + + if not agents.has_agent(name): + raise HTTPException(404, f"Agent '{name}' not found") + + target = (req.target or "").strip() + if not target: + raise HTTPException(400, "target is required") + if parse_address(target) is None: + raise HTTPException(400, f"unparseable target address: {target!r}") + + allowlist = agents.get_mesh_outbound_allowlist(name) + if not allowlist_matches(target, allowlist): + raise HTTPException( + 403, + f"target {target!r} not in agent's mesh_outbound_allowlist", + ) + + # Body shape: callers may pass a string (wrapped as {"text": str}) + # or a dict. ``kind`` is always set from the request. + if isinstance(req.body, dict): + body_dict = dict(req.body) + else: + body_dict = {"text": str(req.body or "")} + body_dict["kind"] = req.kind or "msg" + + envelope = build_envelope( + from_=f"ferry://pinkybot/{name}", + to=target, + body=body_dict, + correlation_id=req.correlation_id or None, + reply_to=req.reply_to or None, + priority=req.priority or "normal", + ) + + sender = MeshSender() + result = sender.send(envelope) + + # Audit log: every send attempt, success or failure. Persistence + # failure must not propagate into the API response (defensive try). + try: + target_fleet, target_agent = parse_address(target) or ("", "") + mesh_store.log_message( + direction="outbound", + local_agent=name, + remote_fleet=target_fleet, + remote_agent=target_agent, + correlation_id=envelope.correlation_id or "", + kind=envelope.body.get("kind", "msg"), + body=envelope.body, + envelope_ts=envelope.ts, + reply_to=envelope.reply_to, + error=result.error, + ) + except Exception as exc: # noqa: BLE001 + _log(f"mesh outbound log failed (non-fatal): {exc}") + + return { + "sent": result.sent, + "correlation_id": result.correlation_id, + "subject": result.subject, + "ts": result.ts, + "error": result.error, + } + + @app.get("/agents/{name}/mesh/inbox") + async def get_mesh_inbox( + name: str, + limit: int = 50, + offset: int = 0, + kind: str = "", + sender: str = "", + direction: str = "", + ): + """List mesh messages for an agent, newest first. + + Filters: ``kind`` (exact), ``sender`` (``"agent@fleet"``), + ``direction`` (``"inbound"`` | ``"outbound"`` | ``""``). + """ + if not agents.has_agent(name): + raise HTTPException(404, f"Agent '{name}' not found") + messages = mesh_store.get_inbox( + name, + limit=limit, + offset=offset, + kind=kind, + sender=sender, + direction=direction, + ) + return { + "agent": name, + "count": len(messages), + "messages": [m.to_dict() for m in messages], + } + + @app.get("/federation/peers") + async def list_federation_peers( + fleet: str = "", + seed_source: str = "", + limit: int = 200, + ): + """List known federation peers, newest-seen first. + + Filters: ``fleet`` (exact), ``seed_source`` (``"config"`` | ``"observed"``). + """ + peers = mesh_store.list_peers( + fleet=fleet, seed_source=seed_source, limit=limit, + ) + return { + "count": len(peers), + "peers": [p.to_dict() for p in peers], + } + + @app.post("/federation/peers") + async def upsert_federation_peer(req: FederationPeerUpsertRequest): + """Insert or update a federation peer (admin / config-seeded path). + + ``seed_source='config'`` is the stronger statement of intent — once + set, an observed-update doesn't downgrade it. + """ + if not (req.fleet and req.agent): + raise HTTPException(400, "fleet and agent are required") + seed = req.seed_source if req.seed_source in ("config", "observed") else "config" + mesh_store.upsert_peer( + fleet=req.fleet, + agent=req.agent, + display_name=req.display_name, + seed_source=seed, + ) + peer = mesh_store.get_peer(req.fleet, req.agent) + return {"upserted": True, "peer": peer.to_dict() if peer else None} + + @app.delete("/federation/peers/{fleet}/{agent}") + async def delete_federation_peer(fleet: str, agent: str): + """Hard-delete a federation peer.""" + removed = mesh_store.remove_peer(fleet, agent) + return {"removed": removed, "fleet": fleet, "agent": agent} + # ── Pending Messages (Broker) ────────────────────────── @app.get("/agents/{name}/pending-messages") @@ -6281,12 +6739,12 @@ async def _heartbeat_resurrect(agent_name: str, _session_id: str) -> None: f"session registered" ) return - if ss.is_connected: + if getattr(ss, "is_connected", False): # Race: the in-process retry recovered between heartbeat tick and # the callback running. Also the load-bearing guard for bypassing # the save_my_context restart guard — see docstring above. return - if ss.is_idle_sleeping: + if getattr(ss, "is_idle_sleeping", False): # Session was deliberately disconnected by idle_sleep(). Resurrecting # it here would fight the idle-sleep state and cause an immediate # reconnect/sleep churn cycle. The next genuine wake (scheduler or @@ -6300,8 +6758,26 @@ async def _heartbeat_resurrect(agent_name: str, _session_id: str) -> None: try: # TODO(#338-followup): wrap in asyncio.create_task so the scheduler # tick isn't blocked for up to ~40s during the internal backoff. - await ss.attempt_reconnect() - if ss.is_connected: + attempt_reconnect = getattr(ss, "attempt_reconnect", None) + if callable(attempt_reconnect): + await attempt_reconnect() + else: + # Runtime-tolerant fallback for future StreamingSession-compatible + # implementations that have connect/disconnect but not the richer + # watchdog reconnect helper yet. + try: + disconnect = getattr(ss, "disconnect", None) + if callable(disconnect): + await disconnect() + except Exception as e: + _log(f"api: resurrection pre-disconnect failed for {agent_name}: {e}") + connect = getattr(ss, "connect", None) + if not callable(connect): + raise AttributeError( + f"{ss.__class__.__name__} has no attempt_reconnect or connect" + ) + await connect() + if getattr(ss, "is_connected", False): activity.log( agent_name, "watchdog_resurrect", f"{agent_name} restored by heartbeat watchdog", @@ -6447,7 +6923,16 @@ def _resolve_memory_db(agent_name: str) -> str: await shared_mcp_manager.start() _log(f"startup: shared MCP server started on {shared_mcp_manager.url}") - auto_start_agents = agents.list_auto_start_agents() + # Boot policy (2026-05-11): only the **main agent** auto-resumes at boot. + # Sibling agents stay dormant until something triggers them — an inbound + # message, an agent-to-agent message, or a scheduled wake firing. Their + # autonomy loop + streaming session are created lazily on first dispatch + # (see `_ensure_streaming_session` and `autonomy.push_event`). + # + # `auto_start_agents` is no longer used to gate boot startup. The + # `agent.auto_start` flag is preserved for compat (`/admin/auto-start` + # endpoint still reports it) but does not control which agents come up. + main_name_for_boot = agents.get_main_agent() # Start broker pollers and streaming sessions for all enabled agents. from pinky_daemon.pollers import ( @@ -6537,54 +7022,39 @@ def _resolve_memory_db(agent_name: str) -> str: except Exception as e: _log(f"startup: iMessage poller failed for {agent.name}: {e}") - # Decide which agents get streaming sessions on boot: - # - Agents with auto_start, heartbeat, or main agent: always start (create main if needed) - # - Agents with persisted sessions from before: restore those sessions - # - Other agents: skip (sessions created on-demand via _ensure_streaming_session) - should_auto_start = ( - agent.auto_start - or agent.heartbeat_interval > 0 - or agent.name == agents.get_main_agent() - ) - persisted = agents.list_streaming_session_ids(agent.name) - has_persisted = any(entry["session_id"] for entry in persisted) + # Boot policy: only the main agent auto-resumes its streaming session. + # Sibling sessions are created on-demand via `_ensure_streaming_session` + # when an inbound message / agent-to-agent message / scheduled wake + # routes through the autonomy event queue. Persisted session IDs are + # kept in the DB so resume-by-id still works when siblings later wake. + if agent.name != main_name_for_boot: + _log(f"startup: skipping streaming session for {agent.name} (on-demand only)") + continue # Validate working_dir exists — stale paths from a different machine # (e.g. migrating from Mac Mini to RPi) would cause Fatal errors. if work_dir and not work_dir.is_dir(): _log(f"startup: working_dir missing for {agent.name}: {work_dir} — skipping session") # Clear stale session IDs so we don't retry on next boot - for entry in persisted: + for entry in agents.list_streaming_session_ids(agent.name): if entry["session_id"]: agents.set_streaming_session_id(agent.name, "", label=entry["label"]) continue - if should_auto_start: - # Only start main session on boot — sub-sessions are on-demand - main_resume = agents.get_streaming_session_id(agent.name, label="main") - labels_to_start = {"main": main_resume} - elif has_persisted: - # Agent had sessions before — just restore main - main_resume = agents.get_streaming_session_id(agent.name, label="main") - labels_to_start = {"main": main_resume} - else: - _log(f"startup: skipping streaming session for {agent.name} (on-demand only)") - continue - - for label, resume_id in labels_to_start.items(): - try: - await _start_streaming_session(agent.name, label=label, resume_id=resume_id) - streaming_count += 1 - if resume_id: - _log(f"startup: streaming session resumed for {agent.name}/{label} (session {resume_id[:12]})") - else: - _log(f"startup: streaming session connected for {agent.name}/{label} (new)") - except Exception as e: - _log(f"startup: streaming session failed for {agent.name}/{label}: {e}") - # If resume failed, clear the stale session ID and try fresh on next boot - if resume_id: - _log(f"startup: clearing stale session ID for {agent.name}/{label}") - agents.set_streaming_session_id(agent.name, "", label=label) + main_resume = agents.get_streaming_session_id(agent.name, label="main") + try: + await _start_streaming_session(agent.name, label="main", resume_id=main_resume) + streaming_count += 1 + if main_resume: + _log(f"startup: streaming session resumed for {agent.name}/main (session {main_resume[:12]})") + else: + _log(f"startup: streaming session connected for {agent.name}/main (new)") + except Exception as e: + _log(f"startup: streaming session failed for {agent.name}/main: {e}") + # If resume failed, clear the stale session ID and try fresh on next boot + if main_resume: + _log(f"startup: clearing stale session ID for {agent.name}/main") + agents.set_streaming_session_id(agent.name, "", label="main") # Clean up legacy sessions for agents that now have streaming sessions. # These ghost sessions were restored by SessionManager._restore_sessions() @@ -6602,19 +7072,29 @@ def _resolve_memory_db(agent_name: str) -> str: await autonomy.start() await watchdog.start() - for agent in auto_start_agents: - await autonomy.start_agent_loop(agent.name) - - # Main agent always gets an autonomy loop, even without auto_start + # Boot policy: only the main agent's autonomy loop starts at boot. + # Sibling loops start lazily via `autonomy.push_event` when an event + # arrives for them (inbound message, agent-to-agent message, scheduled + # wake). See `autonomy.push_event` — it ungates the wake on `enabled` + # rather than `auto_start` so any enabled agent can be woken on demand. + main_started = False main_name = agents.get_main_agent() - auto_started_names = {a.name for a in auto_start_agents} - if main_name and main_name not in auto_started_names: + if main_name: main_agent = agents.get(main_name) if main_agent and main_agent.enabled: await autonomy.start_agent_loop(main_name) - _log(f"startup: main agent '{main_name}' auto-started") + main_started = True + _log(f"startup: main agent '{main_name}' autonomy loop started") + else: + _log(f"startup: warn — main agent '{main_name}' missing or disabled") + else: + _log("startup: warn — no main agent configured; no autonomy loop started") - _log(f"startup: scheduler + autonomy + watchdog running, {len(auto_start_agents)} agent(s) auto-started, {len(_broker_pollers)} broker poller(s), {streaming_count} streaming") + _log( + f"startup: scheduler + autonomy + watchdog running, " + f"main={'on' if main_started else 'off'}, " + f"{len(_broker_pollers)} broker poller(s), {streaming_count} streaming" + ) @app.on_event("shutdown") async def on_shutdown(): @@ -6926,10 +7406,12 @@ async def admin_update( summary = "" # Detect dependency changes — rebuild whenever pyproject.toml or uv.lock - # changed in the pull, or when force_deps=True is passed (escape hatch - # for installed-vs-pinned drift that git diff can't see). + # changed in the pull, when force_deps=True is passed, or when the + # installed package versions have drifted from the pyproject pins + # (e.g., pyproject was bumped on an earlier pull that skipped reinstall). deps_rebuilt = False deps_error = "" + deps_drift: list[dict] = [] try: if before_hash != after_hash: changed = sp.check_output( @@ -6940,7 +7422,17 @@ async def admin_update( else: changed = "" - if changed or force_deps: + # Auto-deps drift check: compare installed versions to pyproject pins + # independently of git diff. Catches the "pyproject bumped earlier, + # routine restart skipped reinstall" case that force_deps used to + # paper over manually. + try: + deps_drift = _check_installed_deps_drift(repo_dir) + except Exception as drift_exc: + _log(f"admin: deps drift check failed (non-fatal): {drift_exc}") + deps_drift = [] + + if changed or force_deps or deps_drift: # Prefer project venv pip if present, else use the running daemon's # interpreter (sys.executable). This works for both venv and # system-python deployments — the prior `.venv/bin/pip`-only path @@ -6995,6 +7487,7 @@ async def admin_update( "commits": summary.splitlines() if summary else [], "deps_rebuilt": deps_rebuilt, "deps_error": deps_error or None, + "deps_drift": deps_drift, "frontend_rebuilt": frontend_rebuilt, "frontend_error": frontend_error or None, "forced_reset": forced_reset, diff --git a/src/pinky_daemon/api_models.py b/src/pinky_daemon/api_models.py index f4ba12d8..ccbc559d 100644 --- a/src/pinky_daemon/api_models.py +++ b/src/pinky_daemon/api_models.py @@ -311,6 +311,7 @@ class RegisterAgentRequest(BaseModel): provider_model: str = "" # Model name override for this provider provider_ref: str = "" # ID of a global provider from the providers table thinking_effort: str = "medium" # low/medium/high/xhigh/max + strict_effort_enforcement: bool = False # PR #429 — block tool calls when effort drifts watchdog_config: dict | None = None # Per-agent watchdog overrides @@ -349,6 +350,7 @@ class UpdateAgentRequest(BaseModel): provider_model: str | None = None # Model name override for this provider provider_ref: str | None = None # ID of a global provider from the providers table thinking_effort: str | None = None # low/medium/high/xhigh/max + strict_effort_enforcement: bool | None = None # PR #429 — block tool calls when effort drifts watchdog_config: dict | None = None # Per-agent watchdog overrides @@ -772,3 +774,82 @@ class WikiSaveRequest(BaseModel): content: str sources: list[str] = [] related: list[str] = [] + + +# ── Beta-only additions (merged from beta 2026-05-11) ────── + + +class EffortDriftRequest(BaseModel): + """Effort-drift event — POSTed by hook_verify_effort.py (#429). + + Emitted when the runtime ``$CLAUDE_EFFORT`` (Claude Code v2.1.133+) + diverges from the daemon-injected ``PINKY_EXPECTED_EFFORT``. + """ + + expected: str + actual: str + tool_name: str = "" + strict: bool = False + session_id: str = "" + + +class FederationPeerUpsertRequest(BaseModel): + """Insert or update a federation peer (admin / config-seeded path). + + Composite key: (fleet, agent). ``seed_source='config'`` is the stronger + statement of intent and never gets downgraded by an observed update. + """ + + fleet: str + agent: str + display_name: str = "" + seed_source: str = "config" # "config" | "observed" + + +class MeshAllowlistEntryRequest(BaseModel): + """Add or remove one pattern in an agent's mesh outbound allowlist.""" + + pattern: str # e.g. "pulse@pulse", "*@pulse", "pulse@*" + + +class MeshAllowlistSetRequest(BaseModel): + """Replace the full mesh outbound allowlist for an agent.""" + + patterns: list[str] = [] + + +class MeshSendRequest(BaseModel): + """Publish a ferry envelope to a remote agent via the daemon's mesh sender. + + The agent making this request must have ``target`` matching at least one + pattern in its ``mesh_outbound_allowlist``; default-deny otherwise. + + ``body`` is a free-form payload — caller decides shape; ``kind`` is + inserted into ``body.kind`` per ferry PROTOCOL.md v0.1 §1. + """ + + target: str # "agent_slug@fleet" or "ferry://fleet/agent_slug" + body: str | dict = "" + kind: str = "msg" + correlation_id: str = "" + reply_to: str = "" + priority: str = "normal" + + +class PeerFleetAclEntryRequest(BaseModel): + """Add or remove one peer-fleet ACL selector for an agent. + + Selectors are shaped after `pinky_daemon.ferry.types.AgentCardSelector` — + at-least-one of fleet/agent_id/pinky_type must be non-empty. Wildcards via + agent_id="*" or fleet="*". + """ + + fleet: str = "" + agent_id: str = "" + pinky_type: str = "" + + +class PeerFleetAclSetRequest(BaseModel): + """Replace the full peer-fleet ACL for an agent (full replacement, not merge).""" + + selectors: list[PeerFleetAclEntryRequest] = [] diff --git a/src/pinky_daemon/autonomy.py b/src/pinky_daemon/autonomy.py index 6ef7b5ca..e721d0db 100644 --- a/src/pinky_daemon/autonomy.py +++ b/src/pinky_daemon/autonomy.py @@ -272,14 +272,23 @@ async def stop_agent_loop(self, agent_name: str) -> None: _log(f"autonomy: stopped work loop for {agent_name}") async def push_event(self, event: AgentEvent) -> None: - """Push an event to an agent's queue. Starts loop if not running.""" + """Push an event to an agent's queue. Starts loop if not running. + + Boot policy (2026-05-11): only the main agent's autonomy loop is started + at daemon boot. Sibling agents start their loop here on demand, the + first time any event (inbound message, agent-to-agent message, or + scheduled wake) arrives. The gate is `enabled` only — the older + `agent.auto_start` flag is no longer consulted at wake time, since + gating wake on `auto_start` would strand siblings unable to receive + messages after the boot-policy change. + """ await self._event_queue.push(event) _log(f"autonomy: event {event.type.value} for {event.agent_name}") - # Auto-start loop if agent has auto_start and loop isn't running + # Wake the loop on first event for any enabled agent. if event.agent_name not in self._running_loops: agent = self._registry.get(event.agent_name) - if agent and agent.auto_start and agent.enabled: + if agent and agent.enabled: await self.start_agent_loop(event.agent_name) async def _agent_loop(self, agent_name: str) -> None: diff --git a/src/pinky_daemon/broker.py b/src/pinky_daemon/broker.py index 7480c9e4..accfc9b1 100644 --- a/src/pinky_daemon/broker.py +++ b/src/pinky_daemon/broker.py @@ -631,6 +631,33 @@ async def handle_inbound(self, message: BrokerMessage) -> None: except Exception: pass + async def dispatch_pre_authorized( + self, agent_name: str, message: BrokerMessage, + ) -> None: + """Dispatch a message whose sender is already authorized upstream. + + Bypasses the human-platform onboarding flow that ``handle_inbound`` + runs (``get_user_status`` → ``add_pending_user`` → ``/approve_…`` + Telegram prompt to the owner). Intended for callers that enforce + their own identity gating before invoking the broker — currently + the ferry host-callback (``host_pinky``), where peer-fleet ACL has + already been enforced. Future pre-authorized channels (federation, + MCP-host inbound) should land on this same primitive rather than + reusing ``handle_inbound``. + + Concretely: routes the message to the agent's streaming session + without consulting ``approved_users``. The agent will see the + message in its prompt feed exactly as if the broker had approved + it via the human-platform flow. + + Activity-log emission is intentionally left to the caller — ferry + inbound has its own observability (host-pinky's stats counters), + and the broker's ``message_received`` event is shaped for human + platforms (sender/preview formatting). If a future caller wants + broker-side activity logs, expose that as a separate flag. + """ + await self._route_streaming(agent_name, message) + def _format_prompt(self, message: BrokerMessage) -> str: """Format a single message as a platform-aware prompt line.""" from datetime import datetime diff --git a/src/pinky_daemon/codex_session.py b/src/pinky_daemon/codex_session.py index f1b771c0..91322932 100644 --- a/src/pinky_daemon/codex_session.py +++ b/src/pinky_daemon/codex_session.py @@ -73,6 +73,7 @@ def __init__( self._analytics_store = analytics_store self._registry = registry self._connected = False + self._idle_sleeping = False self._processing = False # True while a codex exec is running self._message_queue: asyncio.Queue[tuple[str, str, str, str]] = asyncio.Queue() self._worker_task: asyncio.Task | None = None @@ -114,11 +115,17 @@ def __init__( async def connect(self) -> None: """Start the session. Sends wake prompt via codex exec.""" + if self._connected: + self._idle_sleeping = False + return + self._connected = True + self._idle_sleeping = False self._analytics_session_started() # Start the message processing worker - self._worker_task = asyncio.create_task(self._message_worker()) + if not self._worker_task or self._worker_task.done(): + self._worker_task = asyncio.create_task(self._message_worker()) _log(f"codex[{self.agent_name}]: connected, worker started") @@ -833,10 +840,49 @@ async def idle_sleep(self) -> bool: _log(f"codex[{self.agent_name}]: memory save failed before idle sleep: {e}") await self.disconnect() + self._idle_sleeping = True self._stats["auto_restarts"] += 1 _log(f"codex[{self.agent_name}]: idle sleep complete") return True + # Reconnect backoff schedule (seconds). Kept in step with StreamingSession's + # watchdog contract so api._heartbeat_resurrect can treat runtimes uniformly. + _RECONNECT_BACKOFF = (2, 8, 30) + + async def attempt_reconnect(self) -> None: + """Attempt to reconnect after a failure with bounded retries.""" + try: + await self.disconnect() + except Exception as e: + _log(f"codex[{self.agent_name}]: pre-reconnect disconnect raised: {e}") + + last_error: Exception | None = None + for attempt_idx, delay in enumerate(self._RECONNECT_BACKOFF, start=1): + self._stats["reconnects"] += 1 + _log( + f"codex[{self.agent_name}]: reconnect attempt {attempt_idx}/" + f"{len(self._RECONNECT_BACKOFF)} (#{self._stats['reconnects']} total) " + f"after {delay}s backoff" + ) + await asyncio.sleep(delay) + try: + await self.connect() + _log(f"codex[{self.agent_name}]: reconnected successfully") + return + except Exception as e: + last_error = e + _log(f"codex[{self.agent_name}]: reconnect attempt {attempt_idx} failed: {e}") + try: + await self.disconnect() + except Exception: + pass + + self._connected = False + _log( + f"codex[{self.agent_name}]: all {len(self._RECONNECT_BACKOFF)} reconnect " + f"attempts failed (last error: {last_error}); session left disconnected" + ) + async def disconnect(self) -> None: """Disconnect — kill any running subprocess and cancel the worker.""" self._connected = False @@ -866,6 +912,11 @@ async def disconnect(self) -> None: def is_connected(self) -> bool: return self._connected + @property + def is_idle_sleeping(self) -> bool: + """True when disconnected deliberately by idle_sleep().""" + return self._idle_sleeping + @property def max_tokens(self) -> int: """Estimated max context tokens for this session's model.""" @@ -902,6 +953,7 @@ def stats(self) -> dict: return { **self._stats, "connected": self._connected, + "idle_sleeping": self._idle_sleeping, "processing": self._processing, "pending_messages": self._message_queue.qsize(), "current_activity": self._current_activity, diff --git a/src/pinky_daemon/daemon.py b/src/pinky_daemon/daemon.py index 8c565e9f..6853d8c7 100644 --- a/src/pinky_daemon/daemon.py +++ b/src/pinky_daemon/daemon.py @@ -255,12 +255,20 @@ async def start(self) -> None: await self._autonomy.start() _log("daemon: autonomy engine started") - # Start work loops for auto-start agents - auto_start_agents = self._registry.list_auto_start_agents() - for agent in auto_start_agents: - await self._autonomy.start_agent_loop(agent.name) - if auto_start_agents: - _log(f"daemon: started autonomy loops for {len(auto_start_agents)} agent(s)") + # Boot policy (2026-05-11): only the main agent's autonomy loop starts + # at boot. Sibling loops start lazily via `autonomy.push_event` when an + # event arrives for them (inbound message, agent-to-agent message, or + # scheduled wake). + main_name = self._registry.get_main_agent() + if main_name: + main_agent = self._registry.get(main_name) + if main_agent and main_agent.enabled: + await self._autonomy.start_agent_loop(main_name) + _log(f"daemon: started main agent autonomy loop ({main_name})") + else: + _log(f"daemon: warn — main agent '{main_name}' missing or disabled") + else: + _log("daemon: warn — no main agent configured; no autonomy loop started") # Start platform pollers if self._config.telegram_token: diff --git a/src/pinky_daemon/ferry/__init__.py b/src/pinky_daemon/ferry/__init__.py new file mode 100644 index 00000000..bec49edd --- /dev/null +++ b/src/pinky_daemon/ferry/__init__.py @@ -0,0 +1,36 @@ +"""Ferry — federated agent messaging integration. + +`@ferry/host-pinky` Python implementation. Receives ferry envelopes addressed +to a PinkyBot agent and routes them either as inbound platform messages +(via broker.handle_inbound) or as substrate-spec memory imports (via +pinky-memory + pinky-self). + +References: +- ferry PROTOCOL.md v0.1 — envelope shape +- substrate v0.1 — memory interchange spec, §12 worked example +- packages/host-pinky/README.md — design doc + +See PinkyBot issue #413 for the behavior contract and acceptance test plan. +""" + +from pinky_daemon.ferry.types import ( + AgentCardSelector, + DeliveryResult, + FerryEnvelope, + PortHistoryEntry, + SubstrateEntry, + SubstrateLifecycle, + SubstrateScope, + SubstrateSource, +) + +__all__ = [ + "AgentCardSelector", + "DeliveryResult", + "FerryEnvelope", + "SubstrateEntry", + "SubstrateScope", + "SubstrateSource", + "SubstrateLifecycle", + "PortHistoryEntry", +] diff --git a/src/pinky_daemon/ferry/host_pinky.py b/src/pinky_daemon/ferry/host_pinky.py new file mode 100644 index 00000000..16f01cc1 --- /dev/null +++ b/src/pinky_daemon/ferry/host_pinky.py @@ -0,0 +1,570 @@ +"""@ferry/host-pinky — PinkyBot host-callback for ferry. + +Receives ferry envelopes addressed to a PinkyBot agent and routes them +either as inbound platform messages (via broker.handle_inbound) or as +substrate-spec memory imports (via pinky-memory + pinky-self). + +The behavior contract is documented in PinkyBot issue #413 and mirrored +in ferry repo's packages/host-pinky/README.md. + +This module is the **adapter** — it does no transport (NATS / leaf-link +plumbing is owned by ferry-core) and does no model invocation (broker + +streaming sessions handle that). host-pinky's job is to translate one +identity primitive (signed ferry agent cards) into another (PinkyBot +agent + chat_id), enforce the peer-fleet ACL on the way in, and keep +substrate's port_history canonicality invariant on inbound (§6.4). +""" + +from __future__ import annotations + +import json +import sqlite3 +import sys +from collections.abc import Callable +from dataclasses import asdict +from typing import Any + +from pinky_daemon.ferry.substrate import ( + classify_entry_destination, + populate_port_history, + to_reflection_input, + to_task_input, +) +from pinky_daemon.ferry.types import ( + AgentCardSelector, + DeliveryResult, + FerryEnvelope, + IngestResult, + SubstrateEntry, +) + + +class _TransientACLLoadError(Exception): + """Raised by ``_load_peer_fleet_acl`` when the registry read failed for + operator-side reasons (DB lock, schema mismatch during rolling deploy, + etc.). Distinct from "ACL is empty" so the broker can replay rather + than treat the message as policy-denied. Internal to host_pinky. + """ + + +def _log(msg: str) -> None: + print(f"host-pinky: {msg}", file=sys.stderr, flush=True) + + +# -- Address parsing ----------------------------------------------------------- + +_FERRY_PINKY_PREFIX = "ferry://pinkybot/" + + +def parse_pinkybot_address(addr: str) -> str | None: + """Extract the agent name from a ferry://pinkybot/ address. + + Returns the agent name on success, or None if the address is not + pointed at this fleet. + """ + if not addr: + return None + if not addr.startswith(_FERRY_PINKY_PREFIX): + return None + name = addr[len(_FERRY_PINKY_PREFIX):].strip() + return name or None + + +def parse_peer_card(addr: str) -> tuple[str, str]: + """Parse a peer agent's address into (fleet, agent_id). + + Examples: + pulse@studio@sigil → ("sigil", "pulse@studio@sigil") + ferry://sigil/pulse → ("sigil", "ferry://sigil/pulse") + misha@pinky.local → ("pinky.local", "misha@pinky.local") + """ + if addr.startswith("ferry://"): + rest = addr[len("ferry://"):] + parts = rest.split("/", 1) + fleet = parts[0] if parts else "" + return (fleet, addr) + # at-form: name@(scope@)?fleet — rightmost segment is the fleet + parts = addr.split("@") + if len(parts) >= 2: + return (parts[-1], addr) + return ("", addr) + + +# -- HostPinky ----------------------------------------------------------------- + + +class HostPinky: + """Per-daemon host-callback. Constructed once at daemon startup. + + Required collaborators: + registry — AgentRegistry (for agent lookup + peer_fleet_acl read) + broker — MessageBroker (for handle_inbound) + memory_store — pinky-memory MemoryStore (for substrate→reflection insert) + task_store — pinky-self TaskStore (for substrate→pending→create_task) + + The collaborators are passed by reference; HostPinky does not own them. + """ + + def __init__( + self, + *, + registry: Any, + broker: Any, + memory_store: Any | None = None, + task_store: Any | None = None, + mesh_store: Any | None = None, + verify_signatures: bool = False, + reflection_factory: Callable[[dict[str, Any]], Any] | None = None, + ) -> None: + """Construct a HostPinky. + + ``reflection_factory`` (optional): callable that builds a Reflection + from a kwargs dict. Defaults to ``pinky_memory.types.Reflection`` when + unset. Inject in tests to avoid the pinky_memory import cost and to + substitute a stand-in shape; production callers leave it as ``None``. + + ``mesh_store`` (optional): a ``MeshStore`` for inbound envelope + audit + federation peer auto-registration (see #419). When ``None``, + delivery proceeds without persistence. + """ + self._registry = registry + self._broker = broker + self._memory_store = memory_store + self._task_store = task_store + self._mesh_store = mesh_store + self._verify_signatures = verify_signatures + self._reflection_factory = reflection_factory + self._stats: dict[str, int] = { + "delivered": 0, + "rejected": 0, + "queued": 0, + "transient_failures": 0, + "substrate_imports": 0, + "messages_routed": 0, + } + + @property + def stats(self) -> dict[str, int]: + return dict(self._stats) + + # -- Main entry point ------------------------------------------------------ + + async def deliver(self, envelope: FerryEnvelope) -> DeliveryResult: + """Deliver a ferry envelope to a PinkyBot agent. + + Implements the four-step contract from the host-pinky design doc: + 1. Verify ferry envelope signatures + 2. Resolve target agent + 3. Peer-fleet ACL check + 4. Dispatch by body.kind + + Side effect: if a ``mesh_store`` was passed at construction time, + every call here logs one ``mesh_messages`` row with the final + disposition (success or rejection reason). + """ + result = await self._deliver_core(envelope) + self._log_inbound_safely(envelope, result) + return result + + async def _deliver_core(self, envelope: FerryEnvelope) -> DeliveryResult: + """The main deliver pipeline. ``deliver()`` wraps this with logging.""" + # 1. Signature verification + if self._verify_signatures: + verify_err = self._verify(envelope) + if verify_err: + return self._reject("auth_failed", verify_err) + + # 2. Resolve target agent + agent_name = parse_pinkybot_address(envelope.to) + if not agent_name: + return self._reject( + "unknown_agent", + f"address {envelope.to!r} is not a PinkyBot ferry address", + ) + if not self._agent_exists(agent_name): + return self._reject("unknown_agent", f"no agent named {agent_name!r}") + + # 3. Peer-fleet ACL check + try: + allowed = self._acl_allows(agent_name, envelope.from_) + except _TransientACLLoadError as e: + # Operator-side hiccup (DB lock, schema mismatch during rolling + # deploy). Not a policy denial — the broker should retry rather + # than terminally drop a message that an ACL would have allowed. + return self._transient( + "acl_load_failed", + f"could not load peer_fleet_acl for {agent_name}: {e}", + ) + if not allowed: + return self._reject( + "acl_denied", + f"peer {envelope.from_!r} not on {agent_name}'s peer_fleet_acl", + ) + + # 4. Dispatch by payload kind + kind = envelope.kind + if kind == "message": + return await self._deliver_message(agent_name, envelope) + if kind in ("substrate.entry", "substrate.batch"): + return await self._deliver_substrate(agent_name, envelope) + + return self._reject("unsupported_payload_kind", f"kind={kind!r}") + + def _log_inbound_safely( + self, + envelope: FerryEnvelope, + result: DeliveryResult, + ) -> None: + """Best-effort log + peer-touch. Never propagates persistence errors — + delivery semantics must not depend on whether mesh persistence is up.""" + if self._mesh_store is None: + return + try: + local_agent = parse_pinkybot_address(envelope.to) or "" + peer_fleet, peer_agent_id = parse_peer_card(envelope.from_) + error: str | None = None + if result.status in ("rejected", "transient_failure"): + error = f"{result.status}:{result.reason or ''}" + self._mesh_store.log_message( + direction="inbound", + local_agent=local_agent, + remote_fleet=peer_fleet, + remote_agent=peer_agent_id, + correlation_id=envelope.correlation_id or "", + kind=envelope.kind, + body=envelope.body if isinstance(envelope.body, dict) else {"_raw": envelope.body}, + envelope_ts=envelope.ts, + reply_to=envelope.reply_to, + error=error, + ) + except Exception as e: # noqa: BLE001 — defensive: persistence must not break delivery + _log(f"mesh_store inbound log failed (non-fatal): {e}") + + # -- Signature verification (deferred for v0.1) --------------------------- + + def _verify(self, envelope: FerryEnvelope) -> str | None: + """Verify envelope.from_ + each traversal record's signature. + + Returns None on success, or a human-readable error string. + + v0.1: stub — accept all. Real implementation lands with the + signed-agent-card subsystem in v0.2/v0.3 (per Pulse §12.3 thread). + """ + # TODO(v0.2): real Ed25519 signature verification. + return None + + # -- Agent lookup --------------------------------------------------------- + + def _agent_exists(self, agent_name: str) -> bool: + try: + return self._registry.has_agent(agent_name) + except AttributeError: + # Fallback: most AgentRegistry impls expose either has_agent or list_agents. + try: + names = {a.get("name") for a in self._registry.list_agents()} + return agent_name in names + except Exception: + return False + + # -- Peer-fleet ACL ------------------------------------------------------- + + def _acl_allows(self, agent_name: str, peer_address: str) -> bool: + """Check peer_fleet_acl for the receiving agent. + + Default-deny: empty/missing ACL means no peer-fleet inbound allowed. + """ + selectors = self._load_peer_fleet_acl(agent_name) + if not selectors: + return False + peer_fleet, peer_agent_id = parse_peer_card(peer_address) + for sel in selectors: + if sel.matches(peer_fleet, peer_agent_id, ""): + return True + return False + + def _load_peer_fleet_acl(self, agent_name: str) -> list[AgentCardSelector]: + """Load the agent's peer_fleet_acl selectors. + + Returns [] when the ACL is unset (policy: empty list = deny all). + + Raises ``_TransientACLLoadError`` when the registry read fails for + operator-side reasons (DB lock, schema mismatch). The caller in + ``deliver()`` translates that into a ``transient_failure`` delivery + result so the broker retries instead of treating the message as + terminally rejected. + + Per-item parse errors (malformed selector dict, invalid field + combinations) are skipped silently — they don't taint the rest of + the ACL or trigger a transient. + """ + try: + raw = self._registry.get_peer_fleet_acl(agent_name) + except AttributeError: + # Registry implementation doesn't expose peer_fleet_acl — older + # registry shape. Treat as empty (deny). Not transient. + return [] + except sqlite3.Error as e: + raise _TransientACLLoadError(str(e)) from e + if not raw: + return [] + out: list[AgentCardSelector] = [] + for item in raw: + try: + if isinstance(item, AgentCardSelector): + out.append(item) + elif isinstance(item, dict): + out.append(AgentCardSelector(**item)) + except (TypeError, ValueError, json.JSONDecodeError) as e: + _log(f"skipping invalid ACL entry for {agent_name}: {item} ({e})") + return out + + # -- Dispatch: message → broker.handle_inbound ---------------------------- + + async def _deliver_message( + self, + agent_name: str, + envelope: FerryEnvelope, + ) -> DeliveryResult: + """Route a ferry message envelope to the agent's streaming session. + + Identity-primitive separation: peer-fleet ACL was enforced + upstream in this module's ``_acl_allows`` step. The peer-fleet + identity (``ferry:``) is **not** written into the + receiving agent's human-platform ``approved_users`` table — the + two identity primitives stay disjoint at the broker boundary. + + Concretely: we call ``broker.dispatch_pre_authorized``, which + routes the message straight to the agent's streaming session, + bypassing ``handle_inbound``'s human-onboarding flow + (``get_user_status`` → ``add_pending_user`` → ``/approve_…`` + Telegram prompt). That bypass is what makes the dedicated-path + claim load-bearing-true at the broker surface. + """ + from pinky_daemon.broker import BrokerMessage # local import to avoid cycles + + text = str(envelope.body.get("text") or envelope.body.get("content") or "").strip() + if not text: + return self._reject("empty_message_body", "body.text was empty") + + peer_address = envelope.from_ + broker_msg = BrokerMessage( + platform="ferry", + chat_id=peer_address, + sender_name=peer_address, + sender_id=f"ferry:{peer_address}", + content=text, + agent_name=agent_name, + timestamp=envelope.ts / 1000.0, + message_id=envelope.id, + chat_title=envelope.subject or "", + is_group=False, + metadata={ + "ferry": { + "envelope_id": envelope.id, + "from": envelope.from_, + "to": envelope.to, + "ts": envelope.ts, + "wake": envelope.wake, + "ack": envelope.ack, + "correlation_id": envelope.correlation_id, + "reply_to": envelope.reply_to, + "headers": dict(envelope.headers), + "traversal": [asdict(t) for t in envelope.traversal], + } + }, + ) + + try: + await self._broker.dispatch_pre_authorized(agent_name, broker_msg) + except Exception as e: + _log( + f"broker.dispatch_pre_authorized failed for ferry envelope " + f"{envelope.id}: {e}" + ) + return DeliveryResult(status="rejected", reason="broker_error", detail={"error": str(e)}) + + self._stats["delivered"] += 1 + self._stats["messages_routed"] += 1 + return DeliveryResult(status="delivered") + + # -- Dispatch: substrate.entry / substrate.batch -------------------------- + + async def _deliver_substrate( + self, + agent_name: str, + envelope: FerryEnvelope, + ) -> DeliveryResult: + """Import substrate entries into the receiving agent's stores.""" + entries = self._extract_substrate_entries(envelope) + if not entries: + return self._reject("empty_substrate_payload", "no entries in body") + + receiving_address = f"ferry://pinkybot/{agent_name}" + results: list[IngestResult] = [] + for entry in entries: + populated = populate_port_history(entry, envelope, receiving_address) + try: + result = self._ingest_one(agent_name, populated) + except Exception as e: + _log(f"ingest failed for substrate {entry.id}: {e}") + results.append( + IngestResult( + substrate_id=entry.id, + target_store="skipped", + note=f"error: {e}", + ) + ) + continue + results.append(result) + + self._stats["delivered"] += 1 + self._stats["substrate_imports"] += sum(1 for r in results if r.target_store != "skipped") + return DeliveryResult( + status="delivered", + detail={"ingested": [asdict(r) for r in results]}, + ) + + def _ingest_one(self, agent_name: str, entry: SubstrateEntry) -> IngestResult: + """Route a single entry to its destination store.""" + destination = classify_entry_destination(entry) + + if destination == "pinky-memory": + if self._memory_store is None: + return IngestResult( + substrate_id=entry.id, + target_store="skipped", + note="memory_store not configured", + ) + ref_kwargs = to_reflection_input(entry) + ref_kwargs["project"] = agent_name + inserted = self._insert_reflection(ref_kwargs) + return IngestResult( + substrate_id=entry.id, + target_store="pinky-memory", + target_id=str(inserted), + ) + + if destination == "pinky-self": + if self._task_store is None: + return IngestResult( + substrate_id=entry.id, + target_store="skipped", + note="task_store not configured", + ) + task_kwargs = to_task_input(entry) + task_kwargs["assigned_agent"] = agent_name + task_kwargs["created_by"] = entry.source.by + inserted = self._insert_task(task_kwargs) + return IngestResult( + substrate_id=entry.id, + target_store="pinky-self", + target_id=str(inserted), + ) + + return IngestResult( + substrate_id=entry.id, + target_store="skipped", + note=f"destination={destination}", + ) + + def _insert_reflection(self, ref_kwargs: dict[str, Any]) -> str: + """Insert into pinky-memory. Returns the inserted reflection id. + + Uses the configured ``reflection_factory`` if provided (test-only + injection point), otherwise falls back to ``pinky_memory.types.Reflection``. + """ + if self._reflection_factory is not None: + reflection = self._reflection_factory(ref_kwargs) + else: + # Local import: pinky_memory is an optional dep at call time so + # ferry tests don't have to install it. + from pinky_memory.types import Reflection + reflection = Reflection(**ref_kwargs) + result = self._memory_store.insert(reflection) + return getattr(result, "id", "") or getattr(reflection, "id", "") + + def _insert_task(self, task_kwargs: dict[str, Any]) -> str: + """Insert into pinky-self task store. Returns task id (as str). + + ``task_store.create_task`` may return either a dict (older signature) + or an object with ``.id`` (newer signature). Handle both explicitly + rather than via a ternary that becomes ambiguous when ``.id`` is + empty (the prior implementation would stringify the entire object + as a "task id" in that case — see PR #414 round 2 finding #2). + """ + result = self._task_store.create_task(**task_kwargs) + if isinstance(result, dict): + return str(result.get("id", "")) + return str(getattr(result, "id", "") or "") + + # -- substrate envelope unwrapping ---------------------------------------- + + def _extract_substrate_entries(self, envelope: FerryEnvelope) -> list[SubstrateEntry]: + """Pull SubstrateEntry objects out of envelope.body. + + Supports both kinds: + body.kind == "substrate.entry" → body.entry is one entry-shaped dict + body.kind == "substrate.batch" → body.entries is a list of entry-shaped dicts + """ + from pinky_daemon.ferry.types import ( + PortHistoryEntry, + SubstrateLifecycle, + SubstrateScope, + SubstrateSource, + ) + + kind = envelope.kind + raw_entries: list[dict[str, Any]] = [] + if kind == "substrate.entry": + entry = envelope.body.get("entry") + if isinstance(entry, dict): + raw_entries.append(entry) + elif kind == "substrate.batch": + entries = envelope.body.get("entries") + if isinstance(entries, list): + raw_entries = [e for e in entries if isinstance(e, dict)] + + out: list[SubstrateEntry] = [] + for raw in raw_entries: + try: + out.append( + SubstrateEntry( + id=str(raw.get("id", "")), + type=raw.get("type", "fact"), + scope=SubstrateScope(**raw["scope"]), + source=SubstrateSource(**raw["source"]), + created_at=str(raw.get("created_at", "")), + updated_at=str(raw.get("updated_at", "")), + content=str(raw.get("content", "")), + links=list(raw.get("links") or []), + lifecycle=SubstrateLifecycle(**(raw.get("lifecycle") or {})), + port_history=[ + PortHistoryEntry(**ph) if isinstance(ph, dict) else ph + for ph in (raw.get("port_history") or []) + ], + ) + ) + except Exception as e: + _log(f"skipping malformed substrate entry: {e}") + return out + + # -- Helpers -------------------------------------------------------------- + + def _reject(self, reason: str, detail_text: str) -> DeliveryResult: + self._stats["rejected"] += 1 + _log(f"reject reason={reason}: {detail_text}") + return DeliveryResult( + status="rejected", + reason=reason, + detail={"detail": detail_text}, + ) + + def _transient(self, reason: str, detail_text: str) -> DeliveryResult: + """Return a ``transient_failure`` result. Broker should retry.""" + self._stats["transient_failures"] += 1 + _log(f"transient_failure reason={reason}: {detail_text}") + return DeliveryResult( + status="transient_failure", + reason=reason, + detail={"detail": detail_text}, + ) diff --git a/src/pinky_daemon/ferry/outbound.py b/src/pinky_daemon/ferry/outbound.py new file mode 100644 index 00000000..5aaabb42 --- /dev/null +++ b/src/pinky_daemon/ferry/outbound.py @@ -0,0 +1,363 @@ +"""@ferry/host-pinky — outbound side: build + publish ferry envelopes. + +Companion to host_pinky.py (inbound). Where host_pinky.py receives envelopes +and routes them into PinkyBot, this module builds envelopes from PinkyBot +agents and publishes them onto the ferry transport. + +Design choice — shellout, not in-process NATS client: + host_pinky.py's docstring is explicit that the Python module is the + **adapter** and "does no transport (NATS / leaf-link plumbing is owned + by ferry-core)". To keep that boundary clean on the outbound side, we + shell out to the `nats` CLI (https://github.com/nats-io/natscli) rather + than introduce nats-py as a dep. One-shot publishes are the only outbound + pattern we need; the CLI matches that shape and we already proved it + works end-to-end (Barsik smoke 2026-05-10 12:14 PDT, sub-2s). + +If we later need a long-lived in-process publisher (e.g. for high-throughput +streaming), this module's surface (`MeshSender.send`) lets us swap the +shellout for an async nats-py client without touching callers. + +References: +- ferry PROTOCOL.md v0.1 §1 — envelope schema (mirrored by ferry/types.py) +- PinkyBot issue #418 — outbound mesh_remote_send tool +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import time +import uuid +from dataclasses import dataclass +from typing import Any + +from pinky_daemon.ferry.types import FerryEnvelope + +# -- Address parsing ----------------------------------------------------------- + +_FERRY_SCHEME = "ferry://" + + +def parse_address(addr: str) -> tuple[str, str] | None: + """Parse a ferry address into ``(fleet, agent_slug)``. + + Accepts two forms: + - ``ferry:///`` — preferred canonical form + - ``@`` — at-form for casual use + + Returns ``None`` for malformed input. ``agent_slug`` may itself contain + ``@`` (e.g. for scoped peer addresses like ``pulse@studio@sigil``); only + the rightmost ``@`` is the fleet boundary. + """ + if not addr or not isinstance(addr, str): + return None + s = addr.strip() + if not s: + return None + + if s.startswith(_FERRY_SCHEME): + rest = s[len(_FERRY_SCHEME):] + parts = rest.split("/", 1) + if len(parts) != 2: + return None + fleet, agent_slug = parts[0].strip(), parts[1].strip() + if not fleet or not agent_slug: + return None + return (fleet, agent_slug) + + # at-form: rightmost @ is the fleet boundary + if "@" not in s: + return None + head, _, fleet = s.rpartition("@") + head, fleet = head.strip(), fleet.strip() + if not head or not fleet: + return None + return (fleet, head) + + +def _sanitize_token(s: str) -> str: + """Make a string safe to use as a NATS subject token. + + NATS subjects are dot-separated; ``.`` and ``@`` are replaced with ``-`` + so each segment occupies exactly one token. Sanitization is reversible + at the application layer via the envelope's ``from`` / ``to`` fields, + which preserve the original address forms. + """ + return s.replace(".", "-").replace("@", "-") + + +def derive_subject(fleet: str, agent_slug: str) -> str: + """NATS subject for a ferry envelope addressed to ``agent_slug@fleet``. + + Convention: ``ferry...inbox``. Both ``fleet`` and + ``agent_slug`` are sanitized — see ``_sanitize_token``. + + The ``ferry.>`` prefix matches the ``pinkybot_fleet`` ACL granted to + PinkyBot on Nova (confirmed by Pulse 2026-05-10). + """ + return f"ferry.{_sanitize_token(fleet)}.{_sanitize_token(agent_slug)}.inbox" + + +# -- Allowlist matching -------------------------------------------------------- + + +def allowlist_matches(target: str, allowlist: list[str]) -> bool: + """True if ``target`` matches any pattern in ``allowlist``. + + Patterns: + - exact match (e.g. ``pulse@pulse``) + - wildcard agent: ``*@`` (any agent on the fleet) + - wildcard fleet: ``@*`` (any fleet for that agent slug) + - global: ``*@*`` or ``*`` (any target — use sparingly) + + Empty allowlist means default-deny. Wildcards do NOT match across + address forms — pass canonical form (at-form) only. + """ + if not allowlist: + return False + parsed_target = parse_address(target) + if parsed_target is None: + return False + t_fleet, t_agent = parsed_target + for pat in allowlist: + if not pat: + continue + p = pat.strip() + if p in ("*", "*@*"): + return True + parsed_pat = parse_address(p) + if parsed_pat is None: + continue + p_fleet, p_agent = parsed_pat + fleet_ok = p_fleet == "*" or p_fleet == t_fleet + agent_ok = p_agent == "*" or p_agent == t_agent + if fleet_ok and agent_ok: + return True + return False + + +# -- Envelope builder ---------------------------------------------------------- + + +def build_envelope( + *, + from_: str, + to: str, + body: dict[str, Any], + correlation_id: str | None = None, + reply_to: str | None = None, + priority: str = "normal", + headers: dict[str, str] | None = None, +) -> FerryEnvelope: + """Build a canonical ferry envelope per PROTOCOL.md v0.1 §1. + + ``body`` MUST contain a ``kind`` field (per spec). Caller is responsible + for setting it; this function does not infer it. + + ``id`` is a uuid (v4 for now; spec calls for uuid-v7 — followup once + Python stdlib lands it in 3.13+ or we add the ``uuid-utils`` dep). + """ + if not isinstance(body, dict) or "kind" not in body: + raise ValueError("envelope.body must be a dict containing a 'kind' field") + return FerryEnvelope( + v="0.1", + id=str(uuid.uuid4()), + from_=from_, + to=to, + ts=int(time.time() * 1000), + body=dict(body), + priority=priority if priority in ("low", "normal", "high", "urgent") else "normal", + correlation_id=correlation_id, + reply_to=reply_to, + headers=dict(headers or {}), + ) + + +def envelope_to_wire(env: FerryEnvelope) -> str: + """Serialize an envelope to its on-wire JSON form. + + Renames ``from_`` → ``from`` (Python keyword workaround), drops + default-valued optional fields to keep wire compact. + """ + payload: dict[str, Any] = { + "v": env.v, + "id": env.id, + "from": env.from_, + "to": env.to, + "ts": env.ts, + "body": env.body, + } + if env.priority != "normal": + payload["priority"] = env.priority + if env.wake != "if-idle": + payload["wake"] = env.wake + if env.ack != "delivery": + payload["ack"] = env.ack + if env.ttl_ms is not None: + payload["ttl_ms"] = env.ttl_ms + if env.correlation_id: + payload["correlation_id"] = env.correlation_id + if env.reply_to: + payload["reply_to"] = env.reply_to + if env.subject: + payload["subject"] = env.subject + if env.traversal: + # FerryEnvelope.traversal is List[TraversalRecord] dataclasses; + # serialize each via __dict__ (small, flat, no nested dataclasses). + payload["traversal"] = [t.__dict__ for t in env.traversal] + if env.headers: + payload["headers"] = env.headers + return json.dumps(payload, separators=(",", ":"), ensure_ascii=False) + + +# -- Sender -------------------------------------------------------------------- + + +@dataclass +class SendResult: + """Outcome of a single ``MeshSender.send`` call.""" + + sent: bool + correlation_id: str | None + subject: str + ts: int + error: str | None = None + + +class MeshSender: + """Daemon-owned outbound publisher. + + One per-daemon instance, constructed at startup. Reads NATS credentials + and server URL from the environment; never accepts them as arguments + so they don't leak into agent context, request bodies, or logs. + + Environment variables: + PINKYBOT_FLEET_USER — NATS user (e.g. "pinkybot_fleet") + PINKYBOT_FLEET_PASSWORD — NATS password (treated as a secret) + PINKYBOT_FLEET_NATS_URL — server URL (default wss://fleet-nats.kostverse.com) + PINKYBOT_FERRY_NATS_BIN — path to nats CLI (default looks up "nats" on PATH) + """ + + DEFAULT_NATS_URL = "wss://fleet-nats.kostverse.com" + DEFAULT_TIMEOUT_S = 10.0 + + def __init__( + self, + *, + env: dict[str, str] | None = None, + bin_path: str | None = None, + timeout_s: float = DEFAULT_TIMEOUT_S, + ) -> None: + e = env if env is not None else dict(os.environ) + self._user = e.get("PINKYBOT_FLEET_USER", "").strip() + self._password = e.get("PINKYBOT_FLEET_PASSWORD", "").strip() + self._nats_url = e.get("PINKYBOT_FLEET_NATS_URL", self.DEFAULT_NATS_URL).strip() + self._bin_path = ( + bin_path + or e.get("PINKYBOT_FERRY_NATS_BIN", "").strip() + or shutil.which("nats") + or "" + ) + self._timeout_s = timeout_s + + @property + def configured(self) -> bool: + """True iff creds + URL + nats CLI binary are all available.""" + return bool(self._user and self._password and self._nats_url and self._bin_path) + + def diagnostics(self) -> dict[str, Any]: + """Operator-facing health snapshot. Never includes the password.""" + return { + "configured": self.configured, + "user_set": bool(self._user), + "password_set": bool(self._password), + "nats_url": self._nats_url, + "nats_bin": self._bin_path, + } + + def send(self, envelope: FerryEnvelope) -> SendResult: + """Publish an envelope to its derived subject. Synchronous. + + Failure modes are returned as ``SendResult(sent=False, error=...)`` + rather than raised — callers (the API endpoint) want a structured + response either way. + """ + ts = envelope.ts + cid = envelope.correlation_id + + if not self.configured: + return SendResult( + sent=False, + correlation_id=cid, + subject="", + ts=ts, + error="mesh sender not configured (missing creds, URL, or nats CLI)", + ) + + parsed = parse_address(envelope.to) + if parsed is None: + return SendResult( + sent=False, + correlation_id=cid, + subject="", + ts=ts, + error=f"unparseable target address: {envelope.to!r}", + ) + fleet, agent_slug = parsed + subject = derive_subject(fleet, agent_slug) + wire = envelope_to_wire(envelope) + + cmd = [ + self._bin_path, + "--user", self._user, + "--password", self._password, + "-s", self._nats_url, + "pub", subject, wire, + ] + try: + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=self._timeout_s, + check=False, + ) + except subprocess.TimeoutExpired: + return SendResult( + sent=False, + correlation_id=cid, + subject=subject, + ts=ts, + error=f"nats CLI timed out after {self._timeout_s}s", + ) + except OSError as exc: + return SendResult( + sent=False, + correlation_id=cid, + subject=subject, + ts=ts, + error=f"nats CLI invocation failed: {exc}", + ) + + if proc.returncode != 0: + # stderr may contain auth/ACL detail — include but do NOT + # include the original command (which has the password). + err_tail = (proc.stderr or proc.stdout or "").strip().splitlines() + err_msg = err_tail[-1] if err_tail else f"nats CLI exit {proc.returncode}" + return SendResult( + sent=False, + correlation_id=cid, + subject=subject, + ts=ts, + error=err_msg, + ) + + return SendResult( + sent=True, + correlation_id=cid, + subject=subject, + ts=ts, + error=None, + ) diff --git a/src/pinky_daemon/ferry/substrate.py b/src/pinky_daemon/ferry/substrate.py new file mode 100644 index 00000000..323ca7ce --- /dev/null +++ b/src/pinky_daemon/ferry/substrate.py @@ -0,0 +1,270 @@ +"""substrate v0.1 → PinkyBot stores translation. + +Implements the five host-side unwrap steps from substrate v0.1 §12.5: + 1. Verify ferry envelope (deferred — see host_pinky.deliver()) + 2. Populate port_history from envelope.traversal (§6.4 canonicality) + 3. Preserve source.by verbatim (§6.1 — never overwrite with immediate sender) + 4. Type-map per §11 reference impl notes (host-implementation-defined) + 5. Index rebuild (handled by pinky-memory's normal insert path) + +Type mapping (host-pinky's choice, not the spec's): + fact / event / reference → pinky-memory `fact` + feedback / decision → pinky-memory `insight` + pattern → pinky-memory `interaction_pattern` + pending → pinky-self `create_task` + +Trust → salience: salience = 1 + round(trust * 4) clamped [1, 5] + (one valid mapping; not the mapping. Documented in host-pinky README.) + +Links: pinky-memory has no native cross-entry edge surface, so substrate +`links` are preserved in `context` JSON sidecar. v0.2 §3.2.1 "no-link-surface +pattern" reference recipe will formalize the recall-time fan-out helper. +""" + +from __future__ import annotations + +import json +import sys +from dataclasses import asdict +from datetime import datetime, timezone +from typing import Any + +from pinky_daemon.ferry.types import ( + FerryEnvelope, + PortHistoryEntry, + SubstrateEntry, +) + + +def _log(msg: str) -> None: + print(f"host-pinky/substrate: {msg}", file=sys.stderr, flush=True) + + +# -- §11 type mapping ---------------------------------------------------------- + +# substrate.type → pinky-memory ReflectionType.value +_PINKY_MEMORY_TYPE_MAP: dict[str, str] = { + "fact": "fact", + "event": "fact", + "reference": "fact", + "feedback": "insight", + "decision": "insight", + "pattern": "interaction_pattern", + # "pending" is special-cased — routes to pinky-self, not pinky-memory +} + + +def map_substrate_type_to_reflection(substrate_type: str) -> str | None: + """Return the pinky-memory ReflectionType.value for a substrate type, or None.""" + return _PINKY_MEMORY_TYPE_MAP.get(substrate_type) + + +def trust_to_salience(trust: float) -> int: + """Trust [0.0, 1.0] → pinky-memory salience [1, 5]. + + Formula: 1 + round(trust * 4), clamped [1, 5]. + + This is host-pinky's mapping, not substrate's. Substrate stays + trust-opaque per §6 — hosts decide how to discretize. + """ + if trust <= 0: + return 1 + if trust >= 1: + return 5 + return max(1, min(5, 1 + round(trust * 4))) + + +# -- port_history population (§6.4) ------------------------------------------- + + +def populate_port_history( + entry: SubstrateEntry, + envelope: FerryEnvelope, + receiving_agent_address: str, +) -> SubstrateEntry: + """Append one port_history entry per ferry traversal hop. + + Per substrate v0.1 §6.4: substrate's port_history is canonical; + ferry's traversal is transport metadata. The receiving host walks + the traversal array on inbound and writes one port_history record + per hop, preserving from/to/at/via. + + Mutates and returns the entry. + """ + if not envelope.traversal: + # First port — no traversal recorded. Emit a single hop from + # envelope.from_ → receiving agent so port_history reflects this + # crossing even when traversal was empty (single-broker delivery). + # `attested_by="receiver"` because no broker stamped this hop — + # the receiver's clock is the only witness on `at`. + entry.port_history.append( + PortHistoryEntry( + from_=envelope.from_, + to=receiving_agent_address, + at=_iso_now(), + via=envelope.id, + re_grounded=False, + attested_by="receiver", + ) + ) + return entry + + prior_address = envelope.from_ + for hop in envelope.traversal: + # Broker-attested: the broker stamped `at` when receiving the hop. + entry.port_history.append( + PortHistoryEntry( + from_=prior_address, + to=hop.broker, + at=_ms_to_iso(hop.at), + via=hop.via or envelope.id, + re_grounded=False, + attested_by="broker", + ) + ) + prior_address = hop.broker + # Final hop: last broker → receiving agent. Receiver-attested because + # no broker stamped this hop (last broker stamped its own arrival, + # not its handoff to this receiver). + entry.port_history.append( + PortHistoryEntry( + from_=prior_address, + to=receiving_agent_address, + at=_iso_now(), + via=envelope.id, + re_grounded=False, + attested_by="receiver", + ) + ) + return entry + + +# -- pinky-memory storage shape ----------------------------------------------- + + +def to_reflection_input(entry: SubstrateEntry) -> dict[str, Any]: + """Build a pinky-memory Reflection-shaped dict for a substrate entry. + + Returned dict is suitable for `MemoryStore.insert(Reflection(**d))`. + Caller is responsible for resolving the project (typically the agent name). + + Keys produced: + - type: pinky-memory ReflectionType.value + - content: substrate entry content (markdown allowed) + - context: JSON-encoded substrate sidecar (provenance, links, port_history) + - salience: derived from trust + - entities: [scope.ref] when scope.kind in {user, agent} + - source_session_id: derived from source.by (for traceability) + """ + reflection_type = map_substrate_type_to_reflection(entry.type) + if reflection_type is None: + raise ValueError( + f"Substrate type {entry.type!r} does not map to a pinky-memory " + "ReflectionType. (Did you mean to route 'pending' to pinky-self?)" + ) + + sidecar = { + "substrate_id": entry.id, + "substrate_type": entry.type, + "scope": asdict(entry.scope), + "source": asdict(entry.source), + "lifecycle": asdict(entry.lifecycle), + "links": list(entry.links), + "port_history": [asdict(p) for p in entry.port_history], + "created_at": entry.created_at, + "updated_at": entry.updated_at, + } + + entities: list[str] = [] + if entry.scope.kind in ("user", "agent") and entry.scope.ref: + entities.append(entry.scope.ref) + + return { + "type": reflection_type, + "content": entry.content, + "context": json.dumps(sidecar, ensure_ascii=False, sort_keys=True), + "salience": trust_to_salience(entry.source.trust), + "entities": entities, + "source_channel": "ferry", + "source_session_id": f"ferry:{entry.source.by}", + } + + +# -- pinky-self task shape (for `pending` entries) ---------------------------- + + +def to_task_input(entry: SubstrateEntry) -> dict[str, Any]: + """Build a pinky-self task-shaped dict for a substrate `pending` entry. + + Lifecycle mapping (substrate → pinky-self task): + pending + expected_resolution_by → pending (with deps if known) + pending + no resolution date → pending + pending + state=active(?) → in_progress (rare on import) + """ + if entry.type != "pending": + raise ValueError( + f"to_task_input called on non-pending entry (type={entry.type!r})" + ) + + title = entry.content.split("\n", 1)[0].strip()[:120] or "(unnamed substrate pending)" + description_parts = [ + entry.content, + "", + "---", + f"Imported from substrate (id: {entry.id})", + f"Author: {entry.source.by} (origin: {entry.source.origin}, trust: {entry.source.trust})", + ] + if entry.lifecycle.expected_resolution_by: + description_parts.append( + f"Expected resolution: {entry.lifecycle.expected_resolution_by}" + ) + if entry.source.evidence: + description_parts.append(f"Evidence: {entry.source.evidence}") + + return { + "title": title, + "description": "\n".join(description_parts), + "priority": _priority_from_trust(entry.source.trust), + "tags": ["substrate", "substrate-pending", f"trust:{entry.source.trust:.2f}"], + # Caller adds project / assigned_agent based on receiving agent. + } + + +def _priority_from_trust(trust: float) -> str: + """Map trust → task priority. Higher trust = higher priority.""" + if trust >= 0.85: + return "high" + if trust >= 0.5: + return "normal" + return "low" + + +# -- Top-level ingest dispatcher ---------------------------------------------- + + +def classify_entry_destination(entry: SubstrateEntry) -> str: + """Return one of: 'pinky-memory', 'pinky-self', 'skipped'. + + `pending` → pinky-self (cross-MCP integration, lifecycle-shaped). + `decayed` lifecycle → skipped (per §5.1, the decay event imports + as its own `event`-type entry, not via the decayed entry itself). + Everything else → pinky-memory. + """ + if entry.lifecycle.state == "decayed": + return "skipped" + if entry.type == "pending": + return "pinky-self" + if map_substrate_type_to_reflection(entry.type) is not None: + return "pinky-memory" + return "skipped" + + +# -- Time helpers -------------------------------------------------------------- + + +def _iso_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _ms_to_iso(ms: int) -> str: + return datetime.fromtimestamp(ms / 1000.0, tz=timezone.utc).isoformat() diff --git a/src/pinky_daemon/ferry/types.py b/src/pinky_daemon/ferry/types.py new file mode 100644 index 00000000..015e4597 --- /dev/null +++ b/src/pinky_daemon/ferry/types.py @@ -0,0 +1,214 @@ +"""Ferry envelope + substrate entry dataclasses. + +Mirrors: +- ferry PROTOCOL.md v0.1 §1 envelope schema +- substrate v0.1 §3 entry schema + +These are wire-shapes — they describe what crosses the boundary, not +how PinkyBot stores things internally. Storage shapes (Reflection in +pinky-memory, Task in pinky-self) are separate; the substrate.py module +translates between the two. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + +# -- Ferry PROTOCOL.md v0.1 §1 ------------------------------------------------- + + +@dataclass +class TraversalRecord: + """One broker hop on a ferry envelope's traversal path. + + Per PROTOCOL.md §12.3, brokers append (never modify) traversal records + as the envelope crosses each broker. The receiving host-callback uses + the array to populate substrate's port_history on inbound (§6.4). + """ + + broker: str # broker identity (e.g. "ferry-broker:nova") + at: int # broker wall-clock millis when received + via: str = "" # link/transport identifier (optional) + signature: str = "" # broker signature over the prior envelope state + + +@dataclass +class FerryEnvelope: + """A ferry envelope, per PROTOCOL.md v0.1 §1. + + Top-level fields are spec-defined; `headers` is the only place + extension metadata may live. + """ + + v: str # protocol version, e.g. "0.1" + id: str # uuid-v7, transport-stable, broker-deduped within 24h + from_: str # immediate sender address + to: str # recipient address + ts: int # sender wall-clock millis + body: dict[str, Any] # application payload — must contain "kind" + + priority: Literal["low", "normal", "high", "urgent"] = "normal" + wake: Literal["none", "if-idle", "when-free", "interrupt"] = "if-idle" + ack: Literal["none", "delivery", "processed"] = "delivery" + + ttl_ms: int | None = None + correlation_id: str | None = None + reply_to: str | None = None + subject: str | None = None + traversal: list[TraversalRecord] = field(default_factory=list) + headers: dict[str, str] = field(default_factory=dict) + + @property + def kind(self) -> str: + """Payload kind — `body.kind` per PROTOCOL.md §1 + ferry overlay docs.""" + return str(self.body.get("kind", "")).strip() + + +# -- DeliveryResult — host-pinky's contract with the ferry broker -------------- + + +@dataclass +class DeliveryResult: + """Result of host-pinky's `deliver(envelope)` call. + + The ferry broker uses the status to decide ack semantics (PROTOCOL.md §6): + - `delivered`: at-least-once delivery satisfied, processing started + - `queued`: agent offline / asleep; envelope will be replayed on wake + - `rejected`: do not retry — auth, ACL, or unsupported payload (terminal) + - `transient_failure`: retry with backoff — operational failure (DB lock, + schema mismatch during rolling deploy, etc.) on the host side. Distinct + from `rejected` so the broker doesn't swallow a real ACL-allowed message + because of an operator-side hiccup. + """ + + status: Literal["delivered", "queued", "rejected", "transient_failure"] + reason: str | None = None + detail: dict[str, Any] = field(default_factory=dict) + + +# -- AgentCardSelector — peer-fleet ACL primitive ------------------------------ + + +@dataclass(frozen=True) +class AgentCardSelector: + """Selector for a peer-fleet agent card. + + At-least-one field must be set. Wildcards via `agent_id="*"` (matches + any agent on the fleet) and `fleet="*"` (matches any fleet — use sparingly). + + Examples: + AgentCardSelector(fleet="sigil", agent_id="pulse@studio@sigil") + AgentCardSelector(fleet="sigil", agent_id="*") # fleet-wide allow + AgentCardSelector(pinky_type="executor") # capability-based + """ + + fleet: str | None = None + agent_id: str | None = None + pinky_type: str | None = None + + def __post_init__(self) -> None: + if not (self.fleet or self.agent_id or self.pinky_type): + raise ValueError( + "AgentCardSelector requires at least one of fleet, agent_id, " + "or pinky_type — empty selectors are not permitted." + ) + + def matches(self, card_fleet: str, card_agent_id: str, card_pinky_type: str = "") -> bool: + """Test whether a peer's signed card matches this selector.""" + if self.fleet is not None and self.fleet != "*" and self.fleet != card_fleet: + return False + if self.agent_id is not None and self.agent_id != "*" and self.agent_id != card_agent_id: + return False + if self.pinky_type is not None and self.pinky_type != card_pinky_type: + return False + return True + + +# -- substrate v0.1 §3 entry schema -------------------------------------------- + + +@dataclass +class SubstrateScope: + """substrate v0.1 §3.3 scope — what the entry is *about*.""" + + kind: Literal["user", "agent", "project", "global"] + ref: str # e.g. "brad", "misha@pinky.local", "ferry-v0.1" + + +@dataclass +class SubstrateSource: + """substrate v0.1 §6 provenance — where the claim came from.""" + + origin: Literal["user-stated", "agent-inferred", "agent-observed", "external"] + by: str # original author identity (e.g. "misha@pinky.local") — §6.1 immutable on port + evidence: str = "" # human-readable trace; quotes, refs, observations + trust: float = 0.5 # [0.0, 1.0]; opaque to spec, host decides what it means + + +@dataclass +class SubstrateLifecycle: + """substrate v0.1 §5 lifecycle state.""" + + state: Literal["active", "pending", "decayed", "superseded"] = "active" + expected_resolution_by: str | None = None # ISO datetime + resolved_by: str | None = None # substrate id of resolver entry + + +@dataclass +class PortHistoryEntry: + """substrate v0.1 §6.4 — one hop in the entry's cross-fleet history. + + Populated on inbound from ferry's traversal array. Substrate's + port_history is canonical (§6.4); ferry's traversal is transport-only. + + Trust boundary on `at`: + - `attested_by="broker"` — `at` mirrors a broker-stamped traversal + record's wall-clock time. Witnessed by a transport intermediary. + - `attested_by="receiver"` — `at` was fabricated by the receiver + (synthetic hop, no broker traversal record). The receiver's clock + is the only witness. Don't conflate with broker-attested `at` + when reasoning about cross-fleet timing. + """ + + from_: str + to: str + at: str # ISO datetime + via: str = "" # ferry msg_id (transport-stable id) + re_grounded: bool = False # whether receiver re-verified the claim + attested_by: Literal["broker", "receiver"] = "broker" + + +@dataclass +class SubstrateEntry: + """A substrate v0.1 §3 entry — wire shape, pre-storage. + + On inbound to host-pinky, this is what we get out of `envelope.body` + (for `kind=substrate.entry`) or each item of `body.entries` (for + `kind=substrate.batch`). The substrate.py module translates this + into pinky-memory Reflection / pinky-self Task primitives. + """ + + id: str # substrate-stable id (uuid) — distinct from ferry transport id + type: Literal["fact", "feedback", "decision", "pattern", "event", "reference", "pending"] + scope: SubstrateScope + source: SubstrateSource + created_at: str # ISO datetime + updated_at: str # ISO datetime + content: str # free-text body (markdown allowed) + links: list[str] = field(default_factory=list) # substrate ids of linked entries + lifecycle: SubstrateLifecycle = field(default_factory=SubstrateLifecycle) + port_history: list[PortHistoryEntry] = field(default_factory=list) + + +# -- Ingest result ------------------------------------------------------------- + + +@dataclass +class IngestResult: + """Outcome of a single substrate-entry import into PinkyBot stores.""" + + substrate_id: str + target_store: Literal["pinky-memory", "pinky-self", "skipped"] + target_id: str = "" # reflection id / task id + note: str = "" diff --git a/src/pinky_daemon/mesh_store.py b/src/pinky_daemon/mesh_store.py new file mode 100644 index 00000000..b553c333 --- /dev/null +++ b/src/pinky_daemon/mesh_store.py @@ -0,0 +1,416 @@ +"""Mesh persistence — ferry envelope log + federation peer registry. + +Two tables in SQLite (its own DB file by default, sibling to the others +under ``data/``): + +- ``mesh_messages`` — every cross-fleet envelope we send or receive, + one row each. Direction-tagged. Indexed on (local_agent, received_at) + for the per-agent inbox view (#420 UI) and on correlation_id for + request/response pairing. +- ``federation_peers`` — known remote agents, keyed by (fleet, agent). + Tracks first/last seen + ``seed_source`` (``"config"`` for + pre-configured peers vs ``"observed"`` for those auto-registered + on first inbound). + +Default-deny still lives in the ``mesh_outbound_allowlist`` (per-agent, +in agent_registry); this store is the *audit log + directory*, not the +permission boundary. + +PinkyBot issue: #419. +""" + +from __future__ import annotations + +import json +import sqlite3 +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +# -- Records ------------------------------------------------------------------- + + +@dataclass +class MeshMessage: + """One row in ``mesh_messages``.""" + + id: int + correlation_id: str + direction: Literal["inbound", "outbound"] + local_agent: str + remote_fleet: str + remote_agent: str + kind: str + body: dict # JSON-decoded + envelope_ts: int # millis from envelope.ts + received_at: float # our wall clock (seconds since epoch) + reply_to: str | None = None + error: str | None = None + + def to_dict(self) -> dict: + return { + "id": self.id, + "correlation_id": self.correlation_id, + "direction": self.direction, + "local_agent": self.local_agent, + "remote_fleet": self.remote_fleet, + "remote_agent": self.remote_agent, + "kind": self.kind, + "body": self.body, + "envelope_ts": self.envelope_ts, + "received_at": self.received_at, + "reply_to": self.reply_to, + "error": self.error, + } + + +@dataclass +class FederationPeer: + """One row in ``federation_peers``.""" + + fleet: str + agent: str + display_name: str + first_seen: float + last_seen: float + seed_source: Literal["config", "observed"] + + def to_dict(self) -> dict: + return { + "fleet": self.fleet, + "agent": self.agent, + "display_name": self.display_name, + "first_seen": self.first_seen, + "last_seen": self.last_seen, + "seed_source": self.seed_source, + "address": f"{self.agent}@{self.fleet}", + } + + +# -- Store --------------------------------------------------------------------- + + +class MeshStore: + """SQLite-backed mesh log + federation peer registry. + + Thread-safe via a per-instance lock around writes; reads use SQLite's + own concurrency story. + """ + + def __init__(self, db_path: str = "data/mesh.db") -> None: + Path(db_path).parent.mkdir(parents=True, exist_ok=True) + self._db = sqlite3.connect(db_path, check_same_thread=False) + self._db.execute("PRAGMA journal_mode=WAL") + self._db.execute("PRAGMA foreign_keys=ON") + self._lock = threading.RLock() + self._init_tables() + + def _init_tables(self) -> None: + self._db.executescript(""" + CREATE TABLE IF NOT EXISTS mesh_messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + correlation_id TEXT NOT NULL DEFAULT '', + direction TEXT NOT NULL CHECK(direction IN ('inbound','outbound')), + local_agent TEXT NOT NULL, + remote_fleet TEXT NOT NULL, + remote_agent TEXT NOT NULL, + kind TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '{}', + envelope_ts INTEGER NOT NULL DEFAULT 0, + received_at REAL NOT NULL, + reply_to TEXT, + error TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_mesh_messages_local + ON mesh_messages(local_agent, received_at DESC); + CREATE INDEX IF NOT EXISTS idx_mesh_messages_corr + ON mesh_messages(correlation_id); + CREATE INDEX IF NOT EXISTS idx_mesh_messages_direction + ON mesh_messages(direction, received_at DESC); + + CREATE TABLE IF NOT EXISTS federation_peers ( + fleet TEXT NOT NULL, + agent TEXT NOT NULL, + display_name TEXT NOT NULL DEFAULT '', + first_seen REAL NOT NULL, + last_seen REAL NOT NULL, + seed_source TEXT NOT NULL DEFAULT 'observed', + PRIMARY KEY (fleet, agent) + ); + + CREATE INDEX IF NOT EXISTS idx_federation_peers_seen + ON federation_peers(last_seen DESC); + """) + self._db.commit() + + # -- mesh_messages -------------------------------------------------------- + + def log_message( + self, + *, + direction: Literal["inbound", "outbound"], + local_agent: str, + remote_fleet: str, + remote_agent: str, + correlation_id: str = "", + kind: str = "", + body: dict | None = None, + envelope_ts: int = 0, + reply_to: str | None = None, + error: str | None = None, + ) -> int: + """Append a row to ``mesh_messages`` and return its id. + + Side effect: also touches ``federation_peers`` — bumps last_seen + if known, registers as ``seed_source='observed'`` if not. For + outbound rows, only updates peer state when ``error is None`` + (a failed publish doesn't prove the peer's reachable). + """ + body_json = json.dumps(body or {}, separators=(",", ":"), ensure_ascii=False) + now = time.time() + with self._lock: + cur = self._db.execute( + """INSERT INTO mesh_messages + (correlation_id, direction, local_agent, + remote_fleet, remote_agent, kind, body, + envelope_ts, received_at, reply_to, error) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + (correlation_id, direction, local_agent, + remote_fleet, remote_agent, kind, body_json, + envelope_ts, now, reply_to, error), + ) + row_id = cur.lastrowid or 0 + # Touch peer for inbound-success and outbound-success only. + should_touch = direction == "inbound" or (direction == "outbound" and error is None) + if should_touch: + self._touch_peer_locked(remote_fleet, remote_agent, now) + self._db.commit() + return row_id + + def get_inbox( + self, + local_agent: str, + *, + limit: int = 50, + offset: int = 0, + kind: str = "", + sender: str = "", + direction: str = "", + ) -> list[MeshMessage]: + """Return mesh messages for ``local_agent``, newest first. + + Filters: ``kind`` exact match, ``sender`` matches against + ``"agent@fleet"``, ``direction`` ∈ ``{"", "inbound", "outbound"}``. + """ + clauses = ["local_agent = ?"] + params: list[Any] = [local_agent] + if kind: + clauses.append("kind = ?") + params.append(kind) + if direction in ("inbound", "outbound"): + clauses.append("direction = ?") + params.append(direction) + if sender: + # sender is "agent@fleet"; split on rightmost @ for matching + if "@" in sender: + agent, _, fleet = sender.rpartition("@") + if agent and fleet: + clauses.append("remote_agent = ? AND remote_fleet = ?") + params.extend([agent, fleet]) + where = " AND ".join(clauses) + params.extend([int(limit), int(offset)]) + rows = self._db.execute( + f"""SELECT id, correlation_id, direction, local_agent, + remote_fleet, remote_agent, kind, body, + envelope_ts, received_at, reply_to, error + FROM mesh_messages + WHERE {where} + ORDER BY received_at DESC + LIMIT ? OFFSET ?""", + params, + ).fetchall() + return [self._row_to_message(r) for r in rows] + + def get_message_by_correlation_id(self, correlation_id: str) -> MeshMessage | None: + """Fetch the most recent message with this correlation_id (any agent). + + Useful for request/response pairing. Returns None if not found. + """ + if not correlation_id: + return None + row = self._db.execute( + """SELECT id, correlation_id, direction, local_agent, + remote_fleet, remote_agent, kind, body, + envelope_ts, received_at, reply_to, error + FROM mesh_messages + WHERE correlation_id = ? + ORDER BY received_at DESC + LIMIT 1""", + (correlation_id,), + ).fetchone() + return self._row_to_message(row) if row else None + + def count_messages(self, local_agent: str = "") -> int: + """Count messages, optionally filtered to one agent.""" + if local_agent: + row = self._db.execute( + "SELECT COUNT(*) FROM mesh_messages WHERE local_agent = ?", + (local_agent,), + ).fetchone() + else: + row = self._db.execute("SELECT COUNT(*) FROM mesh_messages").fetchone() + return int(row[0]) if row else 0 + + def _row_to_message(self, row: tuple) -> MeshMessage: + try: + body = json.loads(row[7]) if row[7] else {} + if not isinstance(body, dict): + body = {"_raw": body} + except (TypeError, ValueError): + body = {"_raw": row[7]} + return MeshMessage( + id=row[0], + correlation_id=row[1], + direction=row[2], + local_agent=row[3], + remote_fleet=row[4], + remote_agent=row[5], + kind=row[6], + body=body, + envelope_ts=row[8], + received_at=row[9], + reply_to=row[10], + error=row[11], + ) + + # -- federation_peers ----------------------------------------------------- + + def _touch_peer_locked(self, fleet: str, agent: str, now: float) -> None: + """Insert-or-update peer's last_seen. Called under self._lock.""" + existing = self._db.execute( + "SELECT first_seen FROM federation_peers WHERE fleet = ? AND agent = ?", + (fleet, agent), + ).fetchone() + if existing: + self._db.execute( + "UPDATE federation_peers SET last_seen = ? WHERE fleet = ? AND agent = ?", + (now, fleet, agent), + ) + else: + self._db.execute( + """INSERT INTO federation_peers + (fleet, agent, display_name, first_seen, last_seen, seed_source) + VALUES (?, ?, '', ?, ?, 'observed')""", + (fleet, agent, now, now), + ) + + def upsert_peer( + self, + *, + fleet: str, + agent: str, + display_name: str = "", + seed_source: Literal["config", "observed"] = "config", + ) -> None: + """Insert or update a peer (admin / config-seeded path). + + ``seed_source='config'`` does NOT downgrade an already-observed + peer to "observed" — config-flagged peers stay flagged. Conversely, + an already-config peer that becomes observed keeps the 'config' + flag (config wins, since it's a stronger statement of intent). + """ + now = time.time() + with self._lock: + existing = self._db.execute( + """SELECT first_seen, seed_source FROM federation_peers + WHERE fleet = ? AND agent = ?""", + (fleet, agent), + ).fetchone() + if existing: + # Keep the stronger seed_source: config beats observed. + final_source = ( + "config" + if seed_source == "config" or existing[1] == "config" + else "observed" + ) + self._db.execute( + """UPDATE federation_peers + SET display_name = ?, last_seen = ?, seed_source = ? + WHERE fleet = ? AND agent = ?""", + (display_name, now, final_source, fleet, agent), + ) + else: + self._db.execute( + """INSERT INTO federation_peers + (fleet, agent, display_name, first_seen, last_seen, seed_source) + VALUES (?, ?, ?, ?, ?, ?)""", + (fleet, agent, display_name, now, now, seed_source), + ) + self._db.commit() + + def list_peers( + self, + *, + fleet: str = "", + seed_source: str = "", + limit: int = 200, + ) -> list[FederationPeer]: + """List peers, newest-seen first. Filter by fleet or seed_source.""" + clauses: list[str] = [] + params: list[Any] = [] + if fleet: + clauses.append("fleet = ?") + params.append(fleet) + if seed_source in ("config", "observed"): + clauses.append("seed_source = ?") + params.append(seed_source) + where = ("WHERE " + " AND ".join(clauses)) if clauses else "" + params.append(int(limit)) + rows = self._db.execute( + f"""SELECT fleet, agent, display_name, first_seen, last_seen, seed_source + FROM federation_peers + {where} + ORDER BY last_seen DESC + LIMIT ?""", + params, + ).fetchall() + return [ + FederationPeer( + fleet=r[0], agent=r[1], display_name=r[2], + first_seen=r[3], last_seen=r[4], seed_source=r[5], + ) + for r in rows + ] + + def get_peer(self, fleet: str, agent: str) -> FederationPeer | None: + """Look up one peer by composite key.""" + row = self._db.execute( + """SELECT fleet, agent, display_name, first_seen, last_seen, seed_source + FROM federation_peers + WHERE fleet = ? AND agent = ?""", + (fleet, agent), + ).fetchone() + if not row: + return None + return FederationPeer( + fleet=row[0], agent=row[1], display_name=row[2], + first_seen=row[3], last_seen=row[4], seed_source=row[5], + ) + + def remove_peer(self, fleet: str, agent: str) -> bool: + """Hard-delete a peer. Returns True if removed.""" + with self._lock: + cur = self._db.execute( + "DELETE FROM federation_peers WHERE fleet = ? AND agent = ?", + (fleet, agent), + ) + self._db.commit() + return (cur.rowcount or 0) > 0 + + # -- Lifecycle ------------------------------------------------------------ + + def close(self) -> None: + self._db.close() diff --git a/src/pinky_daemon/scheduler.py b/src/pinky_daemon/scheduler.py index 1306ce70..f7b5d327 100644 --- a/src/pinky_daemon/scheduler.py +++ b/src/pinky_daemon/scheduler.py @@ -339,12 +339,20 @@ async def _check_schedules(self, now: float) -> None: async def _check_heartbeats(self, now: float) -> None: """Check heartbeat health for all agents with heartbeat_interval > 0.""" agents = self._registry.list(enabled_only=True) + streaming_sessions = {} + if self._streaming_sessions_fn: + try: + streaming_sessions = self._streaming_sessions_fn() or {} + except Exception: + streaming_sessions = {} for agent in agents: if agent.heartbeat_interval <= 0: continue hb = self._registry.get_latest_heartbeat(agent.name) + if self._reconcile_server_liveness(agent, hb, now, streaming_sessions): + continue if not hb: # No heartbeat ever recorded — mark stale self._registry.record_heartbeat( @@ -431,6 +439,86 @@ async def _maybe_resurrect( except Exception as e: _log(f"scheduler: resurrection callback failed for {agent_name}: {e}") + def _reconcile_server_liveness( + self, + agent, + hb, + now: float, + streaming_sessions: dict, + ) -> bool: + """Return True when server-side liveness should suppress death handling. + + Heartbeat rows are agent-reported, but the daemon also has authoritative + transport evidence: connected streaming sessions and server-stamped + last_seen_at. Without reconciling those signals, a stale dead heartbeat + can make the scheduler resurrect an already-live runtime forever. + """ + sessions = streaming_sessions.get(agent.name, {}) or {} + connected_label = "" + connected_session = None + main_session = sessions.get("main") + if main_session is not None and getattr(main_session, "is_connected", False): + connected_label = "main" + connected_session = main_session + else: + for label, session in sessions.items(): + if getattr(session, "is_connected", False): + connected_label = label + connected_session = session + break + + hb_ts = hb.timestamp if hb else 0 + server_ts = getattr(agent, "last_seen_at", 0.0) or 0.0 + grace_seconds = agent.heartbeat_interval * 2 + fresh_server_seen = server_ts > hb_ts and (now - server_ts) <= grace_seconds + + if not connected_session and not fresh_server_seen: + return False + + should_record = ( + not hb + or hb.status != "alive" + or (now - hb.timestamp) >= agent.heartbeat_interval + or server_ts > hb_ts + ) + if should_record: + context_pct = 0.0 + message_count = 0 + session_id = f"{agent.name}-main" + metadata = {"source": "server_presence"} + if connected_session: + session_id = getattr(connected_session, "id", session_id) + metadata["reason"] = "connected_streaming_session" + metadata["label"] = connected_label + try: + context_pct = float(getattr(connected_session, "context_used_pct", 0.0) or 0.0) + except Exception: + context_pct = 0.0 + stats = getattr(connected_session, "stats", {}) or {} + if isinstance(stats, dict): + message_count = int(stats.get("messages_sent", 0) or 0) + int( + stats.get("turns", 0) or 0 + ) + else: + metadata["reason"] = "fresh_last_seen" + metadata["last_seen_at"] = server_ts + + self._registry.record_heartbeat( + agent.name, + session_id=session_id, + status="alive", + context_pct=context_pct, + message_count=message_count, + metadata=metadata, + ) + _log( + f"scheduler: reconciled heartbeat for '{agent.name}' from " + f"{metadata['reason']}" + ) + + self._resurrection_attempts.pop(agent.name, None) + return True + async def _check_idle_sessions(self, now: float) -> None: """Put idle streaming sessions to sleep to save resources.""" if not self._streaming_sessions_fn: diff --git a/src/pinky_daemon/sdk_runner.py b/src/pinky_daemon/sdk_runner.py index 0abc8218..18bb18f6 100644 --- a/src/pinky_daemon/sdk_runner.py +++ b/src/pinky_daemon/sdk_runner.py @@ -62,6 +62,10 @@ class SDKRunnerConfig: # Thinking effort: low, medium, high, max thinking_effort: str = "medium" + # When True, the verify_effort CLI hook blocks tool calls if the runtime + # effort drifts from thinking_effort. Default False (warn-only). See #429. + strict_effort_enforcement: bool = False + # Provider overrides — set these to use Ollama or other compatible endpoints provider_url: str = "" # ANTHROPIC_BASE_URL override provider_key: str = "" # ANTHROPIC_API_KEY override @@ -158,6 +162,16 @@ async def run( env_overrides["ANTHROPIC_BASE_URL"] = self._config.provider_url if self._config.provider_key: env_overrides["ANTHROPIC_API_KEY"] = self._config.provider_key + # #429: surface configured effort + agent identity to CLI hooks so + # verify_effort.py can detect drift from PINKY_EXPECTED_EFFORT vs + # $CLAUDE_EFFORT at PreToolUse time. The hook no-ops on "auto" / + # empty (intentionally adaptive). + if self._agent_name: + env_overrides["PINKY_AGENT_NAME"] = self._agent_name + if effort: + env_overrides["PINKY_EXPECTED_EFFORT"] = effort + if self._config.strict_effort_enforcement: + env_overrides["PINKY_STRICT_EFFORT"] = "1" options.env = env_overrides # Session management diff --git a/src/pinky_daemon/streaming_session.py b/src/pinky_daemon/streaming_session.py index e0d238ba..c4c5d8ef 100644 --- a/src/pinky_daemon/streaming_session.py +++ b/src/pinky_daemon/streaming_session.py @@ -66,6 +66,9 @@ class StreamingSessionConfig: provider_url: str = "" # ANTHROPIC_BASE_URL override (e.g. "http://localhost:11434" for Ollama) provider_key: str = "" # ANTHROPIC_API_KEY override (empty = use env var) thinking_effort: str = "medium" # low, medium, high, xhigh, max — default thinking depth + # When True, the verify_effort CLI hook blocks tool calls if the runtime + # effort drifts from thinking_effort. Default False (warn-only). See #429. + strict_effort_enforcement: bool = False restart_reason: str = "" # "context_restart", "auto_restart", etc. — cleared after wake prompt @@ -293,12 +296,24 @@ async def connect(self) -> None: options.effort = effort # Build provider env overrides (Ollama / custom compatible endpoints) - provider_env = {} + provider_env: dict[str, str] = {} if self._config.provider_url: provider_env["ANTHROPIC_BASE_URL"] = self._config.provider_url if self._config.provider_key: provider_env["ANTHROPIC_API_KEY"] = self._config.provider_key provider_env["ANTHROPIC_AUTH_TOKEN"] = self._config.provider_key + + # #429: surface configured effort + agent identity to CLI hooks so + # hook_verify_effort.py can detect drift from PINKY_EXPECTED_EFFORT + # vs $CLAUDE_EFFORT at PreToolUse time. The hook no-ops on "auto" / + # empty (intentionally adaptive). + if self.agent_name: + provider_env["PINKY_AGENT_NAME"] = self.agent_name + if effort: + provider_env["PINKY_EXPECTED_EFFORT"] = effort + if self._config.strict_effort_enforcement: + provider_env["PINKY_STRICT_EFFORT"] = "1" + if provider_env: # Generous timeout for slow local/third-party models (30 min) provider_env.setdefault("API_TIMEOUT_MS", "1800000") diff --git a/src/pinky_self/server.py b/src/pinky_self/server.py index 897465aa..f8f23fc7 100644 --- a/src/pinky_self/server.py +++ b/src/pinky_self/server.py @@ -1143,10 +1143,13 @@ def send_heartbeat(status: str = "ok", context_pct: float = 0.0, notes: str = "" @mcp.tool() def set_thinking_effort(level: str) -> str: """Set thinking effort for this session. Match depth to task complexity. - level: low | medium | high | max | auto (clears override). + level: low | medium | high | xhigh | max | auto (clears override). """ - if level not in ("low", "medium", "high", "max", "auto"): - return f"Invalid level: {level}. Use low, medium, high, max, or auto." + if level not in ("low", "medium", "high", "xhigh", "max", "auto"): + return ( + f"Invalid level: {level}. " + "Use low, medium, high, xhigh, max, or auto." + ) result = _api("POST", f"/agents/{agent_name}/sessions/main/effort", { "effort": level, }) @@ -1408,6 +1411,50 @@ def send_to_agent( return f"Message queued for {to} (they're offline). They'll see it in their inbox when they wake up." return f"Message delivered to {to}." + @mcp.tool() + def mesh_remote_send( + target: str, + body: str = "", + kind: str = "msg", + correlation_id: str = "", + reply_to: str = "", + priority: str = "normal", + ) -> str: + """Send a ferry envelope to a remote agent on another fleet. + + Cross-fleet counterpart to ``send_to_agent``. Builds a canonical + ferry envelope (PROTOCOL.md v0.1) addressed to ``target`` and + publishes via the daemon's mesh sender. + + Args: + target: ``"agent_slug@fleet"`` (e.g. ``"pulse@pulse"``) or + ``"ferry://fleet/agent_slug"``. + body: Free-form payload string. Wrapped as ``{"text": body, "kind": kind}``. + Pass JSON if you need structured data. + kind: Envelope ``body.kind`` — ``"msg"`` (default), ``"smoke"``, + ``"ack"``, etc. Receivers route on this. + correlation_id: Optional correlation id for request/response pairing. + Auto-generated by the receiver if omitted. + reply_to: Optional ``correlation_id`` of the message being replied to. + priority: ``"low"`` | ``"normal"`` | ``"high"`` | ``"urgent"``. + + Permission: ``target`` must match a pattern in this agent's + ``mesh_outbound_allowlist`` (default-deny if empty). + + Returns a JSON string with ``{sent, correlation_id, subject, ts, error}``. + """ + result = _api("POST", f"/agents/{agent_name}/mesh/send", { + "target": target, + "body": body, + "kind": kind, + "correlation_id": correlation_id, + "reply_to": reply_to, + "priority": priority, + }) + if "error" in result and "sent" not in result: + return json.dumps({"sent": False, "error": result.get("error", "unknown")}) + return json.dumps(result) + @mcp.tool() def send_file_to_agent( to_agent: str, diff --git a/tests/test_admin_update.py b/tests/test_admin_update.py index f34362a0..99e26a47 100644 --- a/tests/test_admin_update.py +++ b/tests/test_admin_update.py @@ -308,11 +308,18 @@ def test_force_and_force_deps_combine(self): assert len(gm.pip_calls) == 1 def test_no_force_deps_skips_pip_when_pyproject_unchanged(self): - """Default behavior: don't reinstall when pyproject.toml didn't change.""" + """Default behavior: don't reinstall when pyproject.toml didn't change. + + The drift detector is also a reinstall trigger (see + TestInstalledDepsDriftDetection) so we patch it to report no drift — + this test pins the pyproject-diff axis specifically. + """ gm = _GitMock(dirty_files=[]) wrapped = self._track_pip_calls(gm) with ( patch("subprocess.check_output", side_effect=wrapped), + patch("pinky_daemon.api._check_installed_deps_drift", + return_value=[]), patch("shutil.which", return_value=None), patch("os.kill"), ): @@ -347,3 +354,223 @@ def fail_on_pip(cmd, **kwargs): assert body.get("deps_rebuilt") is False assert body.get("deps_error") assert "pip install failed" in body["deps_error"] + + +class TestInstalledDepsDriftDetection: + """Direct unit tests for `_check_installed_deps_drift`. + + The helper compares installed package versions to pyproject.toml pins to + catch the "pyproject bumped earlier, routine restart skipped reinstall" + case that previously needed manual force_deps=True. + """ + + def _write_pyproject(self, tmp_dir: str, deps: list[str], + optional: dict | None = None) -> None: + from pathlib import Path + # TOML literal strings (single-quoted) — no escape processing, + # so embedded double quotes (env markers like `platform_system == "x"`) + # don't need escaping. + content = "[project]\nname = 'test'\nversion = '0.0.0'\ndependencies = [\n" + for d in deps: + content += f" '{d}',\n" + content += "]\n" + if optional: + content += "\n[project.optional-dependencies]\n" + for group, group_deps in optional.items(): + content += f"{group} = [\n" + for d in group_deps: + content += f" '{d}',\n" + content += "]\n" + (Path(tmp_dir) / "pyproject.toml").write_text(content) + + def test_empty_drift_when_all_pins_satisfied(self): + """A pyproject that depends only on packages whose installed versions + satisfy the pins returns an empty list.""" + from pinky_daemon.api import _check_installed_deps_drift + + # `pytest` must be installed (we're literally running under it), + # so any-version pin should be satisfied. + with tempfile.TemporaryDirectory() as tmp: + self._write_pyproject(tmp, ["pytest"]) + drifts = _check_installed_deps_drift(tmp) + assert drifts == [] + + def test_pin_higher_than_installed_yields_drift_entry(self): + """When pyproject pins a version above what's installed, that package + shows up in the drift list with the installed version recorded.""" + from importlib.metadata import version as _v + + from pinky_daemon.api import _check_installed_deps_drift + + installed = _v("pytest") + # Construct a pin that the installed version cannot satisfy. + # Bump major+1 to keep things robust against future pytest releases. + major = int(installed.split(".")[0]) + 1 + unsatisfiable_pin = f"pytest>={major}.0.0" + + with tempfile.TemporaryDirectory() as tmp: + self._write_pyproject(tmp, [unsatisfiable_pin]) + drifts = _check_installed_deps_drift(tmp) + + assert len(drifts) == 1 + entry = drifts[0] + assert entry["package"] == "pytest" + assert entry["installed"] == installed + assert str(major) in entry["specifier"] + + def test_missing_package_records_installed_none(self): + """A pyproject dep that isn't installed at all shows up with + installed=None — distinct from a version mismatch.""" + from pinky_daemon.api import _check_installed_deps_drift + + ghost_pkg = "definitely-not-a-real-package-9b3c" + with tempfile.TemporaryDirectory() as tmp: + self._write_pyproject(tmp, [ghost_pkg]) + drifts = _check_installed_deps_drift(tmp) + + assert len(drifts) == 1 + assert drifts[0]["package"] == ghost_pkg + assert drifts[0]["installed"] is None + + def test_optional_dependencies_are_inspected(self): + """Drift in `project.optional-dependencies` groups is reported just + like core dependencies — keeps `pip install -e .[all]` honest.""" + from pinky_daemon.api import _check_installed_deps_drift + + ghost_pkg = "definitely-not-a-real-package-opt-7a2d" + with tempfile.TemporaryDirectory() as tmp: + self._write_pyproject( + tmp, deps=["pytest"], optional={"extra": [ghost_pkg]} + ) + drifts = _check_installed_deps_drift(tmp) + + ghost_entries = [d for d in drifts if d["package"] == ghost_pkg] + assert len(ghost_entries) == 1 + assert ghost_entries[0]["installed"] is None + + def test_markers_excluding_current_env_are_skipped(self): + """Deps with environment markers that don't match the current + interpreter must be skipped — otherwise a Windows-only pin would + spuriously fire on macOS.""" + from pinky_daemon.api import _check_installed_deps_drift + + # platform_system="DefinitelyNotARealOS" never matches — package + # should be skipped, not reported as missing. + with tempfile.TemporaryDirectory() as tmp: + self._write_pyproject( + tmp, + ['ghost-pkg-marker-3f01 ; platform_system == "DefinitelyNotARealOS"'], + ) + drifts = _check_installed_deps_drift(tmp) + + assert drifts == [] + + def test_unparseable_dependency_is_skipped_silently(self): + """A malformed line in pyproject must not crash the whole check; + unparseable entries are silently skipped so well-formed deps still + get inspected.""" + from pinky_daemon.api import _check_installed_deps_drift + + with tempfile.TemporaryDirectory() as tmp: + # First line is junk, second is a real (satisfied) pin. + self._write_pyproject(tmp, ["===not a requirement===", "pytest"]) + drifts = _check_installed_deps_drift(tmp) + assert drifts == [] + + def test_drift_triggers_reinstall_in_admin_update(self): + """When the drift check reports any entry, admin_update kicks off + pip install even if pyproject.toml didn't change in this pull and + force_deps=False.""" + gm = _GitMock(dirty_files=[], before_hash="same1", after_hash="same1") + + fake_drift = [{ + "package": "claude-agent-sdk", + "specifier": ">=0.1.77", + "installed": "0.1.68", + }] + + pip_calls: list[list[str]] = [] + + def record_subprocess(cmd, **kwargs): + cmd_list = list(cmd) + is_pip = ( + "install" in cmd_list + and (cmd_list[0].endswith("/pip") or cmd_list[0] == "pip" + or "pip" in cmd_list) + ) + if is_pip: + pip_calls.append(cmd_list) + return b"" + return gm(cmd, **kwargs) + + with ( + patch("subprocess.check_output", side_effect=record_subprocess), + patch("pinky_daemon.api._check_installed_deps_drift", + return_value=fake_drift), + patch("shutil.which", return_value=None), + patch("os.kill"), + ): + client = _make_client() + r = client.post("/admin/update?branch=beta") + body = r.json() + assert body.get("deps_rebuilt") is True + assert body.get("deps_drift") == fake_drift + assert pip_calls, "drift should have triggered pip install" + + def test_no_drift_no_diff_no_force_means_no_reinstall(self): + """The drift signal is additive — when there's no drift AND no + pyproject change AND no force_deps, pip is not invoked.""" + gm = _GitMock(dirty_files=[], before_hash="same1", after_hash="same1") + + pip_calls: list[list[str]] = [] + + def record_subprocess(cmd, **kwargs): + cmd_list = list(cmd) + is_pip = ( + "install" in cmd_list + and (cmd_list[0].endswith("/pip") or cmd_list[0] == "pip" + or "pip" in cmd_list) + ) + if is_pip: + pip_calls.append(cmd_list) + return b"" + return gm(cmd, **kwargs) + + with ( + patch("subprocess.check_output", side_effect=record_subprocess), + patch("pinky_daemon.api._check_installed_deps_drift", + return_value=[]), + patch("shutil.which", return_value=None), + patch("os.kill"), + ): + client = _make_client() + r = client.post("/admin/update?branch=beta") + body = r.json() + assert body.get("deps_rebuilt") is False + assert body.get("deps_drift") == [] + assert pip_calls == [] + + def test_drift_check_failure_is_non_fatal(self): + """If the drift check itself raises (e.g. pyproject parse fails), + admin_update keeps going — drift just can't contribute to the + reinstall decision.""" + gm = _GitMock(dirty_files=[], before_hash="same1", after_hash="same1") + + def boom(_repo_dir): + raise RuntimeError("simulated drift-check failure") + + with ( + patch("subprocess.check_output", side_effect=gm), + patch("pinky_daemon.api._check_installed_deps_drift", + side_effect=boom), + patch("shutil.which", return_value=None), + patch("os.kill"), + ): + client = _make_client() + r = client.post("/admin/update?branch=beta") + assert r.status_code == 200 + body = r.json() + # No reinstall (no drift, no diff, no force), and the failure didn't + # propagate as a 500. + assert body.get("updated") is True + assert body.get("deps_drift") == [] diff --git a/tests/test_api.py b/tests/test_api.py index 2dd51705..e274c379 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -906,6 +906,10 @@ async def fake_connect(self): app1 = self._make_app(db_path) with TestClient(app1) as client1: client1.post("/agents", json={"name": "test-agent", "model": "sonnet"}) + # Boot policy (2026-05-11): only the main agent auto-resumes + # its streaming session on restart. Mark this agent as main so + # the cross-boot restore behavior under test still applies. + app1.state.agents.set_main_agent("test-agent") resp = client1.post("/agents/test-agent/streaming-sessions?label=worker") assert resp.status_code == 200 assert app1.state.agents.get_streaming_session_id("test-agent", label="worker") == "test-agent-sdk" diff --git a/tests/test_autonomy.py b/tests/test_autonomy.py new file mode 100644 index 00000000..c9c4bc2b --- /dev/null +++ b/tests/test_autonomy.py @@ -0,0 +1,156 @@ +"""Tests for AutonomyEngine push_event wake gating. + +Boot policy (2026-05-11): only the main agent's autonomy loop starts at boot. +Sibling agents wake on demand the first time `push_event` routes an event to +them. These tests pin the wake gate — it must depend only on `enabled`, not +the legacy `auto_start` flag (which used to gate wake too and would strand +siblings under the new boot policy). +""" + +from __future__ import annotations + +import asyncio +import os +import tempfile + +import pytest + +from pinky_daemon.agent_registry import AgentRegistry +from pinky_daemon.autonomy import AgentEvent, AutonomyEngine, EventType +from pinky_daemon.conversation_store import ConversationStore +from pinky_daemon.task_store import TaskStore + + +@pytest.fixture +def stores(): + fd, path = tempfile.mkstemp(suffix=".db") + os.close(fd) + reg = AgentRegistry(db_path=path) + tasks = TaskStore(db_path=path) + convos = ConversationStore(db_path=path) + yield reg, tasks, convos + reg.close() + os.unlink(path) + + +def _build_engine(registry, tasks, convos): + # session_sender is unused for push_event wake tests; the loop, once + # started, will block on its event queue waiting for more work. + return AutonomyEngine( + registry, + tasks, + convos, + session_sender=None, + idle_check_interval=3600, # don't fire idle exit during the test + ) + + +class TestPushEventWakeGate: + """The wake gate in `push_event` decides whether an inbound event should + spin up an agent's autonomy loop. Under the post-2026-05-11 boot policy + siblings start out without a loop, so this gate is the on-demand wake + path. It must remain permissive for enabled agents and refuse disabled ones. + """ + + @pytest.mark.asyncio + async def test_enabled_non_auto_start_agent_wakes_on_event(self, stores): + registry, tasks, convos = stores + # Critical: auto_start=False (sibling default under the new policy) + registry.register("murzik", model="opus", auto_start=False, enabled=True) + engine = _build_engine(registry, tasks, convos) + await engine.start() + try: + assert "murzik" not in engine._running_loops + + await engine.push_event(AgentEvent( + type=EventType.message_received, + agent_name="murzik", + )) + + # Loop should be started despite auto_start=False + assert "murzik" in engine._running_loops + finally: + await engine.stop() + + @pytest.mark.asyncio + async def test_disabled_agent_does_not_wake_on_event(self, stores): + registry, tasks, convos = stores + registry.register("retired", model="opus", auto_start=True, enabled=False) + engine = _build_engine(registry, tasks, convos) + await engine.start() + try: + await engine.push_event(AgentEvent( + type=EventType.manual_wake, + agent_name="retired", + )) + + # Disabled agent must NOT have its loop started, even with auto_start=True + assert "retired" not in engine._running_loops + finally: + await engine.stop() + + @pytest.mark.asyncio + async def test_event_for_unknown_agent_does_not_wake(self, stores): + registry, tasks, convos = stores + engine = _build_engine(registry, tasks, convos) + await engine.start() + try: + await engine.push_event(AgentEvent( + type=EventType.schedule_wake, + agent_name="ghost", + )) + + assert "ghost" not in engine._running_loops + finally: + await engine.stop() + + @pytest.mark.asyncio + async def test_running_loop_is_not_double_started(self, stores): + registry, tasks, convos = stores + registry.register("barsik", model="opus", auto_start=True, enabled=True) + engine = _build_engine(registry, tasks, convos) + await engine.start() + try: + await engine.start_agent_loop("barsik") + first_task = engine._running_loops["barsik"] + + await engine.push_event(AgentEvent( + type=EventType.message_received, + agent_name="barsik", + )) + + # Same task object — no replacement + assert engine._running_loops["barsik"] is first_task + finally: + await engine.stop() + + @pytest.mark.asyncio + async def test_event_is_queued_even_when_wake_blocked(self, stores): + """A disabled agent shouldn't have its loop started, but events still + land on the queue. If the agent is later re-enabled and its loop is + started manually, the queued event is available — push_event never + drops the message itself. + """ + registry, tasks, convos = stores + registry.register("retired", model="opus", auto_start=False, enabled=False) + engine = _build_engine(registry, tasks, convos) + await engine.start() + try: + ev = AgentEvent( + type=EventType.message_received, + agent_name="retired", + data={"text": "hello"}, + ) + await engine.push_event(ev) + + assert "retired" not in engine._running_loops + # Queue exists and has the event ready to pop + engine._event_queue.ensure_queue("retired") + popped = await asyncio.wait_for( + engine._event_queue.pop("retired", timeout=0.5), + timeout=1.0, + ) + assert popped is not None + assert popped.data == {"text": "hello"} + finally: + await engine.stop() diff --git a/tests/test_codex_session.py b/tests/test_codex_session.py index 8e4d9ee8..33b6878c 100644 --- a/tests/test_codex_session.py +++ b/tests/test_codex_session.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import os import tempfile @@ -43,8 +44,10 @@ def test_properties(self): assert s.agent_name == "test-agent" assert s.id == "test-agent-main" assert s.is_connected is False + assert s.is_idle_sleeping is False assert isinstance(s.stats, dict) assert s.stats["connected"] is False + assert s.stats["idle_sleeping"] is False assert s.stats["account"]["apiProvider"] == "codex_cli" def test_stats_shape(self): @@ -318,6 +321,96 @@ async def test_disconnect_idempotent(self): await s.disconnect() assert not s.is_connected + @pytest.mark.asyncio + async def test_disconnect_alone_does_not_set_idle_sleeping(self): + config = StreamingSessionConfig( + agent_name="test", + working_dir="/tmp", + provider_url="codex_cli", + ) + s = CodexSession(config) + + await s.disconnect() + + assert s.is_connected is False + assert s.is_idle_sleeping is False + + @pytest.mark.asyncio + async def test_idle_sleep_sets_idle_sleeping(self): + config = StreamingSessionConfig( + agent_name="test", + working_dir="/tmp", + provider_url="codex_cli", + ) + s = CodexSession(config) + s._connected = True + + async def fake_exec(prompt: str) -> CodexTurnResult: + return CodexTurnResult() + + s._exec_codex = fake_exec # type: ignore[assignment] + + slept = await s.idle_sleep() + + assert slept is True + assert s.is_connected is False + assert s.is_idle_sleeping is True + assert s.stats["idle_sleeping"] is True + + @pytest.mark.asyncio + async def test_connect_clears_idle_sleeping(self): + config = StreamingSessionConfig( + agent_name="test", + working_dir="/tmp", + provider_url="codex_cli", + ) + s = CodexSession(config) + s._idle_sleeping = True + + async def fake_worker() -> None: + await asyncio.sleep(60) + + s._message_worker = fake_worker # type: ignore[assignment] + + await s.connect() + + assert s.is_connected is True + assert s.is_idle_sleeping is False + await s.disconnect() + + @pytest.mark.asyncio + async def test_attempt_reconnect_uses_connect_and_preserves_codex_session_id(self): + config = StreamingSessionConfig( + agent_name="test", + working_dir="/tmp", + provider_url="codex_cli", + ) + s = CodexSession(config) + s.codex_session_id = "thread-123" + s.session_id = "thread-123" + s._RECONNECT_BACKOFF = (0,) + calls = [] + + async def fake_disconnect() -> None: + calls.append("disconnect") + s._connected = False + + async def fake_connect() -> None: + calls.append("connect") + s._connected = True + s._idle_sleeping = False + + s.disconnect = fake_disconnect # type: ignore[method-assign] + s.connect = fake_connect # type: ignore[method-assign] + + await s.attempt_reconnect() + + assert calls == ["disconnect", "connect"] + assert s.is_connected is True + assert s.codex_session_id == "thread-123" + assert s.session_id == "thread-123" + assert s.stats["reconnects"] == 1 + class TestCodexCommandConstruction: """Pin down `_build_codex_cmd()` against #351 regression: `--sandbox=...` diff --git a/tests/test_effort_verification.py b/tests/test_effort_verification.py new file mode 100644 index 00000000..090e6042 --- /dev/null +++ b/tests/test_effort_verification.py @@ -0,0 +1,692 @@ +"""Tests for the effort-drift verification hook (#429). + +Covers: +- ``Agent.strict_effort_enforcement`` column round-trip +- ``AgentRegistry.record_effort_drift`` + ``get_effort_drift_events`` +- ``ensure_workspace_hooks`` idempotently installs the verify_effort hook +- ``_setup_hooks`` writes valid settings.json with both working + verify hooks +- ``_setup_hooks`` merges verify_effort into pre-existing settings.json + without clobbering user-added hooks +- Subprocess-level behavior of ``hook_verify_effort.py``: + - match → silent no-op + - mismatch + warn mode → POSTs to ``/agents/{name}/effort-drift`` + - mismatch + strict mode → emits a block decision on stdout + - no expected / actual / agent → silent no-op + - ``expected=auto`` → silent no-op (intentionally adaptive) +- ``SDKRunnerConfig`` + ``StreamingSessionConfig`` carry the strict flag +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +import pytest + +from pinky_daemon.agent_registry import ( + AgentRegistry, + _verify_effort_hook_source, +) + + +@pytest.fixture +def registry(): + fd, path = tempfile.mkstemp(suffix=".db") + os.close(fd) + r = AgentRegistry(db_path=path) + yield r + r.close() + os.unlink(path) + + +@pytest.fixture +def hook_script(tmp_path): + """Write the hook source to a temp file we can invoke as a subprocess.""" + p = tmp_path / "hook_verify_effort.py" + p.write_text(_verify_effort_hook_source()) + return p + + +class _CaptureHandler(BaseHTTPRequestHandler): + """Tiny test HTTP server that records POST bodies for assertion.""" + + captured: list = [] + + def do_POST(self): # noqa: N802 — http.server interface + length = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(length).decode() if length else "" + try: + data = json.loads(body) + except Exception: + data = {"raw": body} + type(self).captured.append({ + "path": self.path, + "body": data, + "headers": dict(self.headers), + }) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"ok": true}') + + def log_message(self, *_args, **_kwargs): # silence test noise + pass + + +@pytest.fixture +def fake_daemon(): + """Spin up an in-process HTTP server that captures hook POSTs.""" + + class _Handler(_CaptureHandler): + captured: list = [] + + server = HTTPServer(("127.0.0.1", 0), _Handler) + port = server.server_port + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield port, _Handler.captured + server.shutdown() + server.server_close() + + +def _run_hook( + hook_path: Path, + *, + expected: str = "", + actual: str = "", + agent: str = "", + strict: bool = False, + daemon_url: str = "", + stdin: str = "", +) -> subprocess.CompletedProcess: + env = {**os.environ} + # Wipe any inherited CLAUDE_EFFORT so test env is hermetic + env.pop("CLAUDE_EFFORT", None) + env.pop("PINKY_EXPECTED_EFFORT", None) + env.pop("PINKY_AGENT_NAME", None) + env.pop("PINKY_STRICT_EFFORT", None) + env.pop("PINKY_DAEMON_URL", None) + if actual: + env["CLAUDE_EFFORT"] = actual + if expected: + env["PINKY_EXPECTED_EFFORT"] = expected + if agent: + env["PINKY_AGENT_NAME"] = agent + if strict: + env["PINKY_STRICT_EFFORT"] = "1" + if daemon_url: + env["PINKY_DAEMON_URL"] = daemon_url + return subprocess.run( + [sys.executable, str(hook_path)], + env=env, + input=stdin, + capture_output=True, + text=True, + timeout=10, + ) + + +# ── Schema / dataclass ───────────────────────────────────── + + +class TestStrictEffortColumn: + def test_default_false(self, registry): + agent = registry.register("alpha", model="opus") + assert agent.strict_effort_enforcement is False + + def test_set_via_register(self, registry): + agent = registry.register("alpha", strict_effort_enforcement=True) + assert agent.strict_effort_enforcement is True + + def test_update_round_trip(self, registry): + registry.register("alpha") + registry.register("alpha", strict_effort_enforcement=True) + got = registry.get("alpha") + assert got is not None + assert got.strict_effort_enforcement is True + # And back to off + registry.register("alpha", strict_effort_enforcement=False) + got2 = registry.get("alpha") + assert got2 is not None + assert got2.strict_effort_enforcement is False + + def test_to_dict_includes_field(self, registry): + agent = registry.register("alpha", strict_effort_enforcement=True) + d = agent.to_dict() + assert d["strict_effort_enforcement"] is True + + +# ── record_effort_drift + get_effort_drift_events ────────── + + +class TestEffortDriftRecording: + def test_record_writes_row(self, registry): + registry.register("alpha") + event_id = registry.record_effort_drift( + "alpha", expected="high", actual="medium", + session_id="sess-1", tool_name="Bash", strict=False, + ) + assert event_id > 0 + events = registry.get_effort_drift_events("alpha") + assert len(events) == 1 + e = events[0] + assert e["expected"] == "high" + assert e["actual"] == "medium" + assert e["tool_name"] == "Bash" + assert e["strict"] is False + assert e["agent_name"] == "alpha" + + def test_record_writes_heartbeat_note(self, registry): + registry.register("alpha") + registry.record_effort_drift( + "alpha", expected="max", actual="low", strict=True, + ) + hb = registry.get_latest_heartbeat("alpha") + assert hb is not None + assert "effort drift" in hb.notes + assert "blocked" in hb.notes # strict label + assert "expected=max" in hb.notes + assert "actual=low" in hb.notes + + def test_record_warn_label_in_note(self, registry): + registry.register("alpha") + registry.record_effort_drift( + "alpha", expected="high", actual="medium", strict=False, + ) + hb = registry.get_latest_heartbeat("alpha") + assert hb is not None + assert "warn" in hb.notes + + def test_get_filters_by_agent(self, registry): + registry.register("alpha") + registry.register("beta") + registry.record_effort_drift("alpha", expected="high", actual="low") + registry.record_effort_drift("beta", expected="high", actual="medium") + a_events = registry.get_effort_drift_events("alpha") + b_events = registry.get_effort_drift_events("beta") + assert len(a_events) == 1 + assert len(b_events) == 1 + assert a_events[0]["actual"] == "low" + assert b_events[0]["actual"] == "medium" + + def test_get_filters_by_since(self, registry): + import time + + registry.register("alpha") + registry.record_effort_drift("alpha", expected="high", actual="low") + cutoff = time.time() + 0.01 + time.sleep(0.05) + registry.record_effort_drift("alpha", expected="high", actual="medium") + events = registry.get_effort_drift_events("alpha", since=cutoff) + # Only the second event should be returned + assert len(events) == 1 + assert events[0]["actual"] == "medium" + + def test_get_fleet_wide(self, registry): + registry.register("alpha") + registry.register("beta") + registry.record_effort_drift("alpha", expected="high", actual="low") + registry.record_effort_drift("beta", expected="high", actual="medium") + all_events = registry.get_effort_drift_events("") + assert len(all_events) == 2 + + +# ── settings.json installer ──────────────────────────────── + + +class TestHookInstaller: + def test_setup_creates_verify_effort_script(self, tmp_path): + AgentRegistry._setup_hooks(tmp_path, "alpha") + assert (tmp_path / ".claude" / "hook_verify_effort.py").exists() + + def test_setup_settings_has_verify_hook(self, tmp_path): + AgentRegistry._setup_hooks(tmp_path, "alpha") + settings = json.loads( + (tmp_path / ".claude" / "settings.json").read_text() + ) + commands = [ + h["command"] + for entry in settings["hooks"]["PreToolUse"] + for h in entry["hooks"] + ] + assert any("hook_verify_effort.py" in c for c in commands), commands + # And working/idle still present + assert any("hook_working.py" in c for c in commands), commands + + def test_setup_idempotent_double_call(self, tmp_path): + AgentRegistry._setup_hooks(tmp_path, "alpha") + first = (tmp_path / ".claude" / "settings.json").read_text() + AgentRegistry._setup_hooks(tmp_path, "alpha") + second = (tmp_path / ".claude" / "settings.json").read_text() + # The file should be unchanged on second call (no duplicate hooks) + # Count verify_effort references — must be exactly 1 + assert second.count("hook_verify_effort.py") == 1 + # And working count is also 1 + assert second.count("hook_working.py") == first.count("hook_working.py") + + def test_setup_merges_into_legacy_settings(self, tmp_path): + # Simulate an old agent: settings.json exists with only working hook. + claude_dir = tmp_path / ".claude" + claude_dir.mkdir() + legacy = { + "hooks": { + "PreToolUse": [{ + "matcher": ".*", + "hooks": [{ + "type": "command", + "command": "python3 /custom/user-hook.py || true", + }], + }], + } + } + (claude_dir / "settings.json").write_text(json.dumps(legacy)) + + AgentRegistry._setup_hooks(tmp_path, "alpha") + merged = json.loads((claude_dir / "settings.json").read_text()) + commands = [ + h["command"] + for entry in merged["hooks"]["PreToolUse"] + for h in entry["hooks"] + ] + # User's custom hook preserved + assert any("user-hook.py" in c for c in commands), commands + # Verify hook added + assert any("hook_verify_effort.py" in c for c in commands), commands + + def test_ensure_workspace_hooks_no_agent(self, registry): + # Calling on a non-existent agent must not raise. + registry.ensure_workspace_hooks("nonexistent") # should no-op + + def test_ensure_workspace_hooks_installs(self, registry, tmp_path): + registry.register( + "alpha", + working_dir=str(tmp_path / "agent"), + ) + # Initial register already runs _setup_hooks; remove the + # verify_effort entry to simulate pre-#429 state. + settings_path = tmp_path / "agent" / ".claude" / "settings.json" + settings = json.loads(settings_path.read_text()) + for entry in settings["hooks"]["PreToolUse"]: + entry["hooks"] = [ + h for h in entry["hooks"] + if "hook_verify_effort" not in (h.get("command") or "") + ] + settings_path.write_text(json.dumps(settings)) + assert "hook_verify_effort" not in settings_path.read_text() + + # Now re-ensure — hook should be re-added. + registry.ensure_workspace_hooks("alpha") + assert "hook_verify_effort" in settings_path.read_text() + + +# ── Hook subprocess behavior ─────────────────────────────── + + +class TestVerifyEffortHookBehavior: + def test_match_silent(self, hook_script): + proc = _run_hook( + hook_script, expected="high", actual="high", agent="alpha", + ) + assert proc.returncode == 0 + assert proc.stdout == "" + + def test_no_expected_silent(self, hook_script): + proc = _run_hook(hook_script, actual="high", agent="alpha") + assert proc.returncode == 0 + assert proc.stdout == "" + + def test_no_actual_silent(self, hook_script): + # Older Claude Code without v2.1.133 effort.level + proc = _run_hook(hook_script, expected="high", agent="alpha") + assert proc.returncode == 0 + assert proc.stdout == "" + + def test_auto_expected_silent(self, hook_script): + proc = _run_hook( + hook_script, expected="auto", actual="low", agent="alpha", + ) + assert proc.returncode == 0 + assert proc.stdout == "" + + def test_no_agent_silent(self, hook_script): + # Hook misconfigured — no agent name — must not crash + no output + proc = _run_hook( + hook_script, expected="high", actual="low", + ) + assert proc.returncode == 0 + assert proc.stdout == "" + + def test_mismatch_warn_posts(self, hook_script, fake_daemon): + port, captured = fake_daemon + proc = _run_hook( + hook_script, + expected="high", actual="medium", agent="alpha", + daemon_url=f"http://127.0.0.1:{port}", + ) + assert proc.returncode == 0 + # Warn mode → no block decision printed + assert proc.stdout == "" + assert len(captured) == 1, captured + msg = captured[0] + assert msg["path"] == "/agents/alpha/effort-drift" + assert msg["body"]["expected"] == "high" + assert msg["body"]["actual"] == "medium" + assert msg["body"]["strict"] is False + + def test_mismatch_strict_emits_block(self, hook_script, fake_daemon): + port, captured = fake_daemon + proc = _run_hook( + hook_script, + expected="high", actual="low", agent="alpha", strict=True, + daemon_url=f"http://127.0.0.1:{port}", + ) + assert proc.returncode == 0 + # Strict mode → block decision on stdout + decision = json.loads(proc.stdout) + assert decision["decision"] == "block" + assert "high" in decision["reason"] + assert "low" in decision["reason"] + # Plus the drift POST still happens + assert len(captured) == 1 + assert captured[0]["body"]["strict"] is True + + def test_mismatch_with_tool_name_from_stdin(self, hook_script, fake_daemon): + port, captured = fake_daemon + stdin_payload = json.dumps({"tool_name": "Bash", "session_id": "s1"}) + proc = _run_hook( + hook_script, + expected="high", actual="medium", agent="alpha", + daemon_url=f"http://127.0.0.1:{port}", + stdin=stdin_payload, + ) + assert proc.returncode == 0 + assert len(captured) == 1 + assert captured[0]["body"]["tool_name"] == "Bash" + + # ── Strict-mode remediation allowlist (Murzik review catch) ── + + @pytest.mark.parametrize("tool_name", [ + "set_thinking_effort", + "mcp__pinky-self__set_thinking_effort", + "SET_THINKING_EFFORT", # case-insensitive + ]) + def test_strict_allows_remediation_tool( + self, hook_script, fake_daemon, tool_name, + ): + """Strict mode must NOT block set_thinking_effort — that's the only + way the agent can self-correct drift. Otherwise strict is a trap. + """ + port, captured = fake_daemon + stdin_payload = json.dumps({"tool_name": tool_name}) + proc = _run_hook( + hook_script, + expected="high", actual="medium", agent="alpha", strict=True, + daemon_url=f"http://127.0.0.1:{port}", + stdin=stdin_payload, + ) + assert proc.returncode == 0 + # Drift still recorded + assert len(captured) == 1 + assert captured[0]["body"]["tool_name"] == tool_name + assert captured[0]["body"]["strict"] is True + # But NO block decision on stdout + assert proc.stdout == "", ( + f"Strict mode must not block remediation tool {tool_name!r}; " + f"got stdout: {proc.stdout!r}" + ) + + def test_strict_still_blocks_other_tools(self, hook_script, fake_daemon): + """Sanity: strict mode still blocks non-remediation tools.""" + port, captured = fake_daemon + stdin_payload = json.dumps({"tool_name": "Bash"}) + proc = _run_hook( + hook_script, + expected="high", actual="medium", agent="alpha", strict=True, + daemon_url=f"http://127.0.0.1:{port}", + stdin=stdin_payload, + ) + assert proc.returncode == 0 + decision = json.loads(proc.stdout) + assert decision["decision"] == "block" + + def test_strict_xhigh_remediation_suggests_xhigh( + self, hook_script, fake_daemon, + ): + """When expected=xhigh, the remediation suggestion must be + ``set_thinking_effort('xhigh')`` — not a closer-but-wrong level. + Anything else leaves the agent stuck: calling set_thinking_effort + with the wrong level just produces another drift event. + """ + port, _captured = fake_daemon + proc = _run_hook( + hook_script, + expected="xhigh", actual="medium", agent="alpha", strict=True, + daemon_url=f"http://127.0.0.1:{port}", + ) + assert proc.returncode == 0 + decision = json.loads(proc.stdout) + reason = decision["reason"] + # Must suggest set_thinking_effort('xhigh') — matches the tool's + # accepted levels after #429 widened it. + assert "set_thinking_effort('xhigh')" in reason, reason + # Must NOT suggest a wrong-level call. + assert "set_thinking_effort('high')" not in reason, reason + assert "set_thinking_effort('max')" not in reason, reason + + def test_strict_unreachable_level_says_so( + self, hook_script, fake_daemon, + ): + """If expected is somehow outside the MCP tool's accepted set + (configuration mismatch — shouldn't happen in normal use), the + reason must be honest: tell the agent there's no self-remediation + path so it escalates to the owner instead of spinning. + """ + port, _captured = fake_daemon + proc = _run_hook( + hook_script, + # Synthetic invalid level — hypothetical future registry value + expected="ultra", actual="medium", agent="alpha", strict=True, + daemon_url=f"http://127.0.0.1:{port}", + ) + assert proc.returncode == 0 + decision = json.loads(proc.stdout) + reason = decision["reason"] + assert "no self-remediation path" in reason + # And don't suggest a tool call that won't fix the drift. + assert "set_thinking_effort(" not in reason + + +class TestVerifyEffortScriptOnDisk: + """Ensure stale on-disk verify_effort scripts get refreshed.""" + + def test_setup_overwrites_stale_script(self, tmp_path): + AgentRegistry._setup_hooks(tmp_path, "alpha") + script_path = tmp_path / ".claude" / "hook_verify_effort.py" + # Simulate a stale older version on disk. + script_path.write_text("# stale version\nprint('old')\n") + # Second run should refresh. + AgentRegistry._setup_hooks(tmp_path, "alpha") + content = script_path.read_text() + assert "stale version" not in content + assert "PinkyBot effort-drift verification hook" in content + + def test_setup_keeps_fresh_script_unchanged(self, tmp_path): + AgentRegistry._setup_hooks(tmp_path, "alpha") + script_path = tmp_path / ".claude" / "hook_verify_effort.py" + mtime_first = script_path.stat().st_mtime + # Same content → should not be rewritten (mtime preserved). + import time + time.sleep(0.05) + AgentRegistry._setup_hooks(tmp_path, "alpha") + mtime_second = script_path.stat().st_mtime + assert mtime_first == mtime_second + + +# ── Config plumbing ──────────────────────────────────────── + + +class TestSDKRunnerConfigField: + def test_default(self): + from pinky_daemon.sdk_runner import SDKRunnerConfig + cfg = SDKRunnerConfig() + assert cfg.strict_effort_enforcement is False + + def test_set_true(self): + from pinky_daemon.sdk_runner import SDKRunnerConfig + cfg = SDKRunnerConfig(strict_effort_enforcement=True) + assert cfg.strict_effort_enforcement is True + + +class TestStreamingSessionConfigField: + def test_default(self): + from pinky_daemon.streaming_session import StreamingSessionConfig + cfg = StreamingSessionConfig(agent_name="x", working_dir="/tmp") + assert cfg.strict_effort_enforcement is False + + def test_set_true(self): + from pinky_daemon.streaming_session import StreamingSessionConfig + cfg = StreamingSessionConfig( + agent_name="x", working_dir="/tmp", + strict_effort_enforcement=True, + ) + assert cfg.strict_effort_enforcement is True + + +class TestAPIRoundTrip: + """strict_effort_enforcement must round-trip through the public agent API + (Murzik review catch #2). Without this, the per-agent strict opt-in + promised by the design is unreachable except via direct DB writes. + """ + + def _make_client(self): + from fastapi.testclient import TestClient + + from pinky_daemon.api import create_api + fd, path = tempfile.mkstemp(suffix=".db") + os.close(fd) + app = create_api( + max_sessions=10, default_working_dir="/tmp", db_path=path, + ) + return TestClient(app), path + + def test_register_with_strict_effort(self): + client, db_path = self._make_client() + try: + resp = client.post("/agents", json={ + "name": "alpha", + "model": "opus", + "strict_effort_enforcement": True, + }) + assert resp.status_code == 200, resp.text + assert resp.json()["strict_effort_enforcement"] is True + + # And it persists on GET. + got = client.get("/agents/alpha").json() + assert got["strict_effort_enforcement"] is True + finally: + client.close() + os.unlink(db_path) + + def test_register_default_strict_effort_false(self): + client, db_path = self._make_client() + try: + resp = client.post("/agents", json={ + "name": "alpha", + "model": "opus", + }) + assert resp.status_code == 200 + assert resp.json()["strict_effort_enforcement"] is False + finally: + client.close() + os.unlink(db_path) + + def test_update_toggles_strict_effort(self): + client, db_path = self._make_client() + try: + client.post("/agents", json={"name": "alpha", "model": "opus"}) + # Turn it on + resp = client.put("/agents/alpha", json={ + "strict_effort_enforcement": True, + }) + assert resp.status_code == 200, resp.text + assert resp.json()["strict_effort_enforcement"] is True + # Turn it back off + resp = client.put("/agents/alpha", json={ + "strict_effort_enforcement": False, + }) + assert resp.status_code == 200 + assert resp.json()["strict_effort_enforcement"] is False + finally: + client.close() + os.unlink(db_path) + + +class TestEffortDriftEndpoint: + """Smoke-test the POST/GET /agents/{name}/effort-drift endpoints.""" + + def _make_client(self): + from fastapi.testclient import TestClient + + from pinky_daemon.api import create_api + fd, path = tempfile.mkstemp(suffix=".db") + os.close(fd) + app = create_api( + max_sessions=10, default_working_dir="/tmp", db_path=path, + ) + return TestClient(app), path + + def test_post_records_event(self): + client, db_path = self._make_client() + try: + client.post("/agents", json={"name": "alpha", "model": "opus"}) + resp = client.post("/agents/alpha/effort-drift", json={ + "expected": "high", + "actual": "medium", + "tool_name": "Bash", + "strict": False, + }) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["ok"] is True + assert body["expected"] == "high" + assert body["actual"] == "medium" + assert body["event_id"] > 0 + finally: + client.close() + os.unlink(db_path) + + def test_get_lists_events(self): + client, db_path = self._make_client() + try: + client.post("/agents", json={"name": "alpha", "model": "opus"}) + client.post("/agents/alpha/effort-drift", json={ + "expected": "high", "actual": "low", + }) + client.post("/agents/alpha/effort-drift", json={ + "expected": "high", "actual": "medium", + }) + resp = client.get("/agents/alpha/effort-drift") + assert resp.status_code == 200 + body = resp.json() + assert body["count"] == 2 + assert len(body["events"]) == 2 + finally: + client.close() + os.unlink(db_path) + + def test_post_for_unknown_agent_404s(self): + client, db_path = self._make_client() + try: + resp = client.post("/agents/ghost/effort-drift", json={ + "expected": "high", "actual": "low", + }) + assert resp.status_code == 404 + finally: + client.close() + os.unlink(db_path) diff --git a/tests/test_ferry_host_pinky.py b/tests/test_ferry_host_pinky.py new file mode 100644 index 00000000..2e5adb55 --- /dev/null +++ b/tests/test_ferry_host_pinky.py @@ -0,0 +1,1063 @@ +"""Tests for @ferry/host-pinky — substrate ↔ pinky-memory bridge. + +Acceptance test uses substrate v0.1 §12 worked example as fixtures: + Entry 1: feedback, trust 0.95, scope user/brad ("Brad wants terse responses…") + Entry 2: pattern, trust 0.7, scope user/brad ("Brad's tolerance for response length…") + links → Entry 1 + +See PinkyBot issue #413 and ferry packages/host-pinky/README.md. +""" + +from __future__ import annotations + +import json +import os +import tempfile +from dataclasses import asdict + +import pytest + +from pinky_daemon.agent_registry import AgentRegistry +from pinky_daemon.ferry.host_pinky import ( + HostPinky, + parse_peer_card, + parse_pinkybot_address, +) +from pinky_daemon.ferry.substrate import ( + classify_entry_destination, + map_substrate_type_to_reflection, + populate_port_history, + to_reflection_input, + trust_to_salience, +) +from pinky_daemon.ferry.types import ( + AgentCardSelector, + FerryEnvelope, + SubstrateEntry, + SubstrateLifecycle, + SubstrateScope, + SubstrateSource, + TraversalRecord, +) + +# -- §12 worked example fixtures (substrate v0.1) ------------------------------ + + +def _entry_1_feedback() -> SubstrateEntry: + """Entry 1 from substrate v0.1 §12.2 — user-stated feedback.""" + return SubstrateEntry( + id="0c7e3d12-9f8a-4f1e-b6e2-2d8a8c1f4f9a", + type="feedback", + scope=SubstrateScope(kind="user", ref="brad"), + source=SubstrateSource( + origin="user-stated", + by="misha@pinky.local", + evidence=( + "Brad in DM, 2026-04-12 16:23 PT: 'stop summarizing what you " + "just did at the end of every response, I can read the diff'" + ), + trust=0.95, + ), + created_at="2026-04-12T16:24:11-07:00", + updated_at="2026-04-12T16:24:11-07:00", + content=( + "Brad wants terse responses with no trailing summaries.\n" + "Why: he reads the diff himself; trailing summaries are noise.\n" + "How to apply: end responses at the last substantive sentence; " + "do not append a recap of what was changed." + ), + links=[], + lifecycle=SubstrateLifecycle(state="active"), + port_history=[], + ) + + +def _entry_2_pattern() -> SubstrateEntry: + """Entry 2 from substrate v0.1 §12.3 — agent-inferred pattern.""" + return SubstrateEntry( + id="4a9b2e57-3c1d-4e8f-9a0b-7e6c5d4f3e2a", + type="pattern", + scope=SubstrateScope(kind="user", ref="brad"), + source=SubstrateSource( + origin="agent-inferred", + by="misha@pinky.local", + evidence=( + "Refs to 12 conversation entries: [list of 12 ids elided]; " + "pattern: Brad responds with curtness or 'too long' when " + "responses exceed ~3 sentences without functional content." + ), + trust=0.7, + ), + created_at="2026-05-02T09:11:32-07:00", + updated_at="2026-05-02T09:11:32-07:00", + content=( + "Brad's tolerance for response length appears bounded around 3 " + "sentences of non-functional prose. Beyond that, he often " + "pushes back ('too long', curtness, or no reply).\n" + "This pattern holds even when he hasn't explicitly asked for brevity.\n" + "Implication for response calibration: default to terse; verbose " + "only when Brad explicitly asks for depth or the content is " + "irreducibly long." + ), + links=["0c7e3d12-9f8a-4f1e-b6e2-2d8a8c1f4f9a"], + lifecycle=SubstrateLifecycle(state="active"), + port_history=[], + ) + + +def _build_envelope_substrate_batch(entries: list[SubstrateEntry]) -> FerryEnvelope: + """Wrap §12 entries in a ferry envelope addressed to barsik.""" + return FerryEnvelope( + v="0.1", + id="01891e0e-7a00-7000-9000-000000000042", # uuid-v7 placeholder + from_="misha@pinky.local", + to="ferry://pinkybot/barsik", + ts=1746846000000, + body={ + "kind": "substrate.batch", + "entries": [_entry_to_dict(e) for e in entries], + }, + traversal=[ + TraversalRecord( + broker="ferry-broker:misha-local", + at=1746846000050, + via="leaf-link", + signature="", + ), + ], + ) + + +def _entry_to_dict(e: SubstrateEntry) -> dict: + """Serialize a SubstrateEntry to the wire-shape dict (handles from_ → from).""" + raw = { + "id": e.id, + "type": e.type, + "scope": asdict(e.scope), + "source": asdict(e.source), + "created_at": e.created_at, + "updated_at": e.updated_at, + "content": e.content, + "links": list(e.links), + "lifecycle": asdict(e.lifecycle), + "port_history": [asdict(p) for p in e.port_history], + } + return raw + + +# -- Fakes --------------------------------------------------------------------- + + +class FakeRegistry: + """Minimal AgentRegistry stand-in for host-pinky tests.""" + + def __init__(self, agents: dict[str, list[AgentCardSelector]]) -> None: + self._agents = agents + + def has_agent(self, name: str) -> bool: + return name in self._agents + + def list_agents(self) -> list[dict]: + return [{"name": n} for n in self._agents] + + def get_peer_fleet_acl(self, name: str) -> list[AgentCardSelector]: + return list(self._agents.get(name, [])) + + +class FakeBroker: + """Records every dispatch_pre_authorized call. + + Note: host-pinky calls ``broker.dispatch_pre_authorized(agent_name, + broker_msg)`` — NOT ``broker.handle_inbound``. The latter is the + human-platform onboarding path that ferry inbound MUST bypass to + keep peer-fleet identities out of the ``approved_users`` table. + See PR #414 round-2 — round-1 had this wired to ``handle_inbound``, + which leaked the peer-fleet identity into the human-approval flow. + """ + + def __init__(self) -> None: + self.received: list = [] + self.dispatched_for: list[str] = [] + + async def dispatch_pre_authorized(self, agent_name, broker_msg) -> None: # noqa: ANN001 + self.dispatched_for.append(agent_name) + self.received.append(broker_msg) + + +class FakeReflection: + """Reflection stand-in for FakeMemoryStore.insert.""" + + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + if not getattr(self, "id", ""): + self.id = f"refl-{abs(hash(self.content)) % 10**8}" + + +class FakeMemoryStore: + """Records every insert call.""" + + def __init__(self) -> None: + self.inserted: list = [] + + def insert(self, reflection): # noqa: ANN001 + # Accept both the real Reflection and our FakeReflection stand-in. + # Add an id if not present. + if not getattr(reflection, "id", ""): + reflection.id = f"refl-{len(self.inserted) + 1}" + self.inserted.append(reflection) + return reflection + + +class FakeTaskStore: + """Records every create_task call.""" + + def __init__(self) -> None: + self.created: list = [] + + def create_task(self, **kwargs): + kwargs["id"] = f"task-{len(self.created) + 1}" + self.created.append(kwargs) + return kwargs + + +# -- substrate.py unit tests --------------------------------------------------- + + +class TestTypeMapping: + def test_fact_event_reference_map_to_pinky_memory_fact(self): + for sub in ("fact", "event", "reference"): + assert map_substrate_type_to_reflection(sub) == "fact" + + def test_feedback_decision_map_to_insight(self): + for sub in ("feedback", "decision"): + assert map_substrate_type_to_reflection(sub) == "insight" + + def test_pattern_maps_to_interaction_pattern(self): + assert map_substrate_type_to_reflection("pattern") == "interaction_pattern" + + def test_pending_does_not_map_to_pinky_memory(self): + assert map_substrate_type_to_reflection("pending") is None + + def test_unknown_does_not_map(self): + assert map_substrate_type_to_reflection("nonsense") is None + + +class TestTrustToSalience: + @pytest.mark.parametrize( + "trust,expected", + [ + (0.0, 1), + (0.1, 1), # 1 + round(0.4) = 1 + 0 = 1 + (0.25, 2), # 1 + round(1.0) = 2 + (0.5, 3), + (0.7, 4), # Entry 2 from §12 — trust 0.7 → salience 4 + (0.95, 5), # Entry 1 from §12 — trust 0.95 → salience 5 + (1.0, 5), + (-0.1, 1), # clamped + (1.5, 5), # clamped + ], + ) + def test_formula(self, trust, expected): + assert trust_to_salience(trust) == expected + + +class TestPortHistory: + def test_single_hop_with_traversal_record(self): + entry = _entry_1_feedback() + envelope = _build_envelope_substrate_batch([entry]) + populate_port_history(entry, envelope, "ferry://pinkybot/barsik") + + # One traversal hop in fixture → broker hop + final delivery hop = 2 entries + assert len(entry.port_history) == 2 + assert entry.port_history[0].from_ == "misha@pinky.local" + assert entry.port_history[0].to == "ferry-broker:misha-local" + assert entry.port_history[-1].to == "ferry://pinkybot/barsik" + # All hops re_grounded=False on inbound (§6.4) + assert all(p.re_grounded is False for p in entry.port_history) + # attested_by distinction (round-2 finding #5): + # broker-stamped hop gets "broker", receiver-fabricated final hop "receiver" + assert entry.port_history[0].attested_by == "broker" + assert entry.port_history[-1].attested_by == "receiver" + + def test_no_traversal_writes_single_synthetic_hop(self): + entry = _entry_1_feedback() + envelope = FerryEnvelope( + v="0.1", + id="abc", + from_="misha@pinky.local", + to="ferry://pinkybot/barsik", + ts=1, + body={"kind": "substrate.entry", "entry": _entry_to_dict(entry)}, + ) + populate_port_history(entry, envelope, "ferry://pinkybot/barsik") + assert len(entry.port_history) == 1 + assert entry.port_history[0].from_ == "misha@pinky.local" + assert entry.port_history[0].to == "ferry://pinkybot/barsik" + # No traversal records → entire hop is receiver-attested + assert entry.port_history[0].attested_by == "receiver" + + +class TestToReflectionInput: + def test_entry_1_lands_as_insight_salience_5(self): + e = _entry_1_feedback() + ref = to_reflection_input(e) + + assert ref["type"] == "insight" + assert ref["salience"] == 5 + assert ref["entities"] == ["brad"] + assert ref["source_channel"] == "ferry" + assert "Brad wants terse responses" in ref["content"] + + # Sidecar preserves source.by per §6.1 + sidecar = json.loads(ref["context"]) + assert sidecar["substrate_id"] == e.id + assert sidecar["substrate_type"] == "feedback" + assert sidecar["source"]["by"] == "misha@pinky.local" + assert sidecar["source"]["origin"] == "user-stated" + assert sidecar["scope"] == {"kind": "user", "ref": "brad"} + + def test_entry_2_lands_as_interaction_pattern_salience_4_with_links(self): + e = _entry_2_pattern() + ref = to_reflection_input(e) + + assert ref["type"] == "interaction_pattern" + assert ref["salience"] == 4 + assert ref["entities"] == ["brad"] + + sidecar = json.loads(ref["context"]) + # links surface preserved (no native pinky-memory edge surface; v0.2 §3.2.1 recipe) + assert sidecar["links"] == ["0c7e3d12-9f8a-4f1e-b6e2-2d8a8c1f4f9a"] + + def test_pending_entry_raises(self): + e = _entry_1_feedback() + e.type = "pending" # type: ignore[assignment] + with pytest.raises(ValueError, match="pinky-self"): + to_reflection_input(e) + + +class TestClassifyEntryDestination: + def test_pending_routes_to_pinky_self(self): + e = _entry_1_feedback() + e.type = "pending" # type: ignore[assignment] + assert classify_entry_destination(e) == "pinky-self" + + def test_decayed_lifecycle_skips(self): + e = _entry_1_feedback() + e.lifecycle = SubstrateLifecycle(state="decayed") + assert classify_entry_destination(e) == "skipped" + + def test_active_feedback_routes_to_pinky_memory(self): + assert classify_entry_destination(_entry_1_feedback()) == "pinky-memory" + + +# -- HostPinky.deliver() integration tests ------------------------------------- + + +@pytest.fixture +def allow_misha_acl() -> dict[str, list[AgentCardSelector]]: + """Barsik has an ACL allowing the entire pinky.local fleet.""" + return { + "barsik": [AgentCardSelector(fleet="pinky.local", agent_id="*")], + } + + +@pytest.mark.asyncio +class TestDeliverMessage: + async def test_message_routes_to_broker_with_ferry_metadata(self, allow_misha_acl): + registry = FakeRegistry(allow_misha_acl) + broker = FakeBroker() + host = HostPinky(registry=registry, broker=broker) + + envelope = FerryEnvelope( + v="0.1", + id="msg-001", + from_="misha@pinky.local", + to="ferry://pinkybot/barsik", + ts=1746846000000, + body={"kind": "message", "text": "ping from misha"}, + ) + + result = await host.deliver(envelope) + + assert result.status == "delivered" + assert len(broker.received) == 1 + bm = broker.received[0] + assert bm.platform == "ferry" + assert bm.agent_name == "barsik" + assert bm.content == "ping from misha" + assert bm.message_id == "msg-001" + assert bm.metadata["ferry"]["from"] == "misha@pinky.local" + + async def test_unknown_agent_rejected(self, allow_misha_acl): + registry = FakeRegistry(allow_misha_acl) + broker = FakeBroker() + host = HostPinky(registry=registry, broker=broker) + + envelope = FerryEnvelope( + v="0.1", + id="msg-002", + from_="misha@pinky.local", + to="ferry://pinkybot/nonexistent", + ts=1, + body={"kind": "message", "text": "x"}, + ) + + result = await host.deliver(envelope) + assert result.status == "rejected" + assert result.reason == "unknown_agent" + assert broker.received == [] + + async def test_acl_denied_rejected(self): + # Empty ACL → deny all + registry = FakeRegistry({"barsik": []}) + broker = FakeBroker() + host = HostPinky(registry=registry, broker=broker) + + envelope = FerryEnvelope( + v="0.1", + id="msg-003", + from_="pulse@studio@sigil", + to="ferry://pinkybot/barsik", + ts=1, + body={"kind": "message", "text": "x"}, + ) + + result = await host.deliver(envelope) + assert result.status == "rejected" + assert result.reason == "acl_denied" + assert broker.received == [] + + +@pytest.mark.asyncio +class TestDeliverSubstrateBatch: + """Acceptance test: substrate v0.1 §12 worked example, ported A → B.""" + + async def test_two_entries_land_in_pinky_memory_with_correct_shape( + self, allow_misha_acl + ): + registry = FakeRegistry(allow_misha_acl) + broker = FakeBroker() + memory = FakeMemoryStore() + tasks = FakeTaskStore() + host = HostPinky( + registry=registry, + broker=broker, + memory_store=memory, + task_store=tasks, + ) + + envelope = _build_envelope_substrate_batch( + [_entry_1_feedback(), _entry_2_pattern()] + ) + + # Patch out the real Reflection import inside the host_pinky module so + # we can run the test without the pinky_memory dependency providing a + # concrete model. The test substitutes our FakeReflection. + import pinky_daemon.ferry.host_pinky as host_pinky_mod + + original_insert = host_pinky_mod.HostPinky._insert_reflection + + def _fake_insert(self, ref_kwargs): # noqa: ANN001 + r = FakeReflection(**ref_kwargs) + self._memory_store.insert(r) + return r.id + + host_pinky_mod.HostPinky._insert_reflection = _fake_insert # type: ignore[method-assign] + try: + result = await host.deliver(envelope) + finally: + host_pinky_mod.HostPinky._insert_reflection = original_insert # type: ignore[method-assign] + + assert result.status == "delivered" + assert len(memory.inserted) == 2 + assert len(tasks.created) == 0 # no `pending` entries in §12 example + + # Entry 1: feedback → insight, salience 5 + e1 = memory.inserted[0] + assert e1.type == "insight" + assert e1.salience == 5 + assert "Brad wants terse responses" in e1.content + sidecar1 = json.loads(e1.context) + assert sidecar1["source"]["by"] == "misha@pinky.local" # §6.1 preserved + assert sidecar1["substrate_type"] == "feedback" + assert len(sidecar1["port_history"]) >= 1 + assert sidecar1["port_history"][-1]["to"] == "ferry://pinkybot/barsik" + + # Entry 2: pattern → interaction_pattern, salience 4, links preserved + e2 = memory.inserted[1] + assert e2.type == "interaction_pattern" + assert e2.salience == 4 + sidecar2 = json.loads(e2.context) + assert sidecar2["links"] == ["0c7e3d12-9f8a-4f1e-b6e2-2d8a8c1f4f9a"] + + +# -- Address parsing ----------------------------------------------------------- + + +class TestAddressParsing: + def test_parse_pinkybot_address_extracts_agent(self): + assert parse_pinkybot_address("ferry://pinkybot/barsik") == "barsik" + + def test_parse_pinkybot_address_rejects_other_fleets(self): + assert parse_pinkybot_address("ferry://sigil/pulse") is None + assert parse_pinkybot_address("misha@pinky.local") is None + + def test_parse_peer_card_at_form(self): + assert parse_peer_card("pulse@studio@sigil") == ("sigil", "pulse@studio@sigil") + assert parse_peer_card("misha@pinky.local") == ("pinky.local", "misha@pinky.local") + + def test_parse_peer_card_ferry_form(self): + assert parse_peer_card("ferry://sigil/pulse") == ("sigil", "ferry://sigil/pulse") + + +class TestAgentCardSelector: + def test_empty_selector_rejected(self): + with pytest.raises(ValueError, match="at least one"): + AgentCardSelector() + + def test_fleet_wildcard_matches_any_agent(self): + sel = AgentCardSelector(fleet="sigil", agent_id="*") + assert sel.matches("sigil", "pulse@studio@sigil") is True + assert sel.matches("sigil", "kain@ces-mini@sigil") is True + assert sel.matches("pinky.local", "misha@pinky.local") is False + + def test_specific_match(self): + sel = AgentCardSelector(fleet="sigil", agent_id="pulse@studio@sigil") + assert sel.matches("sigil", "pulse@studio@sigil") is True + assert sel.matches("sigil", "kain@ces-mini@sigil") is False + + +# -- AgentRegistry peer_fleet_acl persistence --------------------------------- + + +@pytest.fixture +def real_registry(): + fd, path = tempfile.mkstemp(suffix=".db") + os.close(fd) + r = AgentRegistry(db_path=path) + r.register("barsik", model="opus") + yield r + r.close() + os.unlink(path) + + +class TestPeerFleetAclPersistence: + def test_unset_returns_empty_list(self, real_registry): + assert real_registry.get_peer_fleet_acl("barsik") == [] + + def test_unknown_agent_returns_empty_list(self, real_registry): + assert real_registry.get_peer_fleet_acl("nonexistent") == [] + + def test_has_agent(self, real_registry): + assert real_registry.has_agent("barsik") is True + assert real_registry.has_agent("nonexistent") is False + + def test_set_and_get_roundtrip(self, real_registry): + real_registry.set_peer_fleet_acl( + "barsik", + [ + {"fleet": "sigil", "agent_id": "pulse@studio@sigil", "pinky_type": None}, + {"fleet": "sigil", "agent_id": "kain@ces-mini@sigil", "pinky_type": None}, + ], + ) + loaded = real_registry.get_peer_fleet_acl("barsik") + assert len(loaded) == 2 + assert loaded[0]["fleet"] == "sigil" + assert loaded[0]["agent_id"] == "pulse@studio@sigil" + + def test_set_drops_empty_selectors(self, real_registry): + real_registry.set_peer_fleet_acl( + "barsik", + [ + {"fleet": "sigil", "agent_id": "pulse@studio@sigil"}, + {"fleet": "", "agent_id": "", "pinky_type": ""}, # empty — dropped + {}, # empty — dropped + "not-a-dict", # type-invalid — dropped + ], + ) + loaded = real_registry.get_peer_fleet_acl("barsik") + assert len(loaded) == 1 + assert loaded[0]["agent_id"] == "pulse@studio@sigil" + + def test_set_replaces_not_merges(self, real_registry): + real_registry.set_peer_fleet_acl( + "barsik", [{"fleet": "sigil", "agent_id": "*"}] + ) + real_registry.set_peer_fleet_acl( + "barsik", [{"fleet": "pinky.local", "agent_id": "*"}] + ) + loaded = real_registry.get_peer_fleet_acl("barsik") + assert len(loaded) == 1 + assert loaded[0]["fleet"] == "pinky.local" + + def test_add_idempotent(self, real_registry): + added1 = real_registry.add_peer_fleet_acl( + "barsik", fleet="sigil", agent_id="pulse@studio@sigil" + ) + added2 = real_registry.add_peer_fleet_acl( + "barsik", fleet="sigil", agent_id="pulse@studio@sigil" + ) + assert added1 is True + assert added2 is True # idempotent — already-present is success + assert len(real_registry.get_peer_fleet_acl("barsik")) == 1 + + def test_add_rejects_empty(self, real_registry): + added = real_registry.add_peer_fleet_acl("barsik") + assert added is False + assert real_registry.get_peer_fleet_acl("barsik") == [] + + def test_remove(self, real_registry): + real_registry.add_peer_fleet_acl("barsik", fleet="sigil", agent_id="pulse@studio@sigil") + real_registry.add_peer_fleet_acl("barsik", fleet="sigil", agent_id="kain@ces-mini@sigil") + removed = real_registry.remove_peer_fleet_acl( + "barsik", fleet="sigil", agent_id="pulse@studio@sigil" + ) + assert removed == 1 + loaded = real_registry.get_peer_fleet_acl("barsik") + assert len(loaded) == 1 + assert loaded[0]["agent_id"] == "kain@ces-mini@sigil" + + def test_remove_no_match_returns_zero(self, real_registry): + real_registry.add_peer_fleet_acl("barsik", fleet="sigil", agent_id="*") + removed = real_registry.remove_peer_fleet_acl( + "barsik", fleet="other", agent_id="x" + ) + assert removed == 0 + + +@pytest.mark.asyncio +class TestHostPinkyAgainstRealRegistry: + """End-to-end: real AgentRegistry + FakeBroker confirms ACL plumbing works.""" + + async def test_acl_allows_then_denies_after_remove(self, real_registry): + broker = FakeBroker() + host = HostPinky(registry=real_registry, broker=broker) + + envelope = FerryEnvelope( + v="0.1", + id="msg-allow-flow", + from_="pulse@studio@sigil", + to="ferry://pinkybot/barsik", + ts=1, + body={"kind": "message", "text": "hello"}, + ) + + # Default: deny-all + result = await host.deliver(envelope) + assert result.status == "rejected" + assert result.reason == "acl_denied" + + # Allow Pulse + real_registry.add_peer_fleet_acl( + "barsik", fleet="sigil", agent_id="pulse@studio@sigil" + ) + result = await host.deliver(envelope) + assert result.status == "delivered" + assert len(broker.received) == 1 + + # Remove Pulse — back to deny + real_registry.remove_peer_fleet_acl( + "barsik", fleet="sigil", agent_id="pulse@studio@sigil" + ) + result = await host.deliver(envelope) + assert result.status == "rejected" + assert result.reason == "acl_denied" + + async def test_unknown_agent_uses_real_has_agent(self, real_registry): + broker = FakeBroker() + host = HostPinky(registry=real_registry, broker=broker) + + envelope = FerryEnvelope( + v="0.1", + id="msg-unknown", + from_="misha@pinky.local", + to="ferry://pinkybot/nonexistent", + ts=1, + body={"kind": "message", "text": "hi"}, + ) + + result = await host.deliver(envelope) + assert result.status == "rejected" + assert result.reason == "unknown_agent" + + +# -- API: peer-fleet-acl endpoints -------------------------------------------- + + +@pytest.fixture +def api_client(): + """FastAPI TestClient with a fresh registry containing 'barsik'.""" + from fastapi.testclient import TestClient + + from pinky_daemon.api import create_api + + fd, path = tempfile.mkstemp(suffix=".db") + os.close(fd) + app = create_api(max_sessions=10, default_working_dir="/tmp", db_path=path) + client = TestClient(app) + # Register barsik via the API + r = client.post("/agents", json={"name": "barsik", "model": "opus"}) + assert r.status_code in (200, 201), f"agent register failed: {r.status_code} {r.text}" + yield client + client.close() + os.unlink(path) + + +class TestPeerFleetAclApi: + def test_get_empty_for_known_agent(self, api_client): + r = api_client.get("/agents/barsik/peer-fleet-acl") + assert r.status_code == 200 + assert r.json() == {"agent": "barsik", "selectors": []} + + def test_get_404_for_unknown_agent(self, api_client): + r = api_client.get("/agents/nonexistent/peer-fleet-acl") + assert r.status_code == 404 + + def test_post_adds_selector(self, api_client): + r = api_client.post( + "/agents/barsik/peer-fleet-acl", + json={"fleet": "sigil", "agent_id": "pulse@studio@sigil"}, + ) + assert r.status_code == 200 + body = r.json() + assert body["added"] is True + assert len(body["selectors"]) == 1 + assert body["selectors"][0]["fleet"] == "sigil" + assert body["selectors"][0]["agent_id"] == "pulse@studio@sigil" + + def test_post_rejects_empty_selector(self, api_client): + r = api_client.post( + "/agents/barsik/peer-fleet-acl", + json={"fleet": "", "agent_id": "", "pinky_type": ""}, + ) + assert r.status_code == 400 + assert "at least one" in r.json()["detail"] + + def test_post_idempotent(self, api_client): + for _ in range(3): + r = api_client.post( + "/agents/barsik/peer-fleet-acl", + json={"fleet": "sigil", "agent_id": "pulse@studio@sigil"}, + ) + assert r.status_code == 200 + r = api_client.get("/agents/barsik/peer-fleet-acl") + assert len(r.json()["selectors"]) == 1 + + def test_put_replaces_full_acl(self, api_client): + # Add two + api_client.post( + "/agents/barsik/peer-fleet-acl", + json={"fleet": "sigil", "agent_id": "pulse@studio@sigil"}, + ) + api_client.post( + "/agents/barsik/peer-fleet-acl", + json={"fleet": "sigil", "agent_id": "kain@ces-mini@sigil"}, + ) + # PUT to replace with just one different selector + r = api_client.put( + "/agents/barsik/peer-fleet-acl", + json={"selectors": [{"fleet": "pinky.local", "agent_id": "*"}]}, + ) + assert r.status_code == 200 + body = r.json() + assert len(body["selectors"]) == 1 + assert body["selectors"][0]["fleet"] == "pinky.local" + + def test_delete_by_query_removes_matching(self, api_client): + api_client.post( + "/agents/barsik/peer-fleet-acl", + json={"fleet": "sigil", "agent_id": "pulse@studio@sigil"}, + ) + api_client.post( + "/agents/barsik/peer-fleet-acl", + json={"fleet": "sigil", "agent_id": "kain@ces-mini@sigil"}, + ) + r = api_client.delete( + "/agents/barsik/peer-fleet-acl", + params={"fleet": "sigil", "agent_id": "pulse@studio@sigil"}, + ) + assert r.status_code == 200 + assert r.json()["removed"] == 1 + remaining = r.json()["selectors"] + assert len(remaining) == 1 + assert remaining[0]["agent_id"] == "kain@ces-mini@sigil" + + def test_delete_no_match_returns_zero(self, api_client): + api_client.post( + "/agents/barsik/peer-fleet-acl", + json={"fleet": "sigil", "agent_id": "*"}, + ) + r = api_client.delete( + "/agents/barsik/peer-fleet-acl", + params={"fleet": "other", "agent_id": "pulse"}, + ) + assert r.status_code == 200 + assert r.json()["removed"] == 0 + + +# -- Round-2 additions: transient ACL, wildcard remove, pending substrate, broker integration -- + + +class _RaisingRegistry: + """Registry stub whose get_peer_fleet_acl raises a sqlite3.Error. + + Models the "DB locked / schema mismatch during rolling deploy" scenario + that Misha's PR #414 review #1 flagged. See round-2 finding #1. + """ + + def __init__(self) -> None: + import sqlite3 as _sq3 + self._exc = _sq3.OperationalError("database is locked") + + def has_agent(self, name: str) -> bool: + return name == "barsik" + + def get_peer_fleet_acl(self, name: str): # noqa: ANN001 + raise self._exc + + +@pytest.mark.asyncio +class TestTransientACLLoadFailure: + """A sqlite3 error during ACL load yields ``transient_failure``, + NOT ``rejected``/``acl_denied`` (round-2 finding #1). + + Without this distinction, a brief DB lock during a rolling deploy + permanently drops a real ACL-allowed message because the broker + can't tell "operator hiccup" from "policy denial." + """ + + async def test_sqlite_error_during_acl_load_yields_transient_failure(self): + registry = _RaisingRegistry() + broker = FakeBroker() + host = HostPinky(registry=registry, broker=broker) + + envelope = FerryEnvelope( + v="0.1", + id="msg-transient", + from_="misha@pinky.local", + to="ferry://pinkybot/barsik", + ts=1, + body={"kind": "message", "text": "x"}, + ) + + result = await host.deliver(envelope) + + assert result.status == "transient_failure" + assert result.reason == "acl_load_failed" + assert "database is locked" in result.detail.get("detail", "") + # Critically: did NOT degrade to rejected/acl_denied + assert result.status != "rejected" + # Broker not invoked (we never got to dispatch) + assert broker.received == [] + # Stats incremented on the right counter + assert host.stats["transient_failures"] == 1 + assert host.stats["rejected"] == 0 + + +class TestACLRemoveWildcardSemantics: + """The exact-match contract on ``remove_peer_fleet_acl`` is documented + on the method (round-2 nit). This test pins the documented behavior: + a stored ``agent_id="*"`` is removed only by passing ``agent_id="*"``. + """ + + def test_remove_without_wildcard_does_not_match_stored_wildcard(self, real_registry): + # Store a fleet-wide allow + real_registry.add_peer_fleet_acl("barsik", fleet="sigil", agent_id="*") + + # Forgetting the wildcard MUST silently zero-remove (don't surprise-delete) + removed = real_registry.remove_peer_fleet_acl("barsik", fleet="sigil") + assert removed == 0 + assert len(real_registry.get_peer_fleet_acl("barsik")) == 1 + + # Passing the exact stored selector removes it + removed = real_registry.remove_peer_fleet_acl( + "barsik", fleet="sigil", agent_id="*", + ) + assert removed == 1 + assert real_registry.get_peer_fleet_acl("barsik") == [] + + +@pytest.mark.asyncio +class TestDeliverPendingSubstrate: + """End-to-end test of the ``pending`` substrate path through ``deliver()``. + + Round-2 finding #2 / test gap: round-1 only exercised entries that + routed to pinky-memory; the pinky-self ``_insert_task`` path was + untested, which kept a latent ternary bug undetectable. This fixture + routes a ``pending`` entry through the full deliver() flow and + confirms it lands in the task store. + """ + + async def test_pending_entry_routes_to_task_store(self, allow_misha_acl): + registry = FakeRegistry(allow_misha_acl) + broker = FakeBroker() + memory = FakeMemoryStore() + tasks = FakeTaskStore() + host = HostPinky( + registry=registry, + broker=broker, + memory_store=memory, + task_store=tasks, + ) + + pending_entry = SubstrateEntry( + id="pending-001", + type="pending", # routes to pinky-self per classify_entry_destination + scope=SubstrateScope(kind="user", ref="brad"), + source=SubstrateSource( + origin="agent-inferred", + by="misha@pinky.local", + evidence="Brad mentioned needing to follow up with Matt re: NATS creds", + trust=0.85, + ), + created_at="2026-05-09T10:00:00-07:00", + updated_at="2026-05-09T10:00:00-07:00", + content="Follow up with Matt about NATS creds for ferry demo", + links=[], + lifecycle=SubstrateLifecycle(state="pending"), + port_history=[], + ) + + envelope = FerryEnvelope( + v="0.1", + id="msg-pending", + from_="misha@pinky.local", + to="ferry://pinkybot/barsik", + ts=1746846000000, + body={ + "kind": "substrate.entry", + "entry": _entry_to_dict(pending_entry), + }, + ) + + result = await host.deliver(envelope) + + assert result.status == "delivered" + # Pending → pinky-self, not pinky-memory + assert len(tasks.created) == 1 + assert len(memory.inserted) == 0 + # Confirm the task carries the right substrate provenance + task = tasks.created[0] + assert "Follow up with Matt" in task["title"] or "Follow up with Matt" in task.get("description", "") + assert task["assigned_agent"] == "barsik" + # IngestResult shape + results = result.detail["ingested"] + assert results[0]["target_store"] == "pinky-self" + assert results[0]["target_id"] # non-empty (round-2 finding #2 — ternary returned object str before) + + +@pytest.mark.asyncio +class TestBrokerIntegrationFerryInbound: + """Regression guard for round-2 blocking finding (broker boundary leak). + + Round-1 wired host-pinky's message dispatch through ``broker.handle_inbound`` + — which runs the human-platform onboarding flow (``get_user_status`` → + ``add_pending_user`` → ``/approve_…`` Telegram prompt to the owner) and + writes the ferry sender_id into the human ``approved_users`` table. That + defeated the load-bearing identity-primitive separation claim of the PR. + + Round-2 fixed it by routing through ``broker.dispatch_pre_authorized`` + (a public broker entry point that bypasses onboarding). This test pipes + a ferry envelope through real Broker + real AgentRegistry and confirms: + + 1. ``approved_users`` does NOT contain the ferry sender_id + 2. The owner does NOT receive an ``/approve_…`` Telegram prompt + 3. No pending message is queued under the ferry sender_id + + If a future refactor of ``handle_inbound`` re-introduces the leak, this + test fails. None of the FakeBroker tests above would have caught it. + """ + + async def test_ferry_inbound_does_not_touch_human_approval_path(self, real_registry): + # Real Broker, real Registry. send_callback records every outbound; + # we assert no /approve_ prompt was emitted. + from pinky_daemon.broker import MessageBroker + + sent: list[tuple[str, str, str, str]] = [] + + async def record_send(agent_name, platform, chat_id, content): # noqa: ANN001 + sent.append((agent_name, platform, chat_id, content)) + + broker = MessageBroker( + real_registry, + session_manager=None, # not exercised — _route_streaming finds no session + send_callback=record_send, + ) + + # Allow misha@pinky.local fleet-wide + real_registry.add_peer_fleet_acl( + "barsik", fleet="pinky.local", agent_id="*", + ) + + host = HostPinky(registry=real_registry, broker=broker) + envelope = FerryEnvelope( + v="0.1", + id="msg-ferry-integration", + from_="misha@pinky.local", + to="ferry://pinkybot/barsik", + ts=1746846000000, + body={"kind": "message", "text": "hello from misha across the fleet"}, + ) + + result = await host.deliver(envelope) + assert result.status == "delivered" + + # 1. ferry sender NOT in approved_users (the human-identity table) + ferry_user_id = "ferry:misha@pinky.local" + status = real_registry.get_user_status("barsik", ferry_user_id) + assert status is None, ( + f"ferry sender_id {ferry_user_id!r} leaked into approved_users " + f"with status={status!r} — round-1 bug regression" + ) + + # 2. no /approve_ prompt sent (would have gone to owner on round-1 bug) + for _, _, _, content in sent: + assert "/approve_" not in content, ( + f"owner received /approve_ prompt for ferry inbound: {content!r}" + ) + assert "wants to talk to" not in content, ( + f"owner received human-onboarding prompt for ferry inbound: {content!r}" + ) + + # 3. no pending message queued under ferry sender_id + pending = real_registry.get_pending_messages("barsik", ferry_user_id) + assert pending == [], ( + f"ferry message queued as pending under {ferry_user_id!r}: {pending}" + ) + + async def test_acl_denied_does_not_leak_either(self, real_registry): + """Same regression guard, denied path. ACL-deny must not fall back + to onboarding either — the agent should never see the message and + the owner should never see an /approve_ prompt.""" + from pinky_daemon.broker import MessageBroker + + sent: list[tuple[str, str, str, str]] = [] + + async def record_send(agent_name, platform, chat_id, content): # noqa: ANN001 + sent.append((agent_name, platform, chat_id, content)) + + broker = MessageBroker( + real_registry, session_manager=None, send_callback=record_send, + ) + # Empty ACL — default-deny + host = HostPinky(registry=real_registry, broker=broker) + envelope = FerryEnvelope( + v="0.1", + id="msg-ferry-deny", + from_="pulse@studio@sigil", # not on barsik's empty ACL + to="ferry://pinkybot/barsik", + ts=1, + body={"kind": "message", "text": "x"}, + ) + + result = await host.deliver(envelope) + assert result.status == "rejected" + assert result.reason == "acl_denied" + + # Owner gets nothing + assert sent == [] + # Registry untouched + assert real_registry.get_user_status( + "barsik", "ferry:pulse@studio@sigil", + ) is None diff --git a/tests/test_ferry_outbound.py b/tests/test_ferry_outbound.py new file mode 100644 index 00000000..f2717d2f --- /dev/null +++ b/tests/test_ferry_outbound.py @@ -0,0 +1,357 @@ +"""Tests for @ferry/host-pinky outbound side — envelope build + publish. + +Companion to test_ferry_host_pinky.py (inbound). Covers: +- address parsing (canonical ferry:// + at-form, edge cases) +- subject derivation (token sanitization) +- allowlist matching (exact, wildcards, default-deny) +- envelope builder (kind enforcement, ts/id stamping) +- envelope-to-wire serialization (key rename, optional-field elision) +- MeshSender (configured/diagnostics/send happy + failure paths) + +The MeshSender tests mock subprocess.run rather than spawning the real +``nats`` CLI — the contract under test is "we shell out the right thing +and surface the right result", not "the nats CLI works" (separately +proven by the smoke fired 2026-05-10 12:14 PDT). + +PinkyBot issue: #418. +""" + +from __future__ import annotations + +import json +import subprocess +from unittest.mock import patch + +import pytest + +from pinky_daemon.ferry.outbound import ( + MeshSender, + SendResult, + allowlist_matches, + build_envelope, + derive_subject, + envelope_to_wire, + parse_address, +) + +# -- parse_address ------------------------------------------------------------- + + +class TestParseAddress: + def test_canonical_ferry_scheme(self): + assert parse_address("ferry://pulse/pulse") == ("pulse", "pulse") + + def test_at_form(self): + assert parse_address("pulse@pulse") == ("pulse", "pulse") + + def test_at_form_with_scope(self): + # Rightmost @ is the fleet boundary; scope segments stay in the slug. + assert parse_address("pulse@studio@sigil") == ("sigil", "pulse@studio") + + def test_canonical_with_complex_slug(self): + assert parse_address("ferry://sigil/pulse@studio") == ("sigil", "pulse@studio") + + @pytest.mark.parametrize("addr", [ + "", + " ", + "no-fleet", + "ferry://", + "ferry://onlyfleet", + "ferry://onlyfleet/", + "ferry:///onlyagent", + "@nofleet", + "noslug@", + None, + ]) + def test_invalid_returns_none(self, addr): + assert parse_address(addr) is None # type: ignore[arg-type] + + def test_strips_whitespace(self): + assert parse_address(" pulse@pulse ") == ("pulse", "pulse") + + +# -- derive_subject ------------------------------------------------------------ + + +class TestDeriveSubject: + def test_simple(self): + assert derive_subject("pulse", "pulse") == "ferry.pulse.pulse.inbox" + + def test_at_in_slug_sanitized(self): + assert derive_subject("sigil", "pulse@studio") == "ferry.sigil.pulse-studio.inbox" + + def test_dot_in_slug_sanitized(self): + # dots are NATS subject separators — must not appear in tokens + assert derive_subject("pinky.local", "misha") == "ferry.pinky-local.misha.inbox" + + +# -- allowlist_matches --------------------------------------------------------- + + +class TestAllowlistMatches: + def test_empty_allowlist_denies_all(self): + assert allowlist_matches("pulse@pulse", []) is False + + def test_exact_match(self): + assert allowlist_matches("pulse@pulse", ["pulse@pulse"]) is True + + def test_no_match(self): + assert allowlist_matches("pulse@pulse", ["misha@pulse"]) is False + + def test_wildcard_agent_for_fleet(self): + assert allowlist_matches("pulse@pulse", ["*@pulse"]) is True + assert allowlist_matches("misha@pulse", ["*@pulse"]) is True + assert allowlist_matches("pulse@sigil", ["*@pulse"]) is False + + def test_wildcard_fleet_for_agent(self): + assert allowlist_matches("pulse@pulse", ["pulse@*"]) is True + assert allowlist_matches("pulse@sigil", ["pulse@*"]) is True + assert allowlist_matches("misha@pulse", ["pulse@*"]) is False + + def test_global_wildcard(self): + assert allowlist_matches("pulse@pulse", ["*"]) is True + assert allowlist_matches("anything@anything", ["*@*"]) is True + + def test_unparseable_target_denied(self): + assert allowlist_matches("not-an-address", ["*"]) is False + + def test_invalid_pattern_skipped_not_matched(self): + # Bad pattern shouldn't crash; good pattern still matches. + assert allowlist_matches( + "pulse@pulse", + ["", "garbage", "pulse@pulse"], + ) is True + + +# -- build_envelope ------------------------------------------------------------ + + +class TestBuildEnvelope: + def _body(self, **extras): + return {"kind": "msg", "text": "hi", **extras} + + def test_basic(self): + env = build_envelope( + from_="ferry://pinkybot/barsik", + to="pulse@pulse", + body=self._body(), + ) + assert env.v == "0.1" + assert env.from_ == "ferry://pinkybot/barsik" + assert env.to == "pulse@pulse" + assert env.body == {"kind": "msg", "text": "hi"} + assert env.id # uuid stamped + assert env.ts > 0 + + def test_kind_enforced(self): + with pytest.raises(ValueError, match="must be a dict containing a 'kind' field"): + build_envelope(from_="x", to="y", body={"text": "no kind"}) + + def test_body_must_be_dict(self): + with pytest.raises(ValueError): + build_envelope(from_="x", to="y", body="not-a-dict") # type: ignore[arg-type] + + def test_correlation_id_passthrough(self): + env = build_envelope( + from_="x", to="y", body=self._body(), + correlation_id="cid-123", + ) + assert env.correlation_id == "cid-123" + + def test_priority_validated(self): + for p in ("low", "normal", "high", "urgent"): + env = build_envelope(from_="x", to="y", body=self._body(), priority=p) + assert env.priority == p + # Bogus priority falls back to "normal" (don't crash on caller typo). + env = build_envelope(from_="x", to="y", body=self._body(), priority="bogus") + assert env.priority == "normal" + + +# -- envelope_to_wire ---------------------------------------------------------- + + +class TestEnvelopeToWire: + def _env(self, **kw): + return build_envelope( + from_="ferry://pinkybot/barsik", + to="pulse@pulse", + body={"kind": "msg", "text": "hi"}, + **kw, + ) + + def test_renames_from(self): + wire = json.loads(envelope_to_wire(self._env())) + assert "from" in wire + assert "from_" not in wire + assert wire["from"] == "ferry://pinkybot/barsik" + + def test_required_fields(self): + wire = json.loads(envelope_to_wire(self._env())) + for field in ("v", "id", "from", "to", "ts", "body"): + assert field in wire + + def test_optional_fields_elided_at_default(self): + wire = json.loads(envelope_to_wire(self._env())) + # Defaults shouldn't bloat the wire. + assert "priority" not in wire # default "normal" + assert "wake" not in wire # default "if-idle" + assert "ack" not in wire # default "delivery" + assert "ttl_ms" not in wire + assert "correlation_id" not in wire + assert "reply_to" not in wire + assert "subject" not in wire + assert "traversal" not in wire + assert "headers" not in wire + + def test_optional_fields_emitted_when_set(self): + env = self._env(correlation_id="cid-1", reply_to="cid-0", priority="high") + wire = json.loads(envelope_to_wire(env)) + assert wire["correlation_id"] == "cid-1" + assert wire["reply_to"] == "cid-0" + assert wire["priority"] == "high" + + +# -- MeshSender ---------------------------------------------------------------- + + +class TestMeshSenderConfig: + def test_unconfigured_when_no_env(self): + s = MeshSender(env={}, bin_path="") + assert s.configured is False + diag = s.diagnostics() + assert diag["configured"] is False + assert diag["user_set"] is False + assert diag["password_set"] is False + # Password value never in diagnostics. + assert "password" not in diag or diag.get("password") in (None, "") + + def test_configured_with_full_env(self, tmp_path): + # Synthesize a fake nats binary that exists, so configured=True. + fake_bin = tmp_path / "nats" + fake_bin.write_text("#!/bin/sh\nexit 0\n") + fake_bin.chmod(0o755) + s = MeshSender( + env={ + "PINKYBOT_FLEET_USER": "pinkybot_fleet", + "PINKYBOT_FLEET_PASSWORD": "secret", + "PINKYBOT_FLEET_NATS_URL": "wss://example.test", + }, + bin_path=str(fake_bin), + ) + assert s.configured is True + diag = s.diagnostics() + assert diag["user_set"] is True + assert diag["password_set"] is True + assert diag["nats_url"] == "wss://example.test" + # Diagnostics MUST NOT leak the password value itself. + for v in diag.values(): + assert v != "secret" + + +class TestMeshSenderSend: + def _sender(self, tmp_path): + fake_bin = tmp_path / "nats" + fake_bin.write_text("#!/bin/sh\nexit 0\n") + fake_bin.chmod(0o755) + return MeshSender( + env={ + "PINKYBOT_FLEET_USER": "pinkybot_fleet", + "PINKYBOT_FLEET_PASSWORD": "secret", + "PINKYBOT_FLEET_NATS_URL": "wss://example.test", + }, + bin_path=str(fake_bin), + ) + + def _envelope(self, **kw): + return build_envelope( + from_="ferry://pinkybot/barsik", + to="pulse@pulse", + body={"kind": "smoke", "text": "hi"}, + **kw, + ) + + def test_unconfigured_returns_error(self): + s = MeshSender(env={}, bin_path="") + result = s.send(self._envelope(correlation_id="cid-x")) + assert result.sent is False + assert "not configured" in (result.error or "") + assert result.correlation_id == "cid-x" + + def test_unparseable_target(self, tmp_path): + env = self._envelope() + env.to = "not-an-address" + result = self._sender(tmp_path).send(env) + assert result.sent is False + assert "unparseable target" in (result.error or "") + + def test_happy_path(self, tmp_path): + sender = self._sender(tmp_path) + env = self._envelope(correlation_id="cid-happy") + + with patch("subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="Published 165 bytes\n", stderr="", + ) + result = sender.send(env) + + assert result.sent is True + assert result.error is None + assert result.correlation_id == "cid-happy" + assert result.subject == "ferry.pulse.pulse.inbox" + + # Verify the subprocess call had the right shape. + call = mock_run.call_args + cmd = call.args[0] + assert cmd[0].endswith("nats") + assert "--user" in cmd + assert "pinkybot_fleet" in cmd + assert "--password" in cmd + assert "secret" in cmd + assert "pub" in cmd + assert "ferry.pulse.pulse.inbox" in cmd + # Wire payload is the last arg; verify it's the envelope JSON. + payload = json.loads(cmd[-1]) + assert payload["from"] == "ferry://pinkybot/barsik" + assert payload["to"] == "pulse@pulse" + assert payload["correlation_id"] == "cid-happy" + + def test_nats_failure_surfaces_error(self, tmp_path): + sender = self._sender(tmp_path) + with patch("subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=1, stdout="", + stderr="nats: error: auth failed\n", + ) + result = sender.send(self._envelope()) + + assert result.sent is False + assert result.error and "auth failed" in result.error + + def test_timeout_surfaces_error(self, tmp_path): + sender = self._sender(tmp_path) + with patch("subprocess.run") as mock_run: + mock_run.side_effect = subprocess.TimeoutExpired(cmd="nats", timeout=10.0) + result = sender.send(self._envelope()) + + assert result.sent is False + assert "timed out" in (result.error or "") + + def test_password_not_in_error_message(self, tmp_path): + """Defense in depth — if a failure surfaces stderr, password must + not appear (we don't put it in stderr by default, but assert it).""" + sender = self._sender(tmp_path) + with patch("subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=1, stdout="", + stderr="nats: error: connection refused\n", + ) + result = sender.send(self._envelope()) + assert result.sent is False + assert "secret" not in (result.error or "") + + +class TestSendResultShape: + def test_smoke(self): + # Quick proof of the dataclass shape — used by API serializer. + r = SendResult(sent=True, correlation_id="cid", subject="s", ts=1, error=None) + assert r.sent and r.correlation_id == "cid" diff --git a/tests/test_mesh_store.py b/tests/test_mesh_store.py new file mode 100644 index 00000000..f90100fc --- /dev/null +++ b/tests/test_mesh_store.py @@ -0,0 +1,299 @@ +"""Tests for MeshStore — ferry envelope log + federation peer registry. + +Covers: +- schema migration / idempotent init +- log_message: inbound/outbound rows, side-effect peer touch, error semantics +- get_inbox: filters (kind, sender, direction), pagination, ordering +- get_message_by_correlation_id: most-recent on duplicate, missing returns None +- federation_peers: upsert (config wins), list filters, get/remove +- defensive: malformed body fields don't crash deserialization + +PinkyBot issue: #419. +""" + +from __future__ import annotations + +import time + +import pytest + +from pinky_daemon.mesh_store import MeshStore + + +@pytest.fixture +def store(tmp_path): + s = MeshStore(db_path=str(tmp_path / "mesh.db")) + yield s + s.close() + + +# -- Schema -------------------------------------------------------------------- + + +class TestSchema: + def test_init_idempotent(self, tmp_path): + path = str(tmp_path / "mesh.db") + s1 = MeshStore(db_path=path) + s1.close() + # Second open should not crash on existing tables. + s2 = MeshStore(db_path=path) + s2.close() + + +# -- log_message --------------------------------------------------------------- + + +class TestLogMessage: + def test_outbound_success_logs_and_touches_peer(self, store): + rid = store.log_message( + direction="outbound", + local_agent="barsik", + remote_fleet="pulse", + remote_agent="pulse", + correlation_id="cid-1", + kind="msg", + body={"text": "hi", "kind": "msg"}, + envelope_ts=1234567890, + ) + assert rid > 0 + peer = store.get_peer("pulse", "pulse") + assert peer is not None + assert peer.seed_source == "observed" + + def test_outbound_failure_does_not_touch_peer(self, store): + store.log_message( + direction="outbound", + local_agent="barsik", + remote_fleet="ghost", + remote_agent="ghost", + correlation_id="cid-fail", + kind="msg", + body={"text": "x", "kind": "msg"}, + error="nats: error: connection refused", + ) + # Failed sends shouldn't auto-register the unreachable peer. + assert store.get_peer("ghost", "ghost") is None + + def test_inbound_always_touches_peer(self, store): + store.log_message( + direction="inbound", + local_agent="barsik", + remote_fleet="pulse", + remote_agent="misha", + correlation_id="cid-in-1", + kind="message", + body={"text": "hello", "kind": "message"}, + error="rejected:acl_denied", # even rejected inbound counts as "we saw them" + ) + peer = store.get_peer("pulse", "misha") + assert peer is not None + assert peer.seed_source == "observed" + + def test_count_messages(self, store): + for i in range(5): + store.log_message( + direction="outbound", + local_agent="barsik", + remote_fleet="pulse", + remote_agent="pulse", + correlation_id=f"cid-{i}", + kind="msg", + body={"kind": "msg"}, + ) + for _ in range(3): + store.log_message( + direction="inbound", + local_agent="pushok", + remote_fleet="pulse", + remote_agent="misha", + kind="message", + body={"kind": "message"}, + ) + assert store.count_messages() == 8 + assert store.count_messages("barsik") == 5 + assert store.count_messages("pushok") == 3 + assert store.count_messages("nobody") == 0 + + +# -- get_inbox ----------------------------------------------------------------- + + +class TestGetInbox: + def _seed(self, store): + # Three messages spaced apart in time so ordering is stable. + for i, kind in enumerate(["msg", "smoke", "msg"]): + store.log_message( + direction="outbound" if i % 2 else "inbound", + local_agent="barsik", + remote_fleet="pulse", + remote_agent="pulse" if i < 2 else "misha", + correlation_id=f"cid-{i}", + kind=kind, + body={"kind": kind, "n": i}, + ) + time.sleep(0.005) + + def test_newest_first(self, store): + self._seed(store) + msgs = store.get_inbox("barsik") + assert len(msgs) == 3 + # received_at descends + assert msgs[0].received_at >= msgs[1].received_at >= msgs[2].received_at + + def test_kind_filter(self, store): + self._seed(store) + smokes = store.get_inbox("barsik", kind="smoke") + assert len(smokes) == 1 + assert smokes[0].kind == "smoke" + + def test_direction_filter(self, store): + self._seed(store) + out = store.get_inbox("barsik", direction="outbound") + assert all(m.direction == "outbound" for m in out) + + def test_sender_filter(self, store): + self._seed(store) + from_misha = store.get_inbox("barsik", sender="misha@pulse") + assert len(from_misha) == 1 + assert from_misha[0].remote_agent == "misha" + + def test_pagination(self, store): + self._seed(store) + page1 = store.get_inbox("barsik", limit=2, offset=0) + page2 = store.get_inbox("barsik", limit=2, offset=2) + assert len(page1) == 2 + assert len(page2) == 1 + # No overlap. + assert page1[1].id != page2[0].id + + def test_other_agent_isolated(self, store): + self._seed(store) + store.log_message( + direction="inbound", + local_agent="pushok", + remote_fleet="pulse", + remote_agent="pulse", + kind="msg", + body={"kind": "msg"}, + ) + assert len(store.get_inbox("barsik")) == 3 + assert len(store.get_inbox("pushok")) == 1 + + +# -- get_message_by_correlation_id -------------------------------------------- + + +class TestGetByCorrelationId: + def test_missing_returns_none(self, store): + assert store.get_message_by_correlation_id("nope") is None + assert store.get_message_by_correlation_id("") is None + + def test_returns_most_recent_on_duplicate(self, store): + store.log_message( + direction="outbound", local_agent="barsik", + remote_fleet="p", remote_agent="p", + correlation_id="cid-dup", kind="msg", body={"kind": "msg", "v": 1}, + ) + time.sleep(0.005) + store.log_message( + direction="inbound", local_agent="barsik", + remote_fleet="p", remote_agent="p", + correlation_id="cid-dup", kind="ack", body={"kind": "ack", "v": 2}, + ) + latest = store.get_message_by_correlation_id("cid-dup") + assert latest is not None + assert latest.kind == "ack" + assert latest.body == {"kind": "ack", "v": 2} + + +# -- federation_peers ---------------------------------------------------------- + + +class TestFederationPeers: + def test_upsert_inserts_then_updates(self, store): + store.upsert_peer(fleet="pulse", agent="pulse", display_name="Pulse") + p1 = store.get_peer("pulse", "pulse") + assert p1 is not None + assert p1.display_name == "Pulse" + assert p1.seed_source == "config" + + time.sleep(0.005) + store.upsert_peer(fleet="pulse", agent="pulse", display_name="Pulse v2") + p2 = store.get_peer("pulse", "pulse") + assert p2 is not None + assert p2.display_name == "Pulse v2" + # first_seen preserved across update + assert p2.first_seen == p1.first_seen + assert p2.last_seen >= p1.last_seen + + def test_config_beats_observed_on_promotion(self, store): + # Observed first. + store.log_message( + direction="inbound", local_agent="barsik", + remote_fleet="pulse", remote_agent="pulse", + kind="msg", body={"kind": "msg"}, + ) + assert store.get_peer("pulse", "pulse").seed_source == "observed" + # Then config-promoted. + store.upsert_peer(fleet="pulse", agent="pulse", display_name="Pulse") + assert store.get_peer("pulse", "pulse").seed_source == "config" + + def test_observed_does_not_downgrade_config(self, store): + store.upsert_peer(fleet="pulse", agent="pulse", display_name="Pulse") + # Subsequent observed traffic shouldn't downgrade seed_source. + store.log_message( + direction="inbound", local_agent="barsik", + remote_fleet="pulse", remote_agent="pulse", + kind="msg", body={"kind": "msg"}, + ) + assert store.get_peer("pulse", "pulse").seed_source == "config" + + def test_list_peers_filters(self, store): + store.upsert_peer(fleet="pulse", agent="pulse", seed_source="config") + store.upsert_peer(fleet="pulse", agent="misha", seed_source="config") + store.upsert_peer(fleet="sigil", agent="agent3", seed_source="observed") + + all_peers = store.list_peers() + assert len(all_peers) == 3 + + pulse_only = store.list_peers(fleet="pulse") + assert len(pulse_only) == 2 + assert {p.agent for p in pulse_only} == {"pulse", "misha"} + + config_only = store.list_peers(seed_source="config") + assert len(config_only) == 2 + + observed_only = store.list_peers(seed_source="observed") + assert len(observed_only) == 1 + assert observed_only[0].fleet == "sigil" + + def test_remove_peer(self, store): + store.upsert_peer(fleet="pulse", agent="pulse") + assert store.remove_peer("pulse", "pulse") is True + assert store.remove_peer("pulse", "pulse") is False # already gone + assert store.get_peer("pulse", "pulse") is None + + def test_address_field_in_to_dict(self, store): + store.upsert_peer(fleet="pulse", agent="misha") + peer = store.get_peer("pulse", "misha") + assert peer.to_dict()["address"] == "misha@pulse" + + +# -- Defensive deserialization ------------------------------------------------- + + +class TestDefensive: + def test_malformed_body_json_does_not_crash(self, store): + # Bypass the public API to write a row with broken JSON. + store._db.execute( + """INSERT INTO mesh_messages + (correlation_id, direction, local_agent, remote_fleet, + remote_agent, kind, body, envelope_ts, received_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ("cid-x", "inbound", "barsik", "p", "p", "msg", + "{not valid json", 0, time.time()), + ) + store._db.commit() + msgs = store.get_inbox("barsik") + assert len(msgs) == 1 + assert "_raw" in msgs[0].body # fell back gracefully diff --git a/tests/test_pinky_self_tools.py b/tests/test_pinky_self_tools.py index 86d41c1d..4d485eb6 100644 --- a/tests/test_pinky_self_tools.py +++ b/tests/test_pinky_self_tools.py @@ -1987,7 +1987,7 @@ def test_error_returns_fallback(self, srv): "claim_task", "complete_task", "context_restart", "context_status", "create_task", "get_next_task", "get_owner_profile", "list_agents", "list_my_skills", "load_my_context", - "load_skill", "request_sleep", "save_my_context", + "load_skill", "mesh_remote_send", "request_sleep", "save_my_context", "search_history", "send_file_to_agent", "send_heartbeat", "send_to_agent", "set_thinking_effort", "who_am_i", } @@ -2040,13 +2040,13 @@ def test_kb_raw_source_id_encodes_path_segment(self, srv): class TestToolGates: - def test_core_only_has_23_tools(self): + def test_core_only_has_24_tools(self): """No gates → only core tools registered.""" srv = create_server(agent_name="test", tool_gates=[]) tools = {t.name for t in srv._tool_manager.list_tools()} assert tools == CORE_TOOLS - def test_all_gates_has_68_tools(self): + def test_all_gates_has_69_tools(self): """All gates → full tool set.""" all_gates = [ "extras", "kb", "research", "presentations", "triggers", @@ -2054,7 +2054,7 @@ def test_all_gates_has_68_tools(self): ] srv = create_server(agent_name="test", tool_gates=all_gates) tools = srv._tool_manager.list_tools() - assert len(tools) == 68 + assert len(tools) == 69 def test_extras_gate_adds_extras_tools(self): """Enabling 'extras' gate adds get_attribution, render_pdf, etc.""" diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 346f900e..2597b3fe 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -436,6 +436,12 @@ class TestHeartbeatResurrection: rate-limited to RESURRECTION_MAX_ATTEMPTS per RESURRECTION_WINDOW_SECONDS. """ + class _FakeStreamingSession: + is_connected = True + id = "ivan-main" + context_used_pct = 12.5 + stats = {"messages_sent": 2, "turns": 3} + @pytest.mark.asyncio async def test_dead_agent_triggers_resurrection_callback(self, registry): registry.register("ivan", model="opus", heartbeat_interval=60) @@ -594,3 +600,68 @@ async def cb(agent_name, session_id): scheduler = AgentScheduler(registry, heartbeat_callback=cb) # Should swallow and log, not propagate await scheduler._check_heartbeats(time.time()) + + @pytest.mark.asyncio + async def test_connected_streaming_session_clears_dead_heartbeat(self, registry): + registry.register("ivan", model="opus", heartbeat_interval=60) + registry.record_heartbeat("ivan", session_id="old", status="dead") + registry._db.execute( + "UPDATE agent_heartbeats SET timestamp = ? WHERE agent_name = ?", + (time.time() - 600, "ivan"), + ) + registry._db.commit() + called = [] + + async def cb(agent_name, session_id): + called.append((agent_name, session_id)) + + scheduler = AgentScheduler( + registry, + heartbeat_callback=cb, + streaming_sessions_fn=lambda: { + "ivan": {"main": self._FakeStreamingSession()}, + }, + ) + await scheduler._check_heartbeats(time.time()) + + latest = registry.get_latest_heartbeat("ivan") + assert called == [] + assert latest is not None + assert latest.status == "alive" + assert latest.session_id == "ivan-main" + assert latest.context_pct == 12.5 + assert latest.message_count == 5 + assert latest.metadata["source"] == "server_presence" + assert latest.metadata["reason"] == "connected_streaming_session" + + @pytest.mark.asyncio + async def test_fresh_last_seen_suppresses_dead_heartbeat_resurrection(self, registry): + registry.register("ivan", model="opus", heartbeat_interval=60) + registry.record_heartbeat("ivan", session_id="old", status="dead") + old_ts = time.time() - 600 + registry._db.execute( + "UPDATE agent_heartbeats SET timestamp = ? WHERE agent_name = ?", + (old_ts, "ivan"), + ) + registry._db.commit() + now = time.time() + registry.stamp_last_seen("ivan", ts=now - 10) + called = [] + + async def cb(agent_name, session_id): + called.append((agent_name, session_id)) + + scheduler = AgentScheduler( + registry, + heartbeat_callback=cb, + streaming_sessions_fn=lambda: {}, + ) + await scheduler._check_heartbeats(now) + + latest = registry.get_latest_heartbeat("ivan") + assert called == [] + assert latest is not None + assert latest.status == "alive" + assert latest.metadata["source"] == "server_presence" + assert latest.metadata["reason"] == "fresh_last_seen" + assert latest.metadata["last_seen_at"] == pytest.approx(now - 10)