From 1b7ce34a5c5df31d8ff06d4123e91001d4709d2c Mon Sep 17 00:00:00 2001 From: Yasyf Mohamedali Date: Sun, 4 Oct 2026 23:50:18 -0700 Subject: [PATCH] fix: evaluate API actor hooks with process credentials --- captain_hook/actor.py | 116 +++++++ captain_hook/context.py | 34 +- captain_hook/dispatch.py | 6 +- captain_hook/worker/__main__.py | 46 +++ internal/hookd/actor.go | 91 ++++++ internal/hookd/actor_test.go | 382 ++++++++++++++++++++++ internal/hookd/command.go | 26 ++ pyproject.toml | 2 +- tests/test_actor_judge.py | 550 ++++++++++++++++++++++++++++++++ uv.lock | 10 +- 10 files changed, 1253 insertions(+), 10 deletions(-) create mode 100644 captain_hook/actor.py create mode 100644 internal/hookd/actor.go create mode 100644 internal/hookd/actor_test.go create mode 100644 tests/test_actor_judge.py diff --git a/captain_hook/actor.py b/captain_hook/actor.py new file mode 100644 index 00000000..9ed645e9 --- /dev/null +++ b/captain_hook/actor.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import os +import shutil +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable + + from spawnllm import LlmBackend + +CREDENTIAL_ENV = ( + "OPENAI_API_KEY", + "CODEX_API_KEY", + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "CLAUDE_CODE_OAUTH_TOKEN", +) +KEY_SOURCES = {"codex": ("OPENAI_API_KEY", "CODEX_API_KEY"), "claude": ("ANTHROPIC_API_KEY",)} +KEY_TARGETS = {"codex": "CODEX_API_KEY", "claude": "ANTHROPIC_API_KEY"} +DEFAULT_MODELS = {"codex": "gpt-6.1-sol:xhigh", "claude": "claude-sonnet-5-5:xhigh"} +TIERS = frozenset({"small", "medium", "large"}) +CLAUDE_ALIASES = frozenset({"haiku", "sonnet", "opus"}) +INFERENCE_REAP_SECONDS = 3.0 + +ACTOR: ActorJudge | None = None + + +class JudgeFailure(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class ActorJudge: + provider: str + keys: dict[str, str] = field(repr=False) + + def route(self, explicit: LlmBackend | None, model: str) -> tuple[LlmBackend, str, dict[str, str] | None]: + from spawnllm import BackendUnavailable, ClaudeCliBackend, CodexCliBackend, OpenAiEndpointBackend + + if isinstance(explicit, OpenAiEndpointBackend): + return explicit, model, None + if explicit is not None and type(explicit) not in (CodexCliBackend, ClaudeCliBackend): + raise BackendUnavailable(f"the {explicit.provider} judge backend is not supported for an API actor") + provider = explicit.provider if explicit is not None else self.judged_provider(model) + backend = explicit or (CodexCliBackend() if provider == "codex" else ClaudeCliBackend()) + if provider not in self.keys: + needed = " or ".join(KEY_SOURCES[provider]) + raise BackendUnavailable(f"the actor holds no {needed} for its {provider} judge") + if not shutil.which(backend.binary_path()): + raise BackendUnavailable(f"the actor's {provider} judge CLI {backend.binary} is not installed") + resolved = backend.resolve_model(model) if model else DEFAULT_MODELS[provider] + if provider == "claude" and ":" in resolved: + raise BackendUnavailable(f"the Claude judge cannot run {resolved!r}: spawnllm passes Claude no effort") + return backend, resolved, {KEY_TARGETS[provider]: self.keys[provider]} + + def judged_provider(self, model: str | None) -> str: + from spawnllm import BackendUnavailable + + if not model: + return self.provider + if model in TIERS or model.startswith("gpt-"): + return "codex" + if model in CLAUDE_ALIASES or model.startswith("claude-"): + return "claude" + raise BackendUnavailable(f"the judge model {model!r} names no provider an API actor can run") + + +def failure_status(exc: Exception) -> str: + from pydantic import ValidationError + from spawnllm import BackendCallError + + if isinstance(exc, TimeoutError): + return "timeout" + if isinstance(exc, BackendCallError): + return "provider-error" + if isinstance(exc, ValidationError): + return "invalid-output" + return "failed" + + +def judged[T](call: Callable[[], T], backend: LlmBackend) -> T: + try: + return call() + except Exception as exc: + failure = JudgeFailure(f"the {backend.provider} judge failed: {type(exc).__name__} ({failure_status(exc)})") + raise failure + + +def single_attempt(attempts: int | None) -> int: + if attempts not in (None, 1): + raise ValueError(f"an API actor's judge runs exactly one attempt, not {attempts}") + return 1 + + +def inference_timeout(timeout: int) -> int: + from captain_hook.util import reqenv + + if (usable := int(reqenv.seconds_left() - INFERENCE_REAP_SECONDS)) < 1: + raise TimeoutError("the API actor's event budget leaves no room for the judge and its cleanup") + return min(timeout, usable) + + +def capture(provider: str) -> ActorJudge: + global ACTOR + found = {name: value for name in CREDENTIAL_ENV if (value := os.environ.pop(name, None))} + keys = { + name: value + for name, sources in KEY_SOURCES.items() + if (value := next((found[source] for source in sources if source in found), None)) + } + if provider not in keys: + raise SystemExit(f"capt-hook: the {provider} actor's environment holds no {' or '.join(KEY_SOURCES[provider])}") + ACTOR = ActorJudge(provider, keys) + return ACTOR diff --git a/captain_hook/context.py b/captain_hook/context.py index 49ded715..fc6a5192 100644 --- a/captain_hook/context.py +++ b/captain_hook/context.py @@ -97,17 +97,23 @@ def run_spec( timeout: int, attempts: int | None, tools: tuple[str, ...] | None, + env: dict[str, str] | None = None, ) -> str | BaseModel: - from spawnllm import ClaudeConfig, RunSpec, run_sync + from spawnllm import ClaudeConfig, CodexConfig, RunSpec, run_sync + configs: dict[str, ClaudeConfig | CodexConfig] = {} if tools is None else {"claude": ClaudeConfig(tools=tools)} + if env is not None and backend.provider == "codex": + configs["codex"] = CodexConfig(bypass_approvals_and_sandbox=True) spec = RunSpec( prompt=prompt, model=backend.resolve_model(model), response_model=response_model, agent=agent, cwd=cwd, + env=env, + api_auth=env is not None, timeout=timeout, - provider_configs={} if tools is None else {"claude": ClaudeConfig(tools=tools)}, + provider_configs=configs, **({} if attempts is None else {"max_attempts": attempts}), ) resp = run_sync(spec, backend=backend) @@ -497,8 +503,13 @@ def call_llm( """ from spawnllm import BackendCallError, call_sync, extract_sync + from captain_hook import actor + reqenv.checkpoint() - serving = backend or ready_backend(specialty, model) + if (judge := actor.ACTOR) is not None: + serving, judged, actor_env = judge.route(backend, model) + else: + serving, judged, actor_env = backend or ready_backend(specialty, model), model, None reqenv.checkpoint() if reqenv.deadline_within(0): raise TimeoutError("the caller deadline passed before the model call") @@ -516,6 +527,23 @@ def call_llm( if evidence: self.release_preparation(prompt) cwd = resolve_project_dir() + if judge is not None: + once, limit = actor.single_attempt(attempts), actor.inference_timeout(timeout) + return actor.judged( + lambda: run_spec( + prompt, + serving, + judged, + response_model, + agent=agent, + cwd=cwd, + timeout=limit, + attempts=once, + tools=tools, + env=actor_env, + ), + serving, + ) timeout = reqenv.clamp_timeout(timeout) try: if attempts is not None or tools is not None: diff --git a/captain_hook/dispatch.py b/captain_hook/dispatch.py index d1bd9594..b59f52e3 100644 --- a/captain_hook/dispatch.py +++ b/captain_hook/dispatch.py @@ -1020,11 +1020,15 @@ def run_background_group( events: Sequence[BaseHookEvent], session_dir: Path | None, ) -> None: + from captain_hook import actor from captain_hook.transcripts import release_transcript try: for index in group: - with reqenv.deadline_in(ASYNC_HOOK_TIMEOUT_SECONDS): + budget = ASYNC_HOOK_TIMEOUT_SECONDS + if actor.ACTOR is not None: + budget = min(budget, reqenv.seconds_left()) + with reqenv.deadline_in(budget): try: execute_hook(entries[index], events[index], session_dir) except EvidenceIncomplete as exc: diff --git a/captain_hook/worker/__main__.py b/captain_hook/worker/__main__.py index f4a7ec01..9c5482f7 100644 --- a/captain_hook/worker/__main__.py +++ b/captain_hook/worker/__main__.py @@ -128,8 +128,54 @@ def evaluate() -> None: json.dump(response.message(), reply) +def evaluate_actor(provider: str) -> None: + """Answer one event for a remote API-key actor, then finish its background hooks before exiting. + + The actor's API keys move from this process's environment into the judge it calls, so no + hook command or git child inherits them, and the reply is the same ``EventResponse`` a host + worker returns. + """ + from captain_hook import actor + + actor.capture(provider) + from captain_hook.worker.protocol import OP_EVENT, PROTOCOL, decode_event + + request = decode_event({"protocol": PROTOCOL, "op": OP_EVENT, "id": 1, "request": json.load(sys.stdin)}) + if request.deadline_unix_ms <= 0: + raise SystemExit("capt-hook: an actor's event request carries no deadline") + from captain_hook.log import setup_logging + + setup_logging(json.loads(request.payload_raw).get("session_id")) + reply = os.fdopen(os.dup(sys.stdout.fileno()), "w") + os.dup2(sys.stderr.fileno(), sys.stdout.fileno()) + from captain_hook.daemon.context import install_context_io + from captain_hook.snapshots.client import client_scope + from captain_hook.worker.runtime import ProductRuntime + from captain_hook.worker.service import BACKGROUND_SNAPSHOT_CLIENT + + bound_transcript_parse_pool() + skip_bundled_cli_version_probe() + install_context_io() + runtime = ProductRuntime(install_writer=False, nlp_warmer=lambda: None) + try: + with client_scope() as client: + token = BACKGROUND_SNAPSHOT_CLIENT.set(client) + try: + response, background = runtime.dispatch(request) + with reply: + json.dump(response.message(), reply) + if background is not None: + background() + finally: + BACKGROUND_SNAPSHOT_CLIENT.reset(token) + finally: + runtime.close() + + if __name__ == "__main__": if sys.argv[1:] == ["evaluate"]: evaluate() + elif sys.argv[1:2] == ["evaluate-actor"] and len(sys.argv) == 3: + evaluate_actor(sys.argv[2]) else: main() diff --git a/internal/hookd/actor.go b/internal/hookd/actor.go new file mode 100644 index 00000000..6ff0d539 --- /dev/null +++ b/internal/hookd/actor.go @@ -0,0 +1,91 @@ +package hookd + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "slices" + "strings" + "time" + + "github.com/yasyf/captain-hook/internal/wireproto" +) + +const actorJudgeEnv = "CAPT_HOOK_ACTOR_JUDGE" + +var ( + actorKeys = map[string][]string{ + "codex": {"OPENAI_API_KEY", "CODEX_API_KEY"}, + "claude": {"ANTHROPIC_API_KEY"}, + } + actorNativeTimeouts = map[string]time.Duration{"MessageDisplay": 10 * time.Second} + actorReapGrace = 2 * time.Second + actorPython = installedPython +) + +func actorReady(judge string) error { + names, ok := actorKeys[judge] + if !ok { + return fmt.Errorf("capt-hookd: %s=%q names no API judge; set codex or claude", actorJudgeEnv, judge) + } + if !slices.ContainsFunc(names, func(name string) bool { return os.Getenv(name) != "" }) { + return fmt.Errorf("capt-hookd: %s=%s, but the actor's environment holds no %s", actorJudgeEnv, judge, strings.Join(names, " or ")) + } + return nil +} + +func actorBudget(event string, timeout time.Duration) time.Duration { + if native, ok := actorNativeTimeouts[event]; ok { + return min(timeout, native-2*actorReapGrace) + } + return timeout +} + +var evaluateActor = func(request wireproto.EventRequest, judge string, timeout time.Duration) (wireproto.EventResponse, error) { + python, err := actorPython() + if err != nil { + return wireproto.EventResponse{}, err + } + budget := actorBudget(request.Event, timeout) + deadline := time.Now().Add(budget) + request.DeadlineUnixMS = deadline.UnixMilli() + payload, err := wireproto.MarshalEventRequest(request) + if err != nil { + return wireproto.EventResponse{}, err + } + ctx, cancel := context.WithDeadline(context.Background(), deadline.Add(actorReapGrace)) + defer cancel() + cmd := exec.CommandContext(ctx, python, "-P", "-m", "captain_hook.worker", "evaluate-actor", judge) + cmd.Dir, cmd.Stdin = workerDir(request.Root), bytes.NewReader(payload) + cmd.WaitDelay = actorReapGrace + reply, err := cmd.StdoutPipe() + if err != nil { + return wireproto.EventResponse{}, err + } + if err := cmd.Start(); err != nil { + return wireproto.EventResponse{}, fmt.Errorf("captain: evaluate for the actor: %w", err) + } + result, readErr := io.ReadAll(reply) + answered := ctx.Err() == nil + waitErr := cmd.Wait() + switch { + case !answered: + return wireproto.EventResponse{}, fmt.Errorf("captain: the actor's evaluator did not answer within the %s event budget", budget) + case readErr != nil: + return wireproto.EventResponse{}, fmt.Errorf("captain: read the actor's evaluation: %w", readErr) + } + var response wireproto.EventResponse + if err := decodeStrict(result, &response); err != nil { + return wireproto.EventResponse{}, fmt.Errorf("captain: decode the actor's event response: %w (evaluator: %v)", err, waitErr) + } + if err := response.Validate(); err != nil { + return wireproto.EventResponse{}, err + } + if waitErr != nil { + response.Stderr += fmt.Sprintf("capt-hookd: the actor's background hooks did not finish within the %s event budget: %v\n", budget, waitErr) + } + return response, nil +} diff --git a/internal/hookd/actor_test.go b/internal/hookd/actor_test.go new file mode 100644 index 00000000..6c913c2c --- /dev/null +++ b/internal/hookd/actor_test.go @@ -0,0 +1,382 @@ +package hookd + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/yasyf/captain-hook/internal/wireproto" +) + +const actorSentinel = "sk-synthetic-actor-key-7f3a" + +func clearActorCredentials(t *testing.T) { + t.Helper() + for _, variable := range []string{"OPENAI_API_KEY", "CODEX_API_KEY", "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "CLAUDE_CODE_OAUTH_TOKEN"} { + t.Setenv(variable, "") + } +} + +type actorEvaluation struct { + response wireproto.EventResponse + err error + judges []string + requests []wireproto.EventRequest +} + +func scriptActor(t *testing.T, evaluation *actorEvaluation) { + t.Helper() + clearActorCredentials(t) + previous := evaluateActor + evaluateActor = func(request wireproto.EventRequest, judge string, _ time.Duration) (wireproto.EventResponse, error) { + evaluation.judges = append(evaluation.judges, judge) + evaluation.requests = append(evaluation.requests, request) + return evaluation.response, evaluation.err + } + t.Cleanup(func() { evaluateActor = previous }) +} + +func TestRunEvaluatesAnActorEventWithoutTheHost(t *testing.T) { + t.Setenv(actorJudgeEnv, "codex") + client := scriptedClient{err: errors.New("the host must not be asked")} + scriptClient(t, &client, nil) + local := localEvaluation{err: errors.New("the transport fallback must not run")} + scriptLocal(t, &local) + deny := `{"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"deny","permissionDecisionReason":"BLOCKED"}}` + "\n" + evaluation := actorEvaluation{response: wireproto.EventResponse{Schema: wireproto.Schema, Status: "ok", Stdout: deny, Guard: wireproto.GuardCompleted}} + scriptActor(t, &evaluation) + t.Setenv("OPENAI_API_KEY", actorSentinel) + code, stdout, stderr := runEvent(t, "PreToolUse", destructivePayload) + if code != 0 || stdout != deny || stderr != "" { + t.Fatalf("exit=%d stdout=%q stderr=%q, want the actor's blocking verdict", code, stdout, stderr) + } + if len(client.requests) != 0 || len(local.requests) != 0 { + t.Fatalf("host requests=%d local requests=%d, want neither", len(client.requests), len(local.requests)) + } + if len(evaluation.judges) != 1 || evaluation.judges[0] != "codex" || !evaluation.requests[0].Mandatory { + t.Fatalf("actor evaluations = %v %+v", evaluation.judges, evaluation.requests) + } + encoded, err := wireproto.MarshalEventRequest(evaluation.requests[0]) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(encoded), actorSentinel) || strings.Contains(string(encoded), "OPENAI_API_KEY") { + t.Fatalf("the actor's key reached the event request: %s", encoded) + } + if evaluation.requests[0].Env[actorJudgeEnv] != "codex" { + t.Fatalf("request env = %v, want the nonsecret judge selection only", evaluation.requests[0].Env) + } +} + +func TestRunRefusesAnActorWithoutItsJudgeVisibly(t *testing.T) { + for name, tc := range map[string]struct { + judge, key, reason string + }{ + "no such judge": {"gemini", "OPENAI_API_KEY", `CAPT_HOOK_ACTOR_JUDGE="gemini" names no API judge`}, + "no codex key": {"codex", "", "holds no OPENAI_API_KEY or CODEX_API_KEY"}, + "another provider key": {"claude", "OPENAI_API_KEY", "holds no ANTHROPIC_API_KEY"}, + } { + for _, event := range []string{"PreToolUse", "PermissionRequest"} { + for _, mandatory := range []bool{true, false} { + t.Run(name+"/"+event+"/"+strconv.FormatBool(mandatory), func(t *testing.T) { + t.Setenv(actorJudgeEnv, tc.judge) + client := scriptedClient{err: errors.New("the host must not be asked")} + scriptClient(t, &client, nil) + local := localEvaluation{err: errors.New("the transport fallback must not run")} + scriptLocal(t, &local) + evaluation := actorEvaluation{} + scriptActor(t, &evaluation) + if tc.key != "" { + t.Setenv(tc.key, actorSentinel) + } + payload := benignPayload + if mandatory { + payload = destructivePayload + } + code, stdout, stderr := runEvent(t, event, payload) + switch { + case !strings.Contains(stderr, tc.reason): + t.Fatalf("stderr = %q, want %q", stderr, tc.reason) + case mandatory && (code != 0 || stdout != wireproto.SkipEnvelope(event, "dependency-unavailable")+"\n" || + !strings.HasSuffix(stderr, "capt-hookd: the session guard did not complete (dependency-unavailable); skipped\n")): + t.Fatalf("exit=%d stdout=%q stderr=%q, want the mandatory skip note", code, stdout, stderr) + case !mandatory && (code != 1 || stdout != ""): + t.Fatalf("exit=%d stdout=%q, want a visible failure", code, stdout) + case len(evaluation.judges) != 0 || len(client.requests) != 0 || len(local.requests) != 0: + t.Fatalf("a refused actor evaluated: actor=%d host=%d local=%d", len(evaluation.judges), len(client.requests), len(local.requests)) + case strings.Contains(stdout+stderr, actorSentinel): + t.Fatalf("the refusal printed the key: %q %q", stdout, stderr) + } + }) + } + } + } +} + +func TestRunGradesAFailedActorEvaluationLikeTheHost(t *testing.T) { + t.Setenv(actorJudgeEnv, "codex") + scriptClient(t, &scriptedClient{err: errors.New("the host must not be asked")}, nil) + timedOut := "captain: the actor's evaluator did not answer within the 6s event budget" + evaluatorError := actorEvaluation{err: errors.New(timedOut)} + for name, tc := range map[string]struct { + evaluation actorEvaluation + payload string + code int + kind, stderr string + }{ + "mandatory evaluator error": {evaluatorError, destructivePayload, 0, "worker-error", timedOut + "\n"}, + "mandatory guard incomplete": {actorEvaluation{response: wireproto.EventResponse{Schema: wireproto.Schema, Status: "ok"}}, + stopPayload, 0, "no-verdict", ""}, + "mandatory worker error": {actorEvaluation{response: wireproto.EventResponse{ + Schema: wireproto.Schema, Status: "error", Exit: 1, Stderr: "Traceback " + actorSentinel + "\n", + }}, destructivePayload, 0, "worker-error", ""}, + "advisory evaluator error": {evaluatorError, benignPayload, 1, "", timedOut + "\n"}, + "advisory worker exit passes": {actorEvaluation{response: wireproto.EventResponse{ + Schema: wireproto.Schema, Status: "error", Exit: 2, Stderr: "boom\n", + }}, benignPayload, 2, "", "boom\n"}, + } { + for _, event := range []string{"PreToolUse", "PermissionRequest"} { + t.Run(name+"/"+event, func(t *testing.T) { + local := localEvaluation{err: errors.New("the transport fallback must not run")} + scriptLocal(t, &local) + evaluation := tc.evaluation + scriptActor(t, &evaluation) + t.Setenv("OPENAI_API_KEY", actorSentinel) + code, stdout, stderr := runEvent(t, event, tc.payload) + wantStdout, wantStderr := "", tc.stderr + if tc.kind != "" { + wantStdout = wireproto.SkipEnvelope(event, tc.kind) + "\n" + wantStderr += "capt-hookd: the session guard did not complete (" + tc.kind + "); skipped\n" + } + if code != tc.code || stdout != wantStdout || stderr != wantStderr || len(local.requests) != 0 { + t.Fatalf("exit=%d stdout=%q stderr=%q local=%d, want %d %q %q and no fallback", + code, stdout, stderr, len(local.requests), tc.code, wantStdout, wantStderr) + } + if strings.Contains(stdout+stderr, actorSentinel) || strings.Contains(stdout, "permissionDecision") || strings.Contains(stdout, "behavior") { + t.Fatalf("stdout=%q stderr=%q carries a decision or the key", stdout, stderr) + } + }) + } + } +} + +func TestRunForwardsAnActorsSettledVerdictUnchanged(t *testing.T) { + t.Setenv(actorJudgeEnv, "codex") + scriptClient(t, &scriptedClient{err: errors.New("the host must not be asked")}, nil) + preToolUseDeny := `{"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"deny",` + + `"permissionDecisionReason":"BLOCKED: pkill signals every process matching a name"},` + + `"systemMessage":"capt-hook: the mandatory hook judged_audit did not complete (JudgeFailure)"}` + "\n" + permissionDeny := `{"hookSpecificOutput":{"decision":{"behavior":"deny","message":"BLOCKED: pkill"},` + + `"hookEventName":"PermissionRequest"}}` + "\n" + for name, tc := range map[string]struct { + event string + response wireproto.EventResponse + code int + }{ + "PreToolUse deny without a guard completion": {"PreToolUse", + wireproto.EventResponse{Schema: wireproto.Schema, Status: "ok", Stdout: preToolUseDeny}, 0}, + "PermissionRequest deny without a guard completion": {"PermissionRequest", + wireproto.EventResponse{Schema: wireproto.Schema, Status: "ok", Stdout: permissionDeny}, 0}, + "blocking exit without a guard completion": {"PreToolUse", + wireproto.EventResponse{Schema: wireproto.Schema, Status: "ok", Stderr: "BLOCKED: pkill\n", Exit: 2}, 2}, + "completed guard with a sibling's stderr": {"PreToolUse", + wireproto.EventResponse{Schema: wireproto.Schema, Status: "ok", Stderr: "warned\n", Guard: wireproto.GuardCompleted}, 0}, + } { + t.Run(name, func(t *testing.T) { + local := localEvaluation{err: errors.New("the transport fallback must not run")} + scriptLocal(t, &local) + evaluation := actorEvaluation{response: tc.response} + scriptActor(t, &evaluation) + t.Setenv("OPENAI_API_KEY", actorSentinel) + code, stdout, stderr := runEvent(t, tc.event, destructivePayload) + if code != tc.code || stdout != tc.response.Stdout || stderr != tc.response.Stderr { + t.Fatalf("exit=%d stdout=%q stderr=%q, want the actor's verdict unchanged: %d %q %q", + code, stdout, stderr, tc.code, tc.response.Stdout, tc.response.Stderr) + } + if len(evaluation.requests) != 1 || !evaluation.requests[0].Mandatory || len(local.requests) != 0 { + t.Fatalf("actor requests=%+v local=%d, want one mandatory actor request", evaluation.requests, len(local.requests)) + } + }) + } +} + +func TestRunKeepsTheResidentRouteWithoutAnActorJudge(t *testing.T) { + client := scriptedClient{response: wireproto.EventResponse{Schema: wireproto.Schema, Status: "ok", Stdout: "resident\n"}} + scriptClient(t, &client, nil) + evaluation := actorEvaluation{err: errors.New("the actor route must not run")} + scriptActor(t, &evaluation) + t.Setenv("OPENAI_API_KEY", actorSentinel) + code, stdout, _ := runEvent(t, "PostToolUse", benignPayload) + if code != 0 || stdout != "resident\n" || len(client.requests) != 1 || len(evaluation.judges) != 0 { + t.Fatalf("exit=%d stdout=%q host=%d actor=%d", code, stdout, len(client.requests), len(evaluation.judges)) + } + if _, leaked := client.requests[0].Env["OPENAI_API_KEY"]; leaked { + t.Fatal("the resident request carries the actor's key") + } +} + +func TestActorBudgetFitsEveryNativeHookTimeout(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "captain_hook", "hooks", "hooks.json")) + if err != nil { + t.Fatal(err) + } + var manifest struct { + Hooks map[string][]struct { + Hooks []struct { + Timeout int `json:"timeout"` + } `json:"hooks"` + } `json:"hooks"` + } + if err := json.Unmarshal(raw, &manifest); err != nil { + t.Fatal(err) + } + for event, groups := range manifest.Hooks { + for _, group := range groups { + for _, handler := range group.Hooks { + if handler.Timeout == 0 { + continue + } + native := time.Duration(handler.Timeout) * time.Second + if actorNativeTimeouts[event] != native { + t.Errorf("%s declares a %s native timeout, the actor budget knows %s", event, native, actorNativeTimeouts[event]) + } + if budget := actorBudget(event, defaultRequestTimeout); budget+actorReapGrace >= native { + t.Errorf("%s budget %s plus reap grace %s does not finish inside its %s native timeout", event, budget, actorReapGrace, native) + } + } + } + } + for event, want := range map[string]time.Duration{"MessageDisplay": 6 * time.Second, "PreToolUse": defaultRequestTimeout, "SessionStart": defaultRequestTimeout} { + if got := actorBudget(event, defaultRequestTimeout); got != want { + t.Errorf("actorBudget(%s) = %s, want %s", event, got, want) + } + } + if got := actorBudget("MessageDisplay", 3*time.Second); got != 3*time.Second { + t.Errorf("a shorter caller budget was raised to %s", got) + } +} + +type fakeEvaluator struct { + dir string +} + +func fakeActorPython(t *testing.T, script string) fakeEvaluator { + t.Helper() + dir := t.TempDir() + body := "#!/bin/sh\nset -eu\necho $$ > " + filepath.Join(dir, "pid") + + "\nprintf 'OPENAI_API_KEY=%s\\n' \"${OPENAI_API_KEY-}\" > " + filepath.Join(dir, "env") + + "\ncat > " + filepath.Join(dir, "stdin") + "\n" + script + path := filepath.Join(dir, "python") + if err := os.WriteFile(path, []byte(body), 0o700); err != nil { + t.Fatal(err) + } + previous, grace := actorPython, actorReapGrace + actorPython = func() (string, error) { return path, nil } + actorReapGrace = 200 * time.Millisecond + t.Cleanup(func() { actorPython, actorReapGrace = previous, grace }) + return fakeEvaluator{dir: dir} +} + +func (f fakeEvaluator) read(t *testing.T, name string) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join(f.dir, name)) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func (f fakeEvaluator) gone(t *testing.T) { + t.Helper() + pid, err := strconv.Atoi(strings.TrimSpace(f.read(t, "pid"))) + if err != nil { + t.Fatal(err) + } + if err := syscall.Kill(pid, 0); !errors.Is(err, syscall.ESRCH) { + t.Fatalf("the owned evaluator %d is still alive: %v", pid, err) + } +} + +func actorRequest(t *testing.T, event string) wireproto.EventRequest { + t.Helper() + clearActorCredentials(t) + t.Setenv("CLAUDE_PROJECT_DIR", t.TempDir()) + t.Setenv("OPENAI_API_KEY", actorSentinel) + request, err := eventRequest(event, strings.NewReader(benignPayload)) + if err != nil { + t.Fatal(err) + } + return request +} + +const fakeReply = `{"schema":1,"status":"ok","stdout":"verdict\n","stderr":"","exit":0,"elapsed_ms":1,"warmup":false}` + +func TestEvaluateActorAwaitsBackgroundHooksAfterTheReply(t *testing.T) { + fake := fakeActorPython(t, "printf '%s' '"+fakeReply+"'\nexec 1>&-\nsleep 0.3\ntouch "+"\"$(dirname \"$0\")/background\"\n") + request := actorRequest(t, "PostToolUse") + started := time.Now() + response, err := evaluateActor(request, "codex", 5*time.Second) + if err != nil || response.Stdout != "verdict\n" || response.Stderr != "" { + t.Fatalf("evaluateActor = %+v, %v", response, err) + } + if _, err := os.Stat(filepath.Join(fake.dir, "background")); err != nil { + t.Fatalf("evaluateActor returned before its background hooks finished: %v", err) + } + if !strings.Contains(fake.read(t, "env"), "OPENAI_API_KEY="+actorSentinel) { + t.Fatal("the owned evaluator did not inherit the actor's key") + } + var sent wireproto.EventRequest + if err := json.Unmarshal([]byte(fake.read(t, "stdin")), &sent); err != nil { + t.Fatal(err) + } + if strings.Contains(fake.read(t, "stdin"), actorSentinel) { + t.Fatal("the actor's key reached the evaluator's request") + } + if deadline := time.UnixMilli(sent.DeadlineUnixMS); deadline.Before(started.Add(5*time.Second-time.Second)) || deadline.After(started.Add(5*time.Second+time.Second)) { + t.Fatalf("request deadline %s is not the event budget from %s", deadline, started) + } + fake.gone(t) +} + +func TestEvaluateActorReapsAnEvaluatorThatNeverAnswers(t *testing.T) { + fake := fakeActorPython(t, "exec sleep 30\n") + request := actorRequest(t, "PostToolUse") + started := time.Now() + _, err := evaluateActor(request, "codex", time.Second) + if err == nil || !strings.Contains(err.Error(), "did not answer within the 1s event budget") { + t.Fatalf("evaluateActor = %v, want the typed timeout", err) + } + if elapsed := time.Since(started); elapsed > time.Second+2*actorReapGrace+time.Second { + t.Fatalf("a stalled evaluator held the hook for %s", elapsed) + } + fake.gone(t) +} + +func TestEvaluateActorBoundsBackgroundHooksToTheEventBudget(t *testing.T) { + fake := fakeActorPython(t, "printf '%s' '"+fakeReply+"'\nexec 1>&-\nexec sleep 30\n") + request := actorRequest(t, "MessageDisplay") + started := time.Now() + response, err := evaluateActor(request, "codex", time.Second) + if err != nil || response.Stdout != "verdict\n" || !strings.Contains(response.Stderr, "background hooks did not finish within the 1s event budget") { + t.Fatalf("evaluateActor = %+v, %v; want the verdict and a visible background timeout", response, err) + } + if elapsed := time.Since(started); elapsed > time.Second+2*actorReapGrace+time.Second { + t.Fatalf("background hooks held the hook for %s", elapsed) + } + fake.gone(t) +} + +func TestEvaluateActorRejectsAMalformedReply(t *testing.T) { + fake := fakeActorPython(t, "printf '%s' '{\"schema\":1,\"status\":\"ok\",\"surprise\":true}'\n") + _, err := evaluateActor(actorRequest(t, "PostToolUse"), "codex", 5*time.Second) + if err == nil || !strings.Contains(err.Error(), "decode the actor's event response") { + t.Fatalf("evaluateActor = %v, want a strict decode failure", err) + } + fake.gone(t) +} diff --git a/internal/hookd/command.go b/internal/hookd/command.go index 0de8d4e2..a92add11 100644 --- a/internal/hookd/command.go +++ b/internal/hookd/command.go @@ -133,6 +133,9 @@ func runCommand(args []string, stdin io.Reader, stdout, stderr io.Writer) int { return 1 } request.Mandatory = wireproto.Mandatory(request.Event, []byte(request.PayloadRaw)) + if judge, ok := os.LookupEnv(actorJudgeEnv); ok { + return runActor(request, judge, timeout, stdout, stderr) + } client, err := openEventClient() var response wireproto.EventResponse if err == nil { @@ -152,6 +155,29 @@ func runCommand(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fmt.Fprintln(stderr, err) return 1 } + return answer(request, response, stdout, stderr) +} + +func runActor(request wireproto.EventRequest, judge string, timeout time.Duration, stdout, stderr io.Writer) int { + if err := actorReady(judge); err != nil { + fmt.Fprintln(stderr, err) + if request.Mandatory { + return skipMandatory(stdout, stderr, request.Event, "dependency-unavailable") + } + return 1 + } + response, err := evaluateActor(request, judge, timeout) + if err != nil { + fmt.Fprintln(stderr, err) + if request.Mandatory { + return skipMandatory(stdout, stderr, request.Event, "worker-error") + } + return 1 + } + return answer(request, response, stdout, stderr) +} + +func answer(request wireproto.EventRequest, response wireproto.EventResponse, stdout, stderr io.Writer) int { if request.Mandatory { if kind := incompleteKind(response); kind != "" { return skipMandatory(stdout, stderr, request.Event, kind) diff --git a/pyproject.toml b/pyproject.toml index a61ad82f..69b07eb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "filelock>=3", "pathspec>=0.12", "loguru>=0.7.3", - "spawnllm[sdk]>=0.15.0,<0.16", + "spawnllm[sdk]>=0.16.0,<0.17", "mcp>=2.0,<3", ] diff --git a/tests/test_actor_judge.py b/tests/test_actor_judge.py new file mode 100644 index 00000000..85cf5041 --- /dev/null +++ b/tests/test_actor_judge.py @@ -0,0 +1,550 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +import threading +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import TYPE_CHECKING, Any +from unittest.mock import MagicMock, patch + +import pytest +from pydantic import BaseModel + +from captain_hook import actor +from captain_hook.app import on +from captain_hook.context import HookContext +from captain_hook.session import SessionStore +from captain_hook.types import Event + +if TYPE_CHECKING: + from collections.abc import Iterator + +SENTINEL = "sk-synthetic-judge-key-41c2" +OTHER = "sk-synthetic-judge-key-9be0" +PROVIDER_TEXT = "synthetic-provider-text-c0d9" +LS = '{"cwd":"/w","tool_name":"Bash","tool_input":{"command":"ls"}}' +PKILL = '{"cwd":"/w","tool_name":"Bash","tool_input":{"command":"pkill -x sleep"}}' +JUDGE_SKIP = ( + "capt-hook: the mandatory hook {name} did not complete " + "(JudgeFailure: the codex judge failed: BackendCallError (provider-error)) " + "and did not check this call. Run `capt-hook logs` to see why." +) + +FAKE_CLI = """#!{python} +import json, os, signal, sys, time +args = sys.argv[1:] +mode = os.environ.get("FAKE_CLI_MODE", "ok") +with open({log!r}, "a") as log: + log.write(json.dumps({{ + "argv": args, + "codex_key": os.environ.get("CODEX_API_KEY"), + "anthropic_key": os.environ.get("ANTHROPIC_API_KEY"), + "openai_key": "OPENAI_API_KEY" in os.environ, + }}) + "\\n") +if args[:1] in (["login"], ["auth"]): + sys.exit(1) +if mode == "fail": + print("{text} out " + str(os.environ.get("CODEX_API_KEY"))) + sys.stderr.write('ERROR: {{"status":503,"text":"{text} err"}}\\n') + sys.exit(1) +if mode == "stall": + open({log!r} + ".pid", "w").write(str(os.getpid())) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + time.sleep(60) +sys.stdin.read() +result = '{{"name": "n", "value": 1}}' if "--output-schema" in args else "ok" +open(args[args.index("-o") + 1], "w").write(result) +""" + +CONCURRENT_ACTOR = """ +import os, sys +from unittest.mock import MagicMock +from captain_hook import actor +from captain_hook.context import HookContext +from captain_hook.session import SessionStore +from captain_hook.util import reqenv +actor.capture("codex") +scope = reqenv.RequestOverrides(env={}, cwd=os.getcwd(), client_ppid=1, session_id="s") +with reqenv.use_request(scope), reqenv.deadline_in(60): + ctx = HookContext(session=SessionStore(None), transcript=MagicMock(), settings=None) + print(ctx.call_llm("judge", model="small")) +""" + + +class Verdict(BaseModel): + name: str + value: int + + +@pytest.fixture(autouse=True) +def clean_actor(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + for name in actor.CREDENTIAL_ENV: + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(actor, "ACTOR", None) + monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(tmp_path)) + + +def fake_clis(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *names: str) -> Path: + bin_dir, log = tmp_path / "bin", tmp_path / "cli.jsonl" + bin_dir.mkdir(exist_ok=True) + for name in names: + path = bin_dir / name + path.write_text(FAKE_CLI.format(python=sys.executable, log=str(log), text=PROVIDER_TEXT)) + path.chmod(0o755) + monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ['PATH']}") + return log + + +def invocations(log: Path) -> list[dict]: + return [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else [] + + +def test_capture_moves_every_credential_out_of_the_evaluator_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENAI_API_KEY", SENTINEL) + monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN", OTHER) + judge = actor.capture("codex") + assert judge is actor.ACTOR + assert judge.keys == {"codex": SENTINEL} + assert not {name for name in actor.CREDENTIAL_ENV if name in os.environ} + assert SENTINEL not in repr(judge) + + +def test_capture_refuses_an_actor_without_its_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ANTHROPIC_API_KEY", SENTINEL) + with pytest.raises(SystemExit, match="OPENAI_API_KEY or CODEX_API_KEY"): + actor.capture("codex") + + +@pytest.mark.parametrize( + ("provider", "keys", "model", "backend", "resolved"), + [ + ("codex", {"OPENAI_API_KEY": SENTINEL}, None, "codex", "gpt-6.1-sol:xhigh"), + ("codex", {"OPENAI_API_KEY": SENTINEL}, "small", "codex", "gpt-6-luna:low"), + ("codex", {"OPENAI_API_KEY": SENTINEL}, "gpt-5.5", "codex", "gpt-5.5"), + ("claude", {"ANTHROPIC_API_KEY": SENTINEL, "OPENAI_API_KEY": OTHER}, "small", "codex", "gpt-6-luna:low"), + ("claude", {"ANTHROPIC_API_KEY": SENTINEL}, "claude-opus-5-5", "claude", "claude-opus-5-5"), + ("codex", {"OPENAI_API_KEY": SENTINEL, "ANTHROPIC_API_KEY": OTHER}, "sonnet", "claude", "sonnet"), + ], +) +def test_route_keeps_explicit_models_and_defaults_only_the_unset_one( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + provider: str, + keys: dict[str, str], + model: str | None, + backend: str, + resolved: str, +) -> None: + fake_clis(tmp_path, monkeypatch, "codex", "claude") + for name, value in keys.items(): + monkeypatch.setenv(name, value) + judge = actor.capture(provider) + serving, judged, env = judge.route(None, model) + target = actor.KEY_TARGETS[backend] + assert (serving.provider, judged, env) == (backend, resolved, {target: judge.keys[backend]}) + + +def test_route_refuses_what_an_api_actor_cannot_run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from spawnllm import BackendUnavailable, ClaudeSdkBackend + + fake_clis(tmp_path, monkeypatch, "claude") + monkeypatch.setenv("ANTHROPIC_API_KEY", SENTINEL) + judge = actor.capture("claude") + with pytest.raises(BackendUnavailable, match="OPENAI_API_KEY or CODEX_API_KEY"): + judge.route(None, "small") + with pytest.raises(BackendUnavailable, match="names no provider"): + judge.route(None, "gemini-3-pro") + with pytest.raises(BackendUnavailable, match="not supported for an API actor"): + judge.route(ClaudeSdkBackend(), "small") + monkeypatch.setenv("PATH", str(tmp_path / "empty")) + with pytest.raises(BackendUnavailable, match="CLI claude is not installed"): + judge.route(None, None) + + +def test_route_leaves_an_explicit_endpoint_judge_alone(monkeypatch: pytest.MonkeyPatch) -> None: + from spawnllm import OpenAiEndpointBackend + + monkeypatch.setenv("OPENAI_API_KEY", SENTINEL) + endpoint = OpenAiEndpointBackend("https://example.invalid/v1", "m", api_key="synthetic") + assert actor.capture("codex").route(endpoint, "small") == (endpoint, "small", None) + + +def test_resident_readiness_probes_a_login_the_api_actor_never_needs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from spawnllm import BackendNotAuthenticated, CodexCliBackend + + log = fake_clis(tmp_path, monkeypatch, "codex") + monkeypatch.setenv("OPENAI_API_KEY", SENTINEL) + assert isinstance(CodexCliBackend().check_status(), BackendNotAuthenticated) + assert [entry["argv"][:2] for entry in invocations(log)] == [["login", "status"]] + + +@pytest.mark.parametrize( + "call", + [ + {}, + {"response_model": Verdict}, + {"tools": ()}, + ], + ids=["call", "extract", "run-spec"], +) +def test_every_call_path_runs_the_selected_api_judge_child( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, call: dict +) -> None: + from captain_hook.util import reqenv + + log = fake_clis(tmp_path, monkeypatch, "codex") + monkeypatch.setenv("OPENAI_API_KEY", SENTINEL) + actor.capture("codex") + ctx = HookContext(session=SessionStore(None), transcript=MagicMock(), settings=None) + scope = reqenv.RequestOverrides(env={}, cwd=str(tmp_path), client_ppid=1, session_id="s") + with ( + patch("spawnllm.select_backend", side_effect=AssertionError("API actors never auto-select")), + reqenv.use_request(scope), + reqenv.deadline_in(60), + ): + result = ctx.call_llm("judge this", model="small", **call) + assert result == (Verdict(name="n", value=1) if "response_model" in call else "ok") + [entry] = invocations(log) + argv = entry["argv"] + assert argv[:2] == ["exec", "--ephemeral"] + assert "--dangerously-bypass-approvals-and-sandbox" in argv + assert "--sandbox" not in argv + assert argv[argv.index("--model") + 1] == "gpt-6-luna" + assert "model_reasoning_effort=low" in argv + assert {"--ignore-user-config", "features.hooks=false", "features.mcp_servers=false"} <= set(argv) + assert (entry["codex_key"], entry["openai_key"], entry["anthropic_key"]) == (SENTINEL, False, None) + assert SENTINEL not in os.environ.values() + + +def test_concurrent_actors_keep_their_own_keys(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + logs = [] + children = [] + for name, key in (("one", SENTINEL), ("two", OTHER)): + root = tmp_path / name + root.mkdir() + logs.append(fake_clis(root, monkeypatch, "codex")) + env = {k: v for k, v in os.environ.items() if k not in actor.CREDENTIAL_ENV} + path = f"{root / 'bin'}{os.pathsep}{env['PATH']}" + env |= {"OPENAI_API_KEY": key, "CLAUDE_PROJECT_DIR": str(root), "PATH": path} + children.append( + subprocess.Popen([sys.executable, "-c", CONCURRENT_ACTOR], env=env, stdout=subprocess.PIPE, text=True) + ) + outputs = [child.communicate(timeout=60)[0] for child in children] + assert [child.returncode for child in children] == [0, 0] + assert outputs == ["ok\n", "ok\n"] + assert [[entry["codex_key"] for entry in invocations(log)] for log in logs] == [[SENTINEL], [OTHER]] + + +def test_a_judge_never_starts_without_room_for_its_cleanup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from captain_hook.util import reqenv + + log = fake_clis(tmp_path, monkeypatch, "codex") + monkeypatch.setenv("OPENAI_API_KEY", SENTINEL) + actor.capture("codex") + ctx = HookContext(session=SessionStore(None), transcript=MagicMock(), settings=None) + scope = reqenv.RequestOverrides(env={}, cwd=str(tmp_path), client_ppid=1, session_id="s") + with reqenv.use_request(scope), reqenv.deadline_in(actor.INFERENCE_REAP_SECONDS + 0.5): + with pytest.raises(TimeoutError, match="no room for the judge and its cleanup"): + ctx.call_llm("judge this", model="small") + assert invocations(log) == [] + with reqenv.use_request(scope), reqenv.deadline_in(actor.INFERENCE_REAP_SECONDS + 5.5): + assert actor.inference_timeout(180) == 5 + + +def judge_scope(tmp_path: Path): + from captain_hook.util import reqenv + + return reqenv.RequestOverrides(env={}, cwd=str(tmp_path), client_ppid=1, session_id="s") + + +def test_a_failing_judge_surfaces_only_typed_metadata(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + import traceback + + from loguru import logger + + from captain_hook import faults + from captain_hook.log import setup_logging + from captain_hook.util import reqenv + + log = fake_clis(tmp_path, monkeypatch, "codex") + monkeypatch.setenv("FAKE_CLI_MODE", "fail") + monkeypatch.setenv("CAPTAIN_HOOK_LOG_DIR", str(tmp_path / "logs")) + monkeypatch.setenv("CAPTAIN_HOOK_STATE_DIR", str(tmp_path / "state")) + monkeypatch.setenv("OPENAI_API_KEY", SENTINEL) + actor.capture("codex") + setup_logging("actor-failure") + ctx = HookContext(session=SessionStore(None), transcript=MagicMock(), settings=None) + with reqenv.use_request(judge_scope(tmp_path)), reqenv.deadline_in(60): + with pytest.raises(actor.JudgeFailure) as raised: + ctx.call_llm("judge this", model="small") + failure = raised.value + assert str(failure) == "the codex judge failed: BackendCallError (provider-error)" + assert failure.__cause__ is None and failure.__context__ is None + logger.bind(hook="general.synthetic:llm_judge").opt(exception=failure).error("hook handler failed") + faults.record("hook general.synthetic:llm_judge", failure, str(tmp_path)) + artifacts = [path.read_text() for path in tmp_path.rglob("*.json")] + drained = faults.drain(str(tmp_path)) + logger.complete() + surfaces = ["".join(traceback.format_exception(failure)), repr(failure), *drained, *artifacts] + surfaces += [path.read_text() for path in (tmp_path / "logs").rglob("*.log")] + assert any("provider-error" in surface for surface in surfaces) + assert not [surface for surface in surfaces if PROVIDER_TEXT in surface or SENTINEL in surface] + assert len(invocations(log)) == 1 + + +@contextmanager +def unavailable_endpoint() -> Iterator[tuple[str, list[str]]]: + requests: list[str] = [] + + class Unavailable(BaseHTTPRequestHandler): + def do_POST(self) -> None: + requests.append(self.path) + body = json.dumps({"error": {"message": PROVIDER_TEXT}}).encode() + self.send_response(503) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Unavailable) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield f"http://127.0.0.1:{server.server_port}/v1", requests + finally: + server.shutdown() + server.server_close() + + +def test_a_failing_endpoint_judge_runs_once_and_surfaces_only_typed_metadata( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import traceback + + from spawnllm import OpenAiEndpointBackend + + from captain_hook import faults + from captain_hook.util import reqenv + + monkeypatch.setenv("CAPTAIN_HOOK_STATE_DIR", str(tmp_path / "state")) + monkeypatch.setenv("OPENAI_API_KEY", SENTINEL) + actor.capture("codex") + ctx = HookContext(session=SessionStore(None), transcript=MagicMock(), settings=None) + with unavailable_endpoint() as (url, requests): + endpoint = OpenAiEndpointBackend(url, "m", api_key=OTHER) + with reqenv.use_request(judge_scope(tmp_path)), reqenv.deadline_in(60): + with pytest.raises(actor.JudgeFailure) as raised: + ctx.call_llm("judge this", backend=endpoint) + failure = raised.value + assert str(failure) == f"the {endpoint.provider} judge failed: BackendCallError (provider-error)" + assert failure.__cause__ is None and failure.__context__ is None + faults.record("plain_english rewrite", failure, str(tmp_path)) + surfaces = ["".join(traceback.format_exception(failure)), repr(failure)] + surfaces += [path.read_text() for path in tmp_path.rglob("*.json")] + assert not [surface for surface in surfaces if PROVIDER_TEXT in surface or OTHER in surface or SENTINEL in surface] + assert requests == ["/v1/chat/completions"] + + +def test_an_api_judge_runs_exactly_one_attempt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from captain_hook.util import reqenv + + log = fake_clis(tmp_path, monkeypatch, "codex") + monkeypatch.setenv("OPENAI_API_KEY", SENTINEL) + actor.capture("codex") + ctx = HookContext(session=SessionStore(None), transcript=MagicMock(), settings=None) + with reqenv.use_request(judge_scope(tmp_path)), reqenv.deadline_in(60): + with pytest.raises(ValueError, match="exactly one attempt, not 3"): + ctx.call_llm("judge this", model="small", attempts=3) + assert invocations(log) == [] + assert ctx.call_llm("judge this", model="small", attempts=1) == "ok" + assert len(invocations(log)) == 1 + + +def test_claude_judge_effort_is_refused_before_any_cli(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from spawnllm import BackendUnavailable, ClaudeCliBackend + + log = fake_clis(tmp_path, monkeypatch, "claude") + monkeypatch.setenv("ANTHROPIC_API_KEY", SENTINEL) + judge = actor.capture("claude") + for model in ("claude-opus-5-5:high", None): + with pytest.raises(BackendUnavailable, match="passes Claude no effort"): + judge.route(None, model) + serving, resolved, _ = judge.route(ClaudeCliBackend(), "claude-opus-5-5") + assert (serving.provider, resolved) == ("claude", "claude-opus-5-5") + assert invocations(log) == [] + + +def test_a_stalled_judge_is_reaped_before_the_deadline(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + import time + + from captain_hook.util import reqenv + + log = fake_clis(tmp_path, monkeypatch, "codex") + monkeypatch.setenv("FAKE_CLI_MODE", "stall") + monkeypatch.setenv("OPENAI_API_KEY", SENTINEL) + actor.capture("codex") + ctx = HookContext(session=SessionStore(None), transcript=MagicMock(), settings=None) + budget = actor.INFERENCE_REAP_SECONDS + 2.0 + started = time.monotonic() + with reqenv.use_request(judge_scope(tmp_path)), reqenv.deadline_in(budget): + with pytest.raises(actor.JudgeFailure, match="timeout"): + ctx.call_llm("judge this", model="small") + assert time.monotonic() - started < budget + pid = int(Path(f"{log}.pid").read_text()) + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + + +def failing_codex_actor(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + log = fake_clis(tmp_path, monkeypatch, "codex") + monkeypatch.setenv("FAKE_CLI_MODE", "fail") + monkeypatch.setenv("OPENAI_API_KEY", SENTINEL) + actor.capture("codex") + return log + + +def actor_response(root: Path, *, event: str = "PreToolUse", payload: str, mandatory: bool) -> Any: + import time + from dataclasses import replace + + from captain_hook import app + from captain_hook.worker.runtime import ProductRuntime + from tests.test_pack_sessions import HOOK_SHELL + from tests.test_worker_runtime import FakeRegistry, request + + runtime = ProductRuntime( + registry_factory=lambda _: FakeRegistry(app.current_state()), + transcript_loader=lambda path: None, + install_writer=False, + nlp_warmer=lambda: None, + ) + deadline = int((time.time() + 60) * 1000) + response, _ = runtime.dispatch( + replace( + request(event=event, payload_raw=payload, mandatory=mandatory), + root=str(root), + cwd=str(root), + env={"CLAUDE_PROJECT_DIR": str(root)}, + client_ppid=HOOK_SHELL, + deadline_unix_ms=deadline, + ) + ) + return response + + +def replied(response: Any) -> dict[str, Any]: + return json.loads(response.stdout.splitlines()[-1]) + + +def leaks(response: Any) -> list[str]: + return [text for text in (response.stdout, response.stderr) if PROVIDER_TEXT in text or SENTINEL in text] + + +@pytest.fixture +def general_guard(isolate_modules: None, monkeypatch: pytest.MonkeyPatch) -> None: + from captain_hook.loader import discover_pack + from captain_hook.util import proc + from tests.test_pack_sessions import MAC, PACKS_DIR + + monkeypatch.setattr(proc, "process_table", lambda **kw: MAC) + monkeypatch.setattr("captain_hook.heartbeat.record_heartbeat", lambda *args: None) + discover_pack("general", PACKS_DIR / "general" / "hooks") + + +@pytest.mark.parametrize("event", ["PreToolUse", "PermissionRequest"]) +def test_a_failed_actor_judge_skips_its_mandatory_hook_with_only_typed_metadata( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, event: str +) -> None: + log = failing_codex_actor(tmp_path, monkeypatch) + + @on(Event[event], mandatory=True) + def judged_policy(evt: Any) -> None: + evt.ctx.call_llm("judge this call", model="small", evidence=False) + + response = actor_response(tmp_path, event=event, payload=LS, mandatory=False) + envelope = replied(response) + assert response.exit == 0 + assert envelope["systemMessage"] == JUDGE_SKIP.format(name="judged_policy") + assert "permissionDecision" not in response.stdout + assert "behavior" not in response.stdout + assert leaks(response) == [] + assert len(invocations(log)) == 1 + + +def test_a_settled_block_stands_beside_a_failed_actor_judge(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + log = failing_codex_actor(tmp_path, monkeypatch) + + @on(Event.PreToolUse, mandatory=True) + def slack_policy(evt: Any) -> Any: + return evt.block("permission denied") + + @on(Event.PreToolUse, mandatory=True) + def judged_audit(evt: Any) -> None: + evt.ctx.call_llm("judge this call", model="small", evidence=False) + + response = actor_response(tmp_path, payload=LS, mandatory=False) + envelope = replied(response) + assert response.exit == 0 + assert envelope["hookSpecificOutput"]["permissionDecision"] == "deny" + assert "permission denied" in envelope["hookSpecificOutput"]["permissionDecisionReason"] + assert JUDGE_SKIP.format(name="judged_audit") in envelope["systemMessage"] + assert leaks(response) == [] + assert len(invocations(log)) == 1 + + +def test_an_actor_completes_the_guard_only_when_every_mandatory_hook_finished( + general_guard: None, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + failing_codex_actor(tmp_path, monkeypatch) + completed = actor_response(tmp_path, payload=PKILL, mandatory=True) + assert (completed.exit, completed.guard) == (0, "completed") + assert replied(completed)["hookSpecificOutput"]["permissionDecision"] == "deny" + + @on(Event.PreToolUse, mandatory=True) + def judged_audit(evt: Any) -> None: + evt.ctx.call_llm("judge this call", model="small", evidence=False) + + response = actor_response(tmp_path, payload=PKILL, mandatory=True) + envelope = replied(response) + assert (response.exit, response.guard) == (0, "") + assert envelope["hookSpecificOutput"]["permissionDecision"] == "deny" + assert JUDGE_SKIP.format(name="judged_audit") in envelope["systemMessage"] + assert leaks(response) == [] + + +def test_an_actor_confirm_failure_notes_only_its_type_once_per_session( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, logcap: Any +) -> None: + from captain_hook import Confirm + from captain_hook.dispatch import execute_hook + from captain_hook.events import PreToolUseEvent + from captain_hook.types import Action, HookResult, HookSpec, RegisteredHook + from captain_hook.util import reqenv + from tests.helpers import build_ctx + + monkeypatch.setenv("OPENAI_API_KEY", SENTINEL) + actor.capture("codex") + spec = HookSpec(events=Event.PreToolUse, message="m", block=True, confirm=Confirm(rule="r")) + guard = RegisteredHook(spec=spec, name="queued_push") + ctx = build_ctx(session_dir=tmp_path) + scope = reqenv.RequestOverrides(env={"CEREBRAS_API_KEY": OTHER}, cwd=str(tmp_path), client_ppid=1, session_id="s") + with unavailable_endpoint() as (url, requests): + monkeypatch.setattr("captain_hook.confirm.CONFIRM_ENDPOINT", url) + with reqenv.use_request(scope), reqenv.deadline_in(60): + results = [ + execute_hook(guard, PreToolUseEvent(_raw={"tool_name": "Bash", "tool_input": {"command": c}}, ctx=ctx)) + for c in ("git push origin feat", "git push origin main") + ] + message = "queued_push: allowed, the confirm step failed (JudgeFailure)" + assert results == [HookResult(action=Action.warn, message=message, approve=False), None] + assert requests == ["/v1/chat/completions"] * 2 + assert "confirm step failed" in logcap.text + assert not [secret for secret in (PROVIDER_TEXT, OTHER, SENTINEL) if secret in logcap.text] diff --git a/uv.lock b/uv.lock index 8736ffed..b1fa2d51 100644 --- a/uv.lock +++ b/uv.lock @@ -298,7 +298,7 @@ requires-dist = [ { name = "rich", specifier = ">=13" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.8" }, { name = "spacy", specifier = ">=3.7" }, - { name = "spawnllm", extras = ["sdk"], specifier = ">=0.15.0,<0.16" }, + { name = "spawnllm", extras = ["sdk"], specifier = ">=0.16.0,<0.17" }, { name = "wn", specifier = ">=1.1.0" }, ] provides-extras = ["dev"] @@ -2516,7 +2516,7 @@ wheels = [ [[package]] name = "spawnllm" -version = "0.15.0" +version = "0.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -2525,10 +2525,10 @@ dependencies = [ { name = "pydantic" }, { name = "wasmtime" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/79/63/fe1e8232aa6967d608399a7a0c98630cbf646152dfa49105776eb5fe36c3/spawnllm-0.15.0.tar.gz", hash = "sha256:f1fa3d899120b87b56d5bd372fb556dfeb3dd9e6c7ff259dda46e62960dd8723", size = 279861, upload-time = "2026-10-02T02:53:07.7Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/f8/5979aa5ec740aa149bfd4194e00b6242da5abb7252bb583bee21edad98e2/spawnllm-0.16.0.tar.gz", hash = "sha256:5e152ea1e6e27bc3b4bd925e6e526bf26e6910f3070c4632e09dec081eb0f90b", size = 280421, upload-time = "2026-10-05T06:21:53.398Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/9b/94b44c7257f6bac4ed71253564d00adba46419d0a62b3b75aeddb2ff2eb0/spawnllm-0.15.0-py3-none-any.whl", hash = "sha256:a6ca913af659c36f2a9433c3736e2ff20b8ace71af182f682cf8e3bc78eb71f9", size = 290812, upload-time = "2026-10-02T02:53:04.092Z" }, - { url = "https://files.pythonhosted.org/packages/7a/d2/4c9836b3f37064949d836e3c81b12caf5b85f8cd2da653478010bd2a7d4e/spawnllm-0.15.0-py3-none-macosx_26_0_arm64.whl", hash = "sha256:759703658766f5e988bbb1e4cec35e28bda2cdf7c0274b19823b5be344f80176", size = 377110, upload-time = "2026-10-02T02:53:05.947Z" }, + { url = "https://files.pythonhosted.org/packages/38/60/3e122b6c6546ed3f6a4252e452f8d6411484e3aca700faae65fec0257f1d/spawnllm-0.16.0-py3-none-any.whl", hash = "sha256:f031c1e44a86968275096b4ae8216e93bcb7654071b84e02db98728a8960cdc1", size = 291340, upload-time = "2026-10-05T06:21:50.133Z" }, + { url = "https://files.pythonhosted.org/packages/bf/63/1e5a6db35f43cb178f8e30f98d112fcc7978a6b591806d551ee0d1feb81a/spawnllm-0.16.0-py3-none-macosx_26_0_arm64.whl", hash = "sha256:d783fba22ac0613460c36ef550fa5758ee9cbbf4253b98a2b45d9f522f4d7ec8", size = 377698, upload-time = "2026-10-05T06:21:51.852Z" }, ] [package.optional-dependencies]