fix: evaluate API actor hooks with process credentials

captain_hook/actor.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import os
+import shutil
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from spawnllm import LlmBackend
+
+CREDENTIAL_ENV = (
+    "OPENAI_API_KEY",
+    "CODEX_API_KEY",
+    "ANTHROPIC_API_KEY",
+    "ANTHROPIC_AUTH_TOKEN",
+    "CLAUDE_CODE_OAUTH_TOKEN",
+)
+KEY_SOURCES = {"codex": ("OPENAI_API_KEY", "CODEX_API_KEY"), "claude": ("ANTHROPIC_API_KEY",)}
+KEY_TARGETS = {"codex": "CODEX_API_KEY", "claude": "ANTHROPIC_API_KEY"}
+DEFAULT_MODELS = {"codex": "gpt-6.1-sol:xhigh", "claude": "claude-sonnet-5-5:xhigh"}
+TIERS = frozenset({"small", "medium", "large"})
+CLAUDE_ALIASES = frozenset({"haiku", "sonnet", "opus"})
+INFERENCE_REAP_SECONDS = 3.0
+
+ACTOR: ActorJudge | None = None
+
+
+class JudgeFailure(RuntimeError):
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class ActorJudge:
+    provider: str
+    keys: dict[str, str] = field(repr=False)
+
+    def route(self, explicit: LlmBackend | None, model: str) -> tuple[LlmBackend, str, dict[str, str] | None]:
+        from spawnllm import BackendUnavailable, ClaudeCliBackend, CodexCliBackend, OpenAiEndpointBackend
+
+        if isinstance(explicit, OpenAiEndpointBackend):
+            return explicit, model, None
+        if explicit is not None and type(explicit) not in (CodexCliBackend, ClaudeCliBackend):
+            raise BackendUnavailable(f"the {explicit.provider} judge backend is not supported for an API actor")
+        provider = explicit.provider if explicit is not None else self.judged_provider(model)
+        backend = explicit or (CodexCliBackend() if provider == "codex" else ClaudeCliBackend())
+        if provider not in self.keys:
+            needed = " or ".join(KEY_SOURCES[provider])
+            raise BackendUnavailable(f"the actor holds no {needed} for its {provider} judge")
+        if not shutil.which(backend.binary_path()):
+            raise BackendUnavailable(f"the actor's {provider} judge CLI {backend.binary} is not installed")
+        resolved = backend.resolve_model(model) if model else DEFAULT_MODELS[provider]
+        if provider == "claude" and ":" in resolved:
+            raise BackendUnavailable(f"the Claude judge cannot run {resolved!r}: spawnllm passes Claude no effort")
+        return backend, resolved, {KEY_TARGETS[provider]: self.keys[provider]}
+
+    def judged_provider(self, model: str | None) -> str:
+        from spawnllm import BackendUnavailable
+
+        if not model:
+            return self.provider
+        if model in TIERS or model.startswith("gpt-"):
+            return "codex"
+        if model in CLAUDE_ALIASES or model.startswith("claude-"):
+            return "claude"
+        raise BackendUnavailable(f"the judge model {model!r} names no provider an API actor can run")
+
+
+def failure_status(exc: Exception) -> str:
+    from pydantic import ValidationError
+    from spawnllm import BackendCallError
+
+    if isinstance(exc, TimeoutError):
+        return "timeout"
+    if isinstance(exc, BackendCallError):
+        return "provider-error"
+    if isinstance(exc, ValidationError):
+        return "invalid-output"
+    return "failed"
+
+
+def judged[T](call: Callable[[], T], backend: LlmBackend) -> T:
+    try:
+        return call()
+    except Exception as exc:
+        failure = JudgeFailure(f"the {backend.provider} judge failed: {type(exc).__name__} ({failure_status(exc)})")
+    raise failure
+
+
+def single_attempt(attempts: int | None) -> int:
+    if attempts not in (None, 1):
+        raise ValueError(f"an API actor's judge runs exactly one attempt, not {attempts}")
+    return 1
+
+
+def inference_timeout(timeout: int) -> int:
+    from captain_hook.util import reqenv
+
+    if (usable := int(reqenv.seconds_left() - INFERENCE_REAP_SECONDS)) < 1:
+        raise TimeoutError("the API actor's event budget leaves no room for the judge and its cleanup")
+    return min(timeout, usable)
+
+
+def capture(provider: str) -> ActorJudge:
+    global ACTOR
+    found = {name: value for name in CREDENTIAL_ENV if (value := os.environ.pop(name, None))}
+    keys = {
+        name: value
+        for name, sources in KEY_SOURCES.items()
+        if (value := next((found[source] for source in sources if source in found), None))
+    }
+    if provider not in keys:
+        raise SystemExit(f"capt-hook: the {provider} actor's environment holds no {' or '.join(KEY_SOURCES[provider])}")
+    ACTOR = ActorJudge(provider, keys)
+    return ACTOR

captain_hook/context.py

@@ -97,17 +97,23 @@ def run_spec(
     timeout: int,
     attempts: int | None,
     tools: tuple[str, ...] | None,
+    env: dict[str, str] | None = None,
 ) -> str | BaseModel:
-    from spawnllm import ClaudeConfig, RunSpec, run_sync
+    from spawnllm import ClaudeConfig, CodexConfig, RunSpec, run_sync
 
+    configs: dict[str, ClaudeConfig | CodexConfig] = {} if tools is None else {"claude": ClaudeConfig(tools=tools)}
+    if env is not None and backend.provider == "codex":
+        configs["codex"] = CodexConfig(bypass_approvals_and_sandbox=True)
     spec = RunSpec(
         prompt=prompt,
         model=backend.resolve_model(model),
         response_model=response_model,
         agent=agent,
         cwd=cwd,
+        env=env,
+        api_auth=env is not None,
         timeout=timeout,
-        provider_configs={} if tools is None else {"claude": ClaudeConfig(tools=tools)},
+        provider_configs=configs,
         **({} if attempts is None else {"max_attempts": attempts}),
     )
     resp = run_sync(spec, backend=backend)
@@ -497,8 +503,13 @@ def call_llm(
         """
         from spawnllm import BackendCallError, call_sync, extract_sync
 
+        from captain_hook import actor
+
         reqenv.checkpoint()
-        serving = backend or ready_backend(specialty, model)
+        if (judge := actor.ACTOR) is not None:
+            serving, judged, actor_env = judge.route(backend, model)
+        else:
+            serving, judged, actor_env = backend or ready_backend(specialty, model), model, None
         reqenv.checkpoint()
         if reqenv.deadline_within(0):
             raise TimeoutError("the caller deadline passed before the model call")
@@ -516,6 +527,23 @@ def call_llm(
         if evidence:
             self.release_preparation(prompt)
         cwd = resolve_project_dir()
+        if judge is not None:
+            once, limit = actor.single_attempt(attempts), actor.inference_timeout(timeout)
+            return actor.judged(
+                lambda: run_spec(
+                    prompt,
+                    serving,
+                    judged,
+                    response_model,
+                    agent=agent,
+                    cwd=cwd,
+                    timeout=limit,
+                    attempts=once,
+                    tools=tools,
+                    env=actor_env,
+                ),
+                serving,
+            )
         timeout = reqenv.clamp_timeout(timeout)
         try:
             if attempts is not None or tools is not None:

captain_hook/dispatch.py

@@ -1020,11 +1020,15 @@ def run_background_group(
     events: Sequence[BaseHookEvent],
     session_dir: Path | None,
 ) -> None:
+    from captain_hook import actor
     from captain_hook.transcripts import release_transcript
 
     try:
         for index in group:
-            with reqenv.deadline_in(ASYNC_HOOK_TIMEOUT_SECONDS):
+            budget = ASYNC_HOOK_TIMEOUT_SECONDS
+            if actor.ACTOR is not None:
+                budget = min(budget, reqenv.seconds_left())
+            with reqenv.deadline_in(budget):
                 try:
                     execute_hook(entries[index], events[index], session_dir)
                 except EvidenceIncomplete as exc:

captain_hook/worker/__main__.py

@@ -128,8 +128,54 @@ def evaluate() -> None:
         json.dump(response.message(), reply)
 
 
+def evaluate_actor(provider: str) -> None:
+    """Answer one event for a remote API-key actor, then finish its background hooks before exiting.
+
+    The actor's API keys move from this process's environment into the judge it calls, so no
+    hook command or git child inherits them, and the reply is the same ``EventResponse`` a host
+    worker returns.
+    """
+    from captain_hook import actor
+
+    actor.capture(provider)
+    from captain_hook.worker.protocol import OP_EVENT, PROTOCOL, decode_event
+
+    request = decode_event({"protocol": PROTOCOL, "op": OP_EVENT, "id": 1, "request": json.load(sys.stdin)})
+    if request.deadline_unix_ms <= 0:
+        raise SystemExit("capt-hook: an actor's event request carries no deadline")
+    from captain_hook.log import setup_logging
+
+    setup_logging(json.loads(request.payload_raw).get("session_id"))
+    reply = os.fdopen(os.dup(sys.stdout.fileno()), "w")
+    os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
+    from captain_hook.daemon.context import install_context_io
+    from captain_hook.snapshots.client import client_scope
+    from captain_hook.worker.runtime import ProductRuntime
+    from captain_hook.worker.service import BACKGROUND_SNAPSHOT_CLIENT
+
+    bound_transcript_parse_pool()
+    skip_bundled_cli_version_probe()
+    install_context_io()
+    runtime = ProductRuntime(install_writer=False, nlp_warmer=lambda: None)
+    try:
+        with client_scope() as client:
+            token = BACKGROUND_SNAPSHOT_CLIENT.set(client)
+            try:
+                response, background = runtime.dispatch(request)
+                with reply:
+                    json.dump(response.message(), reply)
+                if background is not None:
+                    background()
+            finally:
+                BACKGROUND_SNAPSHOT_CLIENT.reset(token)
+    finally:
+        runtime.close()
+
+
 if __name__ == "__main__":
     if sys.argv[1:] == ["evaluate"]:
         evaluate()
+    elif sys.argv[1:2] == ["evaluate-actor"] and len(sys.argv) == 3:
+        evaluate_actor(sys.argv[2])
     else:
         main()

internal/hookd/actor.go

@@ -0,0 +1,91 @@
+package hookd
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/yasyf/captain-hook/internal/wireproto"
+)
+
+const actorJudgeEnv = "CAPT_HOOK_ACTOR_JUDGE"
+
+var (
+	actorKeys = map[string][]string{
+		"codex":  {"OPENAI_API_KEY", "CODEX_API_KEY"},
+		"claude": {"ANTHROPIC_API_KEY"},
+	}
+	actorNativeTimeouts = map[string]time.Duration{"MessageDisplay": 10 * time.Second}
+	actorReapGrace      = 2 * time.Second
+	actorPython         = installedPython
+)
+
+func actorReady(judge string) error {
+	names, ok := actorKeys[judge]
+	if !ok {
+		return fmt.Errorf("capt-hookd: %s=%q names no API judge; set codex or claude", actorJudgeEnv, judge)
+	}
+	if !slices.ContainsFunc(names, func(name string) bool { return os.Getenv(name) != "" }) {
+		return fmt.Errorf("capt-hookd: %s=%s, but the actor's environment holds no %s", actorJudgeEnv, judge, strings.Join(names, " or "))
+	}
+	return nil
+}
+
+func actorBudget(event string, timeout time.Duration) time.Duration {
+	if native, ok := actorNativeTimeouts[event]; ok {
+		return min(timeout, native-2*actorReapGrace)
+	}
+	return timeout
+}
+
+var evaluateActor = func(request wireproto.EventRequest, judge string, timeout time.Duration) (wireproto.EventResponse, error) {
+	python, err := actorPython()
+	if err != nil {
+		return wireproto.EventResponse{}, err
+	}
+	budget := actorBudget(request.Event, timeout)
+	deadline := time.Now().Add(budget)
+	request.DeadlineUnixMS = deadline.UnixMilli()
+	payload, err := wireproto.MarshalEventRequest(request)
+	if err != nil {
+		return wireproto.EventResponse{}, err
+	}
+	ctx, cancel := context.WithDeadline(context.Background(), deadline.Add(actorReapGrace))
+	defer cancel()
+	cmd := exec.CommandContext(ctx, python, "-P", "-m", "captain_hook.worker", "evaluate-actor", judge)
+	cmd.Dir, cmd.Stdin = workerDir(request.Root), bytes.NewReader(payload)
+	cmd.WaitDelay = actorReapGrace
+	reply, err := cmd.StdoutPipe()
+	if err != nil {
+		return wireproto.EventResponse{}, err
+	}
+	if err := cmd.Start(); err != nil {
+		return wireproto.EventResponse{}, fmt.Errorf("captain: evaluate for the actor: %w", err)
+	}
+	result, readErr := io.ReadAll(reply)
+	answered := ctx.Err() == nil
+	waitErr := cmd.Wait()
+	switch {
+	case !answered:
+		return wireproto.EventResponse{}, fmt.Errorf("captain: the actor's evaluator did not answer within the %s event budget", budget)
+	case readErr != nil:
+		return wireproto.EventResponse{}, fmt.Errorf("captain: read the actor's evaluation: %w", readErr)
+	}
+	var response wireproto.EventResponse
+	if err := decodeStrict(result, &response); err != nil {
+		return wireproto.EventResponse{}, fmt.Errorf("captain: decode the actor's event response: %w (evaluator: %v)", err, waitErr)
+	}
+	if err := response.Validate(); err != nil {
+		return wireproto.EventResponse{}, err
+	}
+	if waitErr != nil {
+		response.Stderr += fmt.Sprintf("capt-hookd: the actor's background hooks did not finish within the %s event budget: %v\n", budget, waitErr)
+	}
+	return response, nil
+}