Visual Truth protocol + runtime deadline propagation

biometrics-cli/internal/contracts/types.go

@@ -1,6 +1,7 @@
 package contracts
 
 import (
+	"context"
 	"strings"
 	"time"
 )
@@ -165,6 +166,7 @@ type AgentEnvelope struct {
 	BlueprintModules   []string          `json:"blueprint_modules,omitempty"`
 	Bootstrap          bool              `json:"bootstrap,omitempty"`
 	Input              map[string]string `json:"input,omitempty"`
+	Ctx                context.Context   `json:"-"`
 	ResponseCh         chan AgentResult  `json:"-"`
 	DispatchedAt       time.Time         `json:"dispatched_at"`
 }

biometrics-cli/internal/executor/opencode/adapter.go

@@ -138,8 +138,14 @@ func (a *Adapter) executeOnce(
 	binaryPath, runID, agentName, prompt, projectID string,
 	metadata routingMetadata,
 ) (string, error) {
-	timeoutCtx, cancel := context.WithTimeout(ctx, 180*time.Second)
-	defer cancel()
+	execCtx := ctx
+	var cancel context.CancelFunc
+	if _, ok := ctx.Deadline(); !ok {
+		execCtx, cancel = context.WithTimeout(ctx, 10*time.Minute)
+	}
+	if cancel != nil {
+		defer cancel()
+	}
 
 	args := []string{"run"}
 	if strings.TrimSpace(agentName) != "" {
@@ -153,7 +159,7 @@ func (a *Adapter) executeOnce(
 	}
 	args = append(args, prompt)
 
-	cmd := exec.CommandContext(timeoutCtx, binaryPath, args...)
+	cmd := exec.CommandContext(execCtx, binaryPath, args...)
 	cmd.Env = append(os.Environ(),
 		"PROJECT_ID="+projectID,
 		"BIOMETRICS_RUN_ID="+runID,

biometrics-cli/internal/runtime/actor/system.go

@@ -2,6 +2,7 @@ package actor
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -75,7 +76,11 @@ func (s *System) Start(ctx context.Context) {
 				case <-loopCtx.Done():
 					return
 				case env := <-a.mailbox:
-					result := a.handler(loopCtx, env)
+					handlerCtx := loopCtx
+					if env.Ctx != nil {
+						handlerCtx = env.Ctx
+					}
+					result := a.handler(handlerCtx, env)
 					if env.ResponseCh != nil {
 						select {
 						case env.ResponseCh <- result:
@@ -99,21 +104,29 @@ func (s *System) Send(ctx context.Context, actorName string, env contracts.Agent
 		timeout = 90 * time.Second
 	}
 
+	taskCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
 	responseCh := make(chan contracts.AgentResult, 1)
 	env.ResponseCh = responseCh
 	env.DispatchedAt = time.Now().UTC()
+	env.Ctx = taskCtx
 
 	select {
-	case <-ctx.Done():
-		return contracts.AgentResult{}, ctx.Err()
+	case <-taskCtx.Done():
+		if errors.Is(taskCtx.Err(), context.DeadlineExceeded) {
+			return contracts.AgentResult{}, fmt.Errorf("actor %s timeout", actorName)
+		}
+		return contracts.AgentResult{}, taskCtx.Err()
 	case a.mailbox <- env:
 	}
 
 	select {
-	case <-ctx.Done():
-		return contracts.AgentResult{}, ctx.Err()
-	case <-time.After(timeout):
-		return contracts.AgentResult{}, fmt.Errorf("actor %s timeout", actorName)
+	case <-taskCtx.Done():
+		if errors.Is(taskCtx.Err(), context.DeadlineExceeded) {
+			return contracts.AgentResult{}, fmt.Errorf("actor %s timeout", actorName)
+		}
+		return contracts.AgentResult{}, taskCtx.Err()
 	case res := <-responseCh:
 		return res, nil
 	}

erotic-models-SSOT.md

@@ -4,115 +4,62 @@ This file is the single source of truth for continuing work on this repo/thread
 
 ## Current Context (Resume Here)
 - Repo root: /Users/jeremy/dev/BIOMETRICS
-- Active branch: codex/v3.1-ga-closure
-- PR: https://github.com/Delqhi/BIOMETRICS/pull/16 (base: main)
-- Goal: get PR #16 fully green, merge to main, then clean up branch.
+- Branch: main (dirty working tree)
+- Last merged PR: https://github.com/Delqhi/BIOMETRICS/pull/16 (merge commit: 8a4cc81)
+- Goal: make GA-closure soak/rehearsal runs reliable locally (controlplane + opencode execution + soak harness).
 
-### What Was Broken (CI)
-- Go race detector failures in background manager and orchestrator run handling.
-- Go tests flaking due to resume endpoint finishing after test cleanup.
-- Delegation priority queue deadlock (mutex + heap.Interface re-locked).
-- Logging Sync() failing on stderr/dev/stderr in tests.
-- Auth CA generation test expectation mismatched behavior.
-- Lint job failing: golangci-lint config invalid (combined enable-all + enable).
-- Windows tests failing: PowerShell parsing of `-coverprofile=coverage.out` causing `.out` package error.
-- CodeQL failing: new HIGH alerts (path traversal + allocation capacity).
+## Current Issue
+- Release rehearsal/soak runs previously had very low success rate (example: `logs/soak/soak-summary-20260226T054141Z.json` shows 1/59 completed).
+- Recent local runs showed:
+  - codex provider is unavailable when not logged in (should fall back to gemini/nim)
+  - actor timeouts did not cancel underlying agent execution cleanly
+  - opencode adapter enforced a hard 180s timeout even when scheduler allows 600s for coder/fixer
 
-### What We Changed (Local Working Tree)
-Files currently modified (commit pending):
-- .github/workflows/ci.yml
-  - Run Windows `go test` step under bash.
-- .github/workflows/codeql.yml
-  - Ignore `archive/**` in CodeQL scan.
-- biometrics-cli/internal/runtime/background/manager.go
-  - Return snapshots under lock; Get() returns a snapshot under RLock.
-- biometrics-cli/internal/runtime/orchestrator/service.go
-  - Add run-generation gating to stop old goroutines after ResumeFromStep.
-  - Clone run state for reads/JSON to avoid races (`cloneRunForRead`).
-  - Enforce arena branch path prefix guard (Abs + Rel).
-- biometrics-cli/internal/api/http/server_test.go
-  - Register cancellation cleanup after TempDir init (prevents TempDir cleanup flake).
-  - Poll run status until terminal state for resume endpoint.
-- biometrics-cli/internal/controlplane/app.go
-  - Prevent UI path traversal (Abs + Rel before ServeFile).
-- biometrics-cli/internal/runtime/scheduler/manager.go
-  - Prevent FS path traversal (Clean + Rel prefix + symlink guard) for ReadFile/ListDir.
-- biometrics-cli/internal/skillops/ops.go
-  - Restrict skill create path to workspace/codex roots (Abs + Rel prefix).
-- biometrics-cli/internal/store/sqlite/store.go
-  - Avoid request-derived slice capacities in list calls.
-- biometrics-cli/internal/evals/dataset.go
-  - Avoid request-derived slice capacity in dataset generation.
-- biometrics-cli/pkg/delegation/queue.go
-  - Remove internal locking from heap.Interface methods; external lock owns heap ops.
-- biometrics-cli/pkg/delegation/delegation_test.go
-  - Register a worker-capable agent in TestWorkerPool.
-- biometrics-cli/pkg/logging/logger.go
-  - Ignore common non-fatal Sync() errors (bad fd/ioctl/invalid arg).
-- biometrics-cli/pkg/logging/logging.go
-  - Apply same ignorable Sync error handling.
-- biometrics-cli/pkg/logging/logging_test.go
-  - BufferedLogger test uses flushAt=1 to match expectation.
-- biometrics-cli/pkg/auth/mtls_test.go
-  - Expect CA generation to create missing dirs (no error).
-- biometrics-cli/.golangci.yml
-  - Simplify linters to minimal set so CI lint job is green.
+## Changes In Working Tree (Not Committed)
+- `biometrics-cli/internal/contracts/types.go`
+  - add `AgentEnvelope.Ctx context.Context` (json ignored) to carry a per-task context
+- `biometrics-cli/internal/runtime/actor/system.go`
+  - create per-send `taskCtx` with timeout; set `env.Ctx`; pass it into handler; remove `time.After`
+- `biometrics-cli/internal/executor/opencode/adapter.go`
+  - honor existing context deadline; only apply 10m default when caller provides none
+- `scripts/run-soak.sh`
+  - default `GOAL_PREFIX` is now a no-op instruction: `soak noop (reply ok only / no file edits / no commands / no internet)`
 
-Local verification already run:
-- (in biometrics-cli/) `gofmt -w <touched files>`
-- (in biometrics-cli/) `go test -race -coverprofile=coverage.out -covermode=atomic ./...`
-- (repo root) `./scripts/release/check-gates.sh`
-- (in biometrics-cli/) `$(go env GOPATH)/bin/golangci-lint run --timeout=5m`
+- `scripts/visual_truth/vt`
+  - Visual Truth step runner: records desktop via `ffmpeg -f avfoundation` into `/tmp/automation_logs/<session>/<step>/...`
+  - Enforces "no recording, no command": refuses to execute step unless mp4 file is created and growing
+  - `vt doctor` prints device list and runs a short probe capture; currently fails until macOS Screen Recording permission is granted
 
-## Essential Commands
+- `scripts/visual_truth/vt_validate.py`
+  - Optional NVIDIA NIM video validation (OpenAI-compatible `POST /v1/chat/completions` with `video_url`)
+  - Hard-gated: requires `VISUAL_TRUTH_ALLOW_UPLOAD=1` plus either `VISUAL_TRUTH_UPLOAD_CMD` or `VISUAL_TRUTH_ALLOW_BASE64=1`
+  - Accepts `VALIDATED` or `STATUS: VALIDATED` (and ERROR variants)
 
-### Repo gates (local)
-- `./scripts/release/check-gates.sh`
+- `scripts/visual_truth/visual_truth.py`
+  - Python context-manager template: `with VideoRecorder(task="..."):` for micro scripts
 
-### Go (control plane / cli)
-- `cd biometrics-cli && go test ./...`
-- `cd biometrics-cli && go test -race ./...`
+- `scripts/release/run-ga-closure-program.sh`
+  - When `VISUAL_TRUTH=1`, each `run_step` is executed via `scripts/visual_truth/vt step --name ga-closure:<step>`
+  - Auto-sets `VISUAL_TRUTH_SESSION=ga-closure-<timestamp>` if not provided
+  - Runs `scripts/visual_truth/vt doctor` preflight when `VISUAL_TRUTH=1`
 
-### PR/CI (GitHub)
-- `gh pr view 16`
-- `gh pr checks 16`
-- `gh run view <run-id> --log-failed`
+- `scripts/release/run-ga-closure-visual-truth.sh`
+  - Strict wrapper that enforces Visual Truth + NIM validation (fails fast if permissions/keys/upload mode missing)
 
-## Key Directories (Mental Map)
-- biometrics-cli/ : Go backend + CLI tooling
-- website/ : Next.js site
-- scripts/ : local gates and automation
-- docs/ : specs/runbooks (OpenAPI etc)
+## Local Verification (2026-03-14)
+- `cd biometrics-cli && go test ./...` PASS
+- `./scripts/release/check-gates.sh` PASS
+- controlplane boot + API smoke: `./scripts/release/runtime-surface-smoke.sh` PASS (needs controlplane running)
+- API smoke runs:
+  - model_preference=gemini run completed; coder output `ok` (run_id: d0880332-7fe1-4862-9ebe-3874d3fafa76)
+  - default routing with 3 work packages (noop goal prefix) completed (run_id: c30b313a-dab4-44b2-bb18-bfb1b374a166)
 
-## Important Interfaces
-- Control plane OpenAPI: docs/api/openapi-v3-controlplane.yaml
-- Default base URL (from docs/README): http://127.0.0.1:59013
+## Running Controlplane (Local)
+- build: `cd biometrics-cli && go build -o ../bin/controlplane ./cmd/controlplane`
+- start: `PORT=59013 BIOMETRICS_BIND_ADDR=127.0.0.1 ./bin/controlplane`
+- pid file: `logs/release/controlplane.pid`
+- stop: `kill $(cat logs/release/controlplane.pid)`
 
-## Decisions
-- `archive/legacy-v2/` removed; references cleaned (CodeQL + no-archive rule).
-
-## Research Sources (CI/Go)
-- https://github.com/golang/go/issues/43547
-- https://github.com/golang/go/issues/51442
-- https://github.com/golang/go/issues/66148
-- https://github.com/golang/go/issues/78131
-- https://github.com/kubernetes/kubernetes/issues/137387
-- https://github.com/gravitational/teleport/issues/13501
-- https://github.com/google/syzkaller/issues/4920
-- https://dev.to/xuanyu/test-in-go-the-order-of-cleanup-is-not-what-you-think-4o8k
-- https://ieftimov.com/posts/testing-in-go-clean-tests-using-t-cleanup/
-- https://dev.to/salesforceeng/subtesting-skipping-and-cleanup-in-the-go-testing-t-49ea
-- https://brandur.org/fragments/go-prefer-t-cleanup-with-parallel-subtests
-- https://lesiw.dev/go/cleanup
-- https://github.com/golangci/golangci-lint/issues/968
-- https://github.com/golangci/golangci-lint/issues/1888
-- https://golangci-lint.run/docs/configuration/
-- https://github.com/golang/go/issues/72015
-- https://stackoverflow.com/questions/79012985/no-required-module-provides-package-out-error-when-running-go-test-coverprof
-- https://github.com/golang/go/issues/51126
-- https://github.com/golang/go/issues/70244
-
-## Next Actions (Minimal)
-1) Commit + push the current working tree fixes on `codex/v3.1-ga-closure`.
-2) Wait for CI rerun; fix only what still fails.
-3) Merge PR #16 when all checks are green.
+## Next Minimal Actions
+1) Commit these 4 files on a new branch and open a PR to main.
+2) After merge, rerun a short soak/rehearsal to confirm success rate is stable.

scripts/release/run-ga-closure-program.sh

@@ -27,6 +27,15 @@ timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
 log_file="${LOG_DIR}/ga-closure-${timestamp}.log"
 exec > >(tee -a "${log_file}") 2>&1
 
+if [[ "${VISUAL_TRUTH:-0}" == "1" && -z "${VISUAL_TRUTH_SESSION:-}" ]]; then
+  export VISUAL_TRUTH_SESSION="ga-closure-${timestamp}"
+fi
+
+if [[ "${VISUAL_TRUTH:-0}" == "1" && -x "${ROOT_DIR}/scripts/visual_truth/vt" ]]; then
+  echo "[ga-closure] visual truth enabled: running vt doctor"
+  "${ROOT_DIR}/scripts/visual_truth/vt" doctor
+fi
+
 usage() {
   cat <<USAGE
 Usage: $(basename "$0") [options]
@@ -280,7 +289,12 @@ run_step() {
   echo "[ga-closure] step=${step} started"
   local cmd_pid=""
   local cmd_rc=0
-  "$@" &
+  local vt_bin="${ROOT_DIR}/scripts/visual_truth/vt"
+  if [[ "${VISUAL_TRUTH:-0}" == "1" && -x "${vt_bin}" ]]; then
+    "${vt_bin}" step --name "ga-closure:${step}" -- "$@" &
+  else
+    "$@" &
+  fi
   cmd_pid="$!"
   append_state_event "${step}" "started" "started pid=${cmd_pid}"
 

scripts/release/run-ga-closure-visual-truth.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+export VISUAL_TRUTH=1
+export VISUAL_TRUTH_DIR="${VISUAL_TRUTH_DIR:-/tmp/automation_logs}"
+export VISUAL_TRUTH_VALIDATE=1
+export VISUAL_TRUTH_VALIDATE_STRICT=1
+export VISUAL_TRUTH_ALLOW_UPLOAD="${VISUAL_TRUTH_ALLOW_UPLOAD:-0}"
+
+if [[ -z "${VISUAL_TRUTH_SESSION:-}" ]]; then
+  export VISUAL_TRUTH_SESSION="ga-closure-vt-$(date -u +%Y%m%dT%H%M%SZ)"
+fi
+
+vt_bin="${ROOT_DIR}/scripts/visual_truth/vt"
+if [[ ! -x "${vt_bin}" ]]; then
+  echo "missing visual truth runner: ${vt_bin}" >&2
+  exit 2
+fi
+
+if [[ -z "${NIM_API_KEY:-}" && -z "${NVIDIA_API_KEY:-}" ]]; then
+  echo "missing NIM_API_KEY (or NVIDIA_API_KEY)" >&2
+  exit 2
+fi
+
+if [[ "${VISUAL_TRUTH_ALLOW_UPLOAD}" != "1" ]]; then
+  echo "VISUAL_TRUTH_ALLOW_UPLOAD must be 1" >&2
+  exit 2
+fi
+
+if [[ -z "${VISUAL_TRUTH_UPLOAD_CMD:-}" && "${VISUAL_TRUTH_ALLOW_BASE64:-0}" != "1" ]]; then
+  echo "set VISUAL_TRUTH_UPLOAD_CMD or VISUAL_TRUTH_ALLOW_BASE64=1" >&2
+  exit 2
+fi
+
+"${vt_bin}" doctor
+exec "${ROOT_DIR}/scripts/release/run-ga-closure-program.sh" "$@"

scripts/run-soak.sh

@@ -5,10 +5,14 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 
 API_BASE="${API_BASE:-http://127.0.0.1:59013}"
 PROJECT_ID="${PROJECT_ID:-soak}"
-GOAL_PREFIX="${GOAL_PREFIX:-soak run}"
+GOAL_PREFIX="${GOAL_PREFIX:-soak noop (reply ok only / no file edits / no commands / no internet)}"
 GOAL_PARTS="${GOAL_PARTS:-50}"
 SCHEDULER_MODE="${SCHEDULER_MODE:-dag_parallel_v1}"
 MAX_PARALLELISM="${MAX_PARALLELISM:-8}"
+MODEL_PREFERENCE="${MODEL_PREFERENCE:-}"
+FALLBACK_CHAIN="${FALLBACK_CHAIN:-}"
+MODEL_ID="${MODEL_ID:-}"
+CONTEXT_BUDGET="${CONTEXT_BUDGET:-}"
 PROFILE_LABEL="${PROFILE_LABEL:-unspecified}"
 RUN_INTERVAL_SECONDS="${RUN_INTERVAL_SECONDS:-15}"
 POLL_INTERVAL_SECONDS="${POLL_INTERVAL_SECONDS:-2}"
@@ -156,18 +160,40 @@ print(", ".join(segments))
 PY
 )"
 
-  payload="$(python3 - "${PROJECT_ID}" "${goal}" "${SCHEDULER_MODE}" "${MAX_PARALLELISM}" <<'PY'
+  payload="$(python3 - "${PROJECT_ID}" "${goal}" "${SCHEDULER_MODE}" "${MAX_PARALLELISM}" "${MODEL_PREFERENCE}" "${FALLBACK_CHAIN}" "${MODEL_ID}" "${CONTEXT_BUDGET}" <<'PY'
 import json
 import sys
 
-project_id, goal, scheduler_mode, max_parallelism = sys.argv[1:5]
+project_id, goal, scheduler_mode, max_parallelism, model_preference, fallback_chain, model_id, context_budget = sys.argv[1:9]
 payload = {
     "project_id": project_id,
     "goal": goal,
     "mode": "autonomous",
     "scheduler_mode": scheduler_mode,
     "max_parallelism": int(max_parallelism),
 }
+
+model_preference = model_preference.strip()
+if model_preference:
+    payload["model_preference"] = model_preference
+
+fallback_chain = fallback_chain.strip()
+if fallback_chain:
+    chain = [p.strip() for p in fallback_chain.split(",") if p.strip()]
+    if chain:
+        payload["fallback_chain"] = chain
+
+model_id = model_id.strip()
+if model_id:
+    payload["model_id"] = model_id
+
+context_budget = context_budget.strip()
+if context_budget:
+    try:
+        payload["context_budget"] = int(context_budget)
+    except Exception:
+        pass
+
 print(json.dumps(payload))
 PY
 )"