From b5cb7bc1185f11fc11a093457874b62f6be51858 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Sun, 15 Mar 2026 03:08:18 +0100 Subject: [PATCH 1/2] fix(runtime): propagate task deadlines into agent execution --- biometrics-cli/internal/contracts/types.go | 2 ++ .../internal/executor/opencode/adapter.go | 12 ++++++--- .../internal/runtime/actor/system.go | 27 ++++++++++++++----- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/biometrics-cli/internal/contracts/types.go b/biometrics-cli/internal/contracts/types.go index c68d6f4..49a1aa6 100644 --- a/biometrics-cli/internal/contracts/types.go +++ b/biometrics-cli/internal/contracts/types.go @@ -1,6 +1,7 @@ package contracts import ( + "context" "strings" "time" ) @@ -165,6 +166,7 @@ type AgentEnvelope struct { BlueprintModules []string `json:"blueprint_modules,omitempty"` Bootstrap bool `json:"bootstrap,omitempty"` Input map[string]string `json:"input,omitempty"` + Ctx context.Context `json:"-"` ResponseCh chan AgentResult `json:"-"` DispatchedAt time.Time `json:"dispatched_at"` } diff --git a/biometrics-cli/internal/executor/opencode/adapter.go b/biometrics-cli/internal/executor/opencode/adapter.go index c94195a..99bbcf5 100644 --- a/biometrics-cli/internal/executor/opencode/adapter.go +++ b/biometrics-cli/internal/executor/opencode/adapter.go @@ -138,8 +138,14 @@ func (a *Adapter) executeOnce( binaryPath, runID, agentName, prompt, projectID string, metadata routingMetadata, ) (string, error) { - timeoutCtx, cancel := context.WithTimeout(ctx, 180*time.Second) - defer cancel() + execCtx := ctx + var cancel context.CancelFunc + if _, ok := ctx.Deadline(); !ok { + execCtx, cancel = context.WithTimeout(ctx, 10*time.Minute) + } + if cancel != nil { + defer cancel() + } args := []string{"run"} if strings.TrimSpace(agentName) != "" { @@ -153,7 +159,7 @@ func (a *Adapter) executeOnce( } args = append(args, prompt) - cmd := exec.CommandContext(timeoutCtx, binaryPath, args...) + cmd := exec.CommandContext(execCtx, binaryPath, args...) cmd.Env = append(os.Environ(), "PROJECT_ID="+projectID, "BIOMETRICS_RUN_ID="+runID, diff --git a/biometrics-cli/internal/runtime/actor/system.go b/biometrics-cli/internal/runtime/actor/system.go index 09c7a79..f08a28e 100644 --- a/biometrics-cli/internal/runtime/actor/system.go +++ b/biometrics-cli/internal/runtime/actor/system.go @@ -2,6 +2,7 @@ package actor import ( "context" + "errors" "fmt" "sync" "time" @@ -75,7 +76,11 @@ func (s *System) Start(ctx context.Context) { case <-loopCtx.Done(): return case env := <-a.mailbox: - result := a.handler(loopCtx, env) + handlerCtx := loopCtx + if env.Ctx != nil { + handlerCtx = env.Ctx + } + result := a.handler(handlerCtx, env) if env.ResponseCh != nil { select { case env.ResponseCh <- result: @@ -99,21 +104,29 @@ func (s *System) Send(ctx context.Context, actorName string, env contracts.Agent timeout = 90 * time.Second } + taskCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + responseCh := make(chan contracts.AgentResult, 1) env.ResponseCh = responseCh env.DispatchedAt = time.Now().UTC() + env.Ctx = taskCtx select { - case <-ctx.Done(): - return contracts.AgentResult{}, ctx.Err() + case <-taskCtx.Done(): + if errors.Is(taskCtx.Err(), context.DeadlineExceeded) { + return contracts.AgentResult{}, fmt.Errorf("actor %s timeout", actorName) + } + return contracts.AgentResult{}, taskCtx.Err() case a.mailbox <- env: } select { - case <-ctx.Done(): - return contracts.AgentResult{}, ctx.Err() - case <-time.After(timeout): - return contracts.AgentResult{}, fmt.Errorf("actor %s timeout", actorName) + case <-taskCtx.Done(): + if errors.Is(taskCtx.Err(), context.DeadlineExceeded) { + return contracts.AgentResult{}, fmt.Errorf("actor %s timeout", actorName) + } + return contracts.AgentResult{}, taskCtx.Err() case res := <-responseCh: return res, nil } From 03515e5c202225112a6b498ebe66ab1d21028e9d Mon Sep 17 00:00:00 2001 From: Jeremy Date: Sun, 15 Mar 2026 03:09:17 +0100 Subject: [PATCH 2/2] release: add Visual Truth step recording and NIM validation hooks --- erotic-models-SSOT.md | 151 ++---- scripts/release/run-ga-closure-program.sh | 16 +- .../release/run-ga-closure-visual-truth.sh | 38 ++ scripts/run-soak.sh | 32 +- scripts/visual_truth/visual_truth.py | 209 +++++++++ scripts/visual_truth/vt | 434 ++++++++++++++++++ scripts/visual_truth/vt_validate.py | 295 ++++++++++++ 7 files changed, 1069 insertions(+), 106 deletions(-) create mode 100755 scripts/release/run-ga-closure-visual-truth.sh create mode 100755 scripts/visual_truth/visual_truth.py create mode 100755 scripts/visual_truth/vt create mode 100755 scripts/visual_truth/vt_validate.py diff --git a/erotic-models-SSOT.md b/erotic-models-SSOT.md index 19a3adc..b7c5317 100644 --- a/erotic-models-SSOT.md +++ b/erotic-models-SSOT.md @@ -4,115 +4,62 @@ This file is the single source of truth for continuing work on this repo/thread ## Current Context (Resume Here) - Repo root: /Users/jeremy/dev/BIOMETRICS -- Active branch: codex/v3.1-ga-closure -- PR: https://github.com/Delqhi/BIOMETRICS/pull/16 (base: main) -- Goal: get PR #16 fully green, merge to main, then clean up branch. +- Branch: main (dirty working tree) +- Last merged PR: https://github.com/Delqhi/BIOMETRICS/pull/16 (merge commit: 8a4cc81) +- Goal: make GA-closure soak/rehearsal runs reliable locally (controlplane + opencode execution + soak harness). -### What Was Broken (CI) -- Go race detector failures in background manager and orchestrator run handling. -- Go tests flaking due to resume endpoint finishing after test cleanup. -- Delegation priority queue deadlock (mutex + heap.Interface re-locked). -- Logging Sync() failing on stderr/dev/stderr in tests. -- Auth CA generation test expectation mismatched behavior. -- Lint job failing: golangci-lint config invalid (combined enable-all + enable). -- Windows tests failing: PowerShell parsing of `-coverprofile=coverage.out` causing `.out` package error. -- CodeQL failing: new HIGH alerts (path traversal + allocation capacity). +## Current Issue +- Release rehearsal/soak runs previously had very low success rate (example: `logs/soak/soak-summary-20260226T054141Z.json` shows 1/59 completed). +- Recent local runs showed: + - codex provider is unavailable when not logged in (should fall back to gemini/nim) + - actor timeouts did not cancel underlying agent execution cleanly + - opencode adapter enforced a hard 180s timeout even when scheduler allows 600s for coder/fixer -### What We Changed (Local Working Tree) -Files currently modified (commit pending): -- .github/workflows/ci.yml - - Run Windows `go test` step under bash. -- .github/workflows/codeql.yml - - Ignore `archive/**` in CodeQL scan. -- biometrics-cli/internal/runtime/background/manager.go - - Return snapshots under lock; Get() returns a snapshot under RLock. -- biometrics-cli/internal/runtime/orchestrator/service.go - - Add run-generation gating to stop old goroutines after ResumeFromStep. - - Clone run state for reads/JSON to avoid races (`cloneRunForRead`). - - Enforce arena branch path prefix guard (Abs + Rel). -- biometrics-cli/internal/api/http/server_test.go - - Register cancellation cleanup after TempDir init (prevents TempDir cleanup flake). - - Poll run status until terminal state for resume endpoint. -- biometrics-cli/internal/controlplane/app.go - - Prevent UI path traversal (Abs + Rel before ServeFile). -- biometrics-cli/internal/runtime/scheduler/manager.go - - Prevent FS path traversal (Clean + Rel prefix + symlink guard) for ReadFile/ListDir. -- biometrics-cli/internal/skillops/ops.go - - Restrict skill create path to workspace/codex roots (Abs + Rel prefix). -- biometrics-cli/internal/store/sqlite/store.go - - Avoid request-derived slice capacities in list calls. -- biometrics-cli/internal/evals/dataset.go - - Avoid request-derived slice capacity in dataset generation. -- biometrics-cli/pkg/delegation/queue.go - - Remove internal locking from heap.Interface methods; external lock owns heap ops. -- biometrics-cli/pkg/delegation/delegation_test.go - - Register a worker-capable agent in TestWorkerPool. -- biometrics-cli/pkg/logging/logger.go - - Ignore common non-fatal Sync() errors (bad fd/ioctl/invalid arg). -- biometrics-cli/pkg/logging/logging.go - - Apply same ignorable Sync error handling. -- biometrics-cli/pkg/logging/logging_test.go - - BufferedLogger test uses flushAt=1 to match expectation. -- biometrics-cli/pkg/auth/mtls_test.go - - Expect CA generation to create missing dirs (no error). -- biometrics-cli/.golangci.yml - - Simplify linters to minimal set so CI lint job is green. +## Changes In Working Tree (Not Committed) +- `biometrics-cli/internal/contracts/types.go` + - add `AgentEnvelope.Ctx context.Context` (json ignored) to carry a per-task context +- `biometrics-cli/internal/runtime/actor/system.go` + - create per-send `taskCtx` with timeout; set `env.Ctx`; pass it into handler; remove `time.After` +- `biometrics-cli/internal/executor/opencode/adapter.go` + - honor existing context deadline; only apply 10m default when caller provides none +- `scripts/run-soak.sh` + - default `GOAL_PREFIX` is now a no-op instruction: `soak noop (reply ok only / no file edits / no commands / no internet)` -Local verification already run: -- (in biometrics-cli/) `gofmt -w ` -- (in biometrics-cli/) `go test -race -coverprofile=coverage.out -covermode=atomic ./...` -- (repo root) `./scripts/release/check-gates.sh` -- (in biometrics-cli/) `$(go env GOPATH)/bin/golangci-lint run --timeout=5m` +- `scripts/visual_truth/vt` + - Visual Truth step runner: records desktop via `ffmpeg -f avfoundation` into `/tmp/automation_logs///...` + - Enforces "no recording, no command": refuses to execute step unless mp4 file is created and growing + - `vt doctor` prints device list and runs a short probe capture; currently fails until macOS Screen Recording permission is granted -## Essential Commands +- `scripts/visual_truth/vt_validate.py` + - Optional NVIDIA NIM video validation (OpenAI-compatible `POST /v1/chat/completions` with `video_url`) + - Hard-gated: requires `VISUAL_TRUTH_ALLOW_UPLOAD=1` plus either `VISUAL_TRUTH_UPLOAD_CMD` or `VISUAL_TRUTH_ALLOW_BASE64=1` + - Accepts `VALIDATED` or `STATUS: VALIDATED` (and ERROR variants) -### Repo gates (local) -- `./scripts/release/check-gates.sh` +- `scripts/visual_truth/visual_truth.py` + - Python context-manager template: `with VideoRecorder(task="..."):` for micro scripts -### Go (control plane / cli) -- `cd biometrics-cli && go test ./...` -- `cd biometrics-cli && go test -race ./...` +- `scripts/release/run-ga-closure-program.sh` + - When `VISUAL_TRUTH=1`, each `run_step` is executed via `scripts/visual_truth/vt step --name ga-closure:` + - Auto-sets `VISUAL_TRUTH_SESSION=ga-closure-` if not provided + - Runs `scripts/visual_truth/vt doctor` preflight when `VISUAL_TRUTH=1` -### PR/CI (GitHub) -- `gh pr view 16` -- `gh pr checks 16` -- `gh run view --log-failed` +- `scripts/release/run-ga-closure-visual-truth.sh` + - Strict wrapper that enforces Visual Truth + NIM validation (fails fast if permissions/keys/upload mode missing) -## Key Directories (Mental Map) -- biometrics-cli/ : Go backend + CLI tooling -- website/ : Next.js site -- scripts/ : local gates and automation -- docs/ : specs/runbooks (OpenAPI etc) +## Local Verification (2026-03-14) +- `cd biometrics-cli && go test ./...` PASS +- `./scripts/release/check-gates.sh` PASS +- controlplane boot + API smoke: `./scripts/release/runtime-surface-smoke.sh` PASS (needs controlplane running) +- API smoke runs: + - model_preference=gemini run completed; coder output `ok` (run_id: d0880332-7fe1-4862-9ebe-3874d3fafa76) + - default routing with 3 work packages (noop goal prefix) completed (run_id: c30b313a-dab4-44b2-bb18-bfb1b374a166) -## Important Interfaces -- Control plane OpenAPI: docs/api/openapi-v3-controlplane.yaml -- Default base URL (from docs/README): http://127.0.0.1:59013 +## Running Controlplane (Local) +- build: `cd biometrics-cli && go build -o ../bin/controlplane ./cmd/controlplane` +- start: `PORT=59013 BIOMETRICS_BIND_ADDR=127.0.0.1 ./bin/controlplane` +- pid file: `logs/release/controlplane.pid` +- stop: `kill $(cat logs/release/controlplane.pid)` -## Decisions -- `archive/legacy-v2/` removed; references cleaned (CodeQL + no-archive rule). - -## Research Sources (CI/Go) -- https://github.com/golang/go/issues/43547 -- https://github.com/golang/go/issues/51442 -- https://github.com/golang/go/issues/66148 -- https://github.com/golang/go/issues/78131 -- https://github.com/kubernetes/kubernetes/issues/137387 -- https://github.com/gravitational/teleport/issues/13501 -- https://github.com/google/syzkaller/issues/4920 -- https://dev.to/xuanyu/test-in-go-the-order-of-cleanup-is-not-what-you-think-4o8k -- https://ieftimov.com/posts/testing-in-go-clean-tests-using-t-cleanup/ -- https://dev.to/salesforceeng/subtesting-skipping-and-cleanup-in-the-go-testing-t-49ea -- https://brandur.org/fragments/go-prefer-t-cleanup-with-parallel-subtests -- https://lesiw.dev/go/cleanup -- https://github.com/golangci/golangci-lint/issues/968 -- https://github.com/golangci/golangci-lint/issues/1888 -- https://golangci-lint.run/docs/configuration/ -- https://github.com/golang/go/issues/72015 -- https://stackoverflow.com/questions/79012985/no-required-module-provides-package-out-error-when-running-go-test-coverprof -- https://github.com/golang/go/issues/51126 -- https://github.com/golang/go/issues/70244 - -## Next Actions (Minimal) -1) Commit + push the current working tree fixes on `codex/v3.1-ga-closure`. -2) Wait for CI rerun; fix only what still fails. -3) Merge PR #16 when all checks are green. +## Next Minimal Actions +1) Commit these 4 files on a new branch and open a PR to main. +2) After merge, rerun a short soak/rehearsal to confirm success rate is stable. diff --git a/scripts/release/run-ga-closure-program.sh b/scripts/release/run-ga-closure-program.sh index a7c4417..65f4e64 100755 --- a/scripts/release/run-ga-closure-program.sh +++ b/scripts/release/run-ga-closure-program.sh @@ -27,6 +27,15 @@ timestamp="$(date -u +%Y%m%dT%H%M%SZ)" log_file="${LOG_DIR}/ga-closure-${timestamp}.log" exec > >(tee -a "${log_file}") 2>&1 +if [[ "${VISUAL_TRUTH:-0}" == "1" && -z "${VISUAL_TRUTH_SESSION:-}" ]]; then + export VISUAL_TRUTH_SESSION="ga-closure-${timestamp}" +fi + +if [[ "${VISUAL_TRUTH:-0}" == "1" && -x "${ROOT_DIR}/scripts/visual_truth/vt" ]]; then + echo "[ga-closure] visual truth enabled: running vt doctor" + "${ROOT_DIR}/scripts/visual_truth/vt" doctor +fi + usage() { cat <&2 + exit 2 +fi + +if [[ -z "${NIM_API_KEY:-}" && -z "${NVIDIA_API_KEY:-}" ]]; then + echo "missing NIM_API_KEY (or NVIDIA_API_KEY)" >&2 + exit 2 +fi + +if [[ "${VISUAL_TRUTH_ALLOW_UPLOAD}" != "1" ]]; then + echo "VISUAL_TRUTH_ALLOW_UPLOAD must be 1" >&2 + exit 2 +fi + +if [[ -z "${VISUAL_TRUTH_UPLOAD_CMD:-}" && "${VISUAL_TRUTH_ALLOW_BASE64:-0}" != "1" ]]; then + echo "set VISUAL_TRUTH_UPLOAD_CMD or VISUAL_TRUTH_ALLOW_BASE64=1" >&2 + exit 2 +fi + +"${vt_bin}" doctor +exec "${ROOT_DIR}/scripts/release/run-ga-closure-program.sh" "$@" diff --git a/scripts/run-soak.sh b/scripts/run-soak.sh index fb2467b..21708ca 100755 --- a/scripts/run-soak.sh +++ b/scripts/run-soak.sh @@ -5,10 +5,14 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" API_BASE="${API_BASE:-http://127.0.0.1:59013}" PROJECT_ID="${PROJECT_ID:-soak}" -GOAL_PREFIX="${GOAL_PREFIX:-soak run}" +GOAL_PREFIX="${GOAL_PREFIX:-soak noop (reply ok only / no file edits / no commands / no internet)}" GOAL_PARTS="${GOAL_PARTS:-50}" SCHEDULER_MODE="${SCHEDULER_MODE:-dag_parallel_v1}" MAX_PARALLELISM="${MAX_PARALLELISM:-8}" +MODEL_PREFERENCE="${MODEL_PREFERENCE:-}" +FALLBACK_CHAIN="${FALLBACK_CHAIN:-}" +MODEL_ID="${MODEL_ID:-}" +CONTEXT_BUDGET="${CONTEXT_BUDGET:-}" PROFILE_LABEL="${PROFILE_LABEL:-unspecified}" RUN_INTERVAL_SECONDS="${RUN_INTERVAL_SECONDS:-15}" POLL_INTERVAL_SECONDS="${POLL_INTERVAL_SECONDS:-2}" @@ -156,11 +160,11 @@ print(", ".join(segments)) PY )" - payload="$(python3 - "${PROJECT_ID}" "${goal}" "${SCHEDULER_MODE}" "${MAX_PARALLELISM}" <<'PY' + payload="$(python3 - "${PROJECT_ID}" "${goal}" "${SCHEDULER_MODE}" "${MAX_PARALLELISM}" "${MODEL_PREFERENCE}" "${FALLBACK_CHAIN}" "${MODEL_ID}" "${CONTEXT_BUDGET}" <<'PY' import json import sys -project_id, goal, scheduler_mode, max_parallelism = sys.argv[1:5] +project_id, goal, scheduler_mode, max_parallelism, model_preference, fallback_chain, model_id, context_budget = sys.argv[1:9] payload = { "project_id": project_id, "goal": goal, @@ -168,6 +172,28 @@ payload = { "scheduler_mode": scheduler_mode, "max_parallelism": int(max_parallelism), } + +model_preference = model_preference.strip() +if model_preference: + payload["model_preference"] = model_preference + +fallback_chain = fallback_chain.strip() +if fallback_chain: + chain = [p.strip() for p in fallback_chain.split(",") if p.strip()] + if chain: + payload["fallback_chain"] = chain + +model_id = model_id.strip() +if model_id: + payload["model_id"] = model_id + +context_budget = context_budget.strip() +if context_budget: + try: + payload["context_budget"] = int(context_budget) + except Exception: + pass + print(json.dumps(payload)) PY )" diff --git a/scripts/visual_truth/visual_truth.py b/scripts/visual_truth/visual_truth.py new file mode 100755 index 0000000..d4b3fef --- /dev/null +++ b/scripts/visual_truth/visual_truth.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 + +import os +import signal +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path + + +def _utc_stamp() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def _sanitize_name(value: str) -> str: + out = [] + for ch in value or "": + if ch.isalnum() or ch in "._-": + out.append(ch) + else: + out.append("_") + return ("".join(out) or "step")[:120] + + +class VideoRecorder: + def __init__( + self, + task: str, + session: str | None = None, + base_dir: str | None = None, + screen_spec: str | None = None, + fps: int | None = None, + min_seconds: int | None = None, + validate: bool | None = None, + strict_validate: bool | None = None, + ): + self.task = task + self.session = ( + session + or os.environ.get("VISUAL_TRUTH_SESSION") + or f"{_utc_stamp()}-{os.getpid()}" + ) + self.base_dir = Path( + base_dir or os.environ.get("VISUAL_TRUTH_DIR") or "/tmp/automation_logs" + ) + self.screen_spec = ( + screen_spec or os.environ.get("VISUAL_TRUTH_SCREEN_SPEC") or "auto" + ) + self.fps = int(fps or os.environ.get("VISUAL_TRUTH_FPS") or 30) + self.min_seconds = int( + min_seconds or os.environ.get("VISUAL_TRUTH_MIN_SECONDS") or 5 + ) + if validate is None: + validate = os.environ.get("VISUAL_TRUTH_VALIDATE", "0").strip() == "1" + if strict_validate is None: + strict_validate = ( + os.environ.get("VISUAL_TRUTH_VALIDATE_STRICT", "1").strip() == "1" + ) + self.validate = bool(validate) + self.strict_validate = bool(strict_validate) + + self.step_dir: Path | None = None + self.video_path: Path | None = None + self.ffmpeg_log: Path | None = None + self.validation_out: Path | None = None + self._proc: subprocess.Popen | None = None + self._started_at = 0.0 + + def __enter__(self): + self.step_dir = self.base_dir / self.session / _sanitize_name(self.task) + self.step_dir.mkdir(parents=True, exist_ok=True) + + ts = _utc_stamp() + self.video_path = self.step_dir / f"{ts}.mp4" + self.ffmpeg_log = self.step_dir / f"{ts}.ffmpeg.log" + self.validation_out = self.step_dir / f"{ts}.validation.json" + + screen_spec = self.screen_spec + if screen_spec == "auto": + screen_spec = "Capture screen 0" + + args = [ + "ffmpeg", + "-nostdin", + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "avfoundation", + "-framerate", + str(self.fps), + "-pixel_format", + "nv12", + "-i", + screen_spec, + "-vf", + "format=yuv420p", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-crf", + "23", + str(self.video_path), + ] + + self._started_at = time.time() + with open(self.ffmpeg_log, "wb") as log_fp: + self._proc = subprocess.Popen( + args, stdout=subprocess.DEVNULL, stderr=log_fp + ) + + ok = False + deadline = time.time() + 4.0 + while time.time() < deadline: + if self._proc.poll() is not None: + break + if self.video_path.exists() and self.video_path.stat().st_size > 4096: + ok = True + break + time.sleep(0.1) + + if not ok: + self._stop_recorder() + raise RuntimeError( + "Visual Truth recording failed (check macOS Screen Recording permission for your terminal app). " + f"ffmpeg_log={self.ffmpeg_log}" + ) + + return self + + def __exit__(self, exc_type, exc, tb): + try: + if self._started_at and self.min_seconds > 0: + elapsed = time.time() - self._started_at + if elapsed < self.min_seconds: + time.sleep(self.min_seconds - elapsed) + finally: + self._stop_recorder() + + if exc_type is not None: + return False + + if self.validate and self.video_path and self.validation_out: + vt_validate = Path(__file__).resolve().parent / "vt_validate.py" + env = dict(os.environ) + env.setdefault("VISUAL_TRUTH_VALIDATE", "1") + env.setdefault( + "VISUAL_TRUTH_VALIDATE_STRICT", "1" if self.strict_validate else "0" + ) + proc = subprocess.run( + [ + "python3", + str(vt_validate), + "--video", + str(self.video_path), + "--task", + self.task, + "--out", + str(self.validation_out), + ], + env=env, + ) + if proc.returncode != 0 and self.strict_validate: + raise RuntimeError( + f"Visual Truth validation failed rc={proc.returncode} out={self.validation_out}" + ) + + return False + + def _stop_recorder(self): + proc = self._proc + if not proc: + return + + if proc.poll() is None: + try: + proc.send_signal(signal.SIGINT) + except Exception: + pass + try: + proc.wait(timeout=2.0) + except Exception: + pass + + if proc.poll() is None: + try: + proc.terminate() + except Exception: + pass + try: + proc.wait(timeout=1.0) + except Exception: + pass + + if proc.poll() is None: + try: + proc.kill() + except Exception: + pass + + self._proc = None + + +if __name__ == "__main__": + with VideoRecorder(task="demo_step"): + print("do_step()") + time.sleep(1) diff --git a/scripts/visual_truth/vt b/scripts/visual_truth/vt new file mode 100755 index 0000000..2b9ebdc --- /dev/null +++ b/scripts/visual_truth/vt @@ -0,0 +1,434 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: + vt doctor + vt step --name -- + +Env (all optional): + VISUAL_TRUTH=1 Enable recording (default: disabled) + VISUAL_TRUTH_DIR=/tmp/automation_logs + VISUAL_TRUTH_SESSION= Session id (default: utc timestamp + pid) + VISUAL_TRUTH_SCREEN_SPEC=auto avfoundation video device name (default: auto) + VISUAL_TRUTH_SCREEN_INDEX=auto (deprecated) legacy device index + VISUAL_TRUTH_FPS=30 + VISUAL_TRUTH_MIN_SECONDS=5 Minimum recording duration per step + VISUAL_TRUTH_PRE_SECONDS=0 Lead-in before command starts (recorded) + VISUAL_TRUTH_POST_SECONDS=0 Tail after command ends (recorded) + VISUAL_TRUTH_VALIDATE=0 Run NVIDIA NIM validation (default: 0) + VISUAL_TRUTH_VALIDATE_STRICT=1 Fail step when validation fails (default: 1) +USAGE +} + +require() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "missing required command: $1" >&2 + exit 127 + fi +} + +now_utc_rfc3339() { + date -u +%Y-%m-%dT%H:%M:%SZ +} + +sanitize_name() { + local in="$1" + in="${in// /_}" + in="${in//\//_}" + in="${in//\\/_}" + in="${in//:/_}" + in="${in//[^A-Za-z0-9_.-]/_}" + printf '%s' "$in" +} + +detect_screen_index() { + local out + out="$(ffmpeg -f avfoundation -list_devices true -i "" 2>&1 || true)" + local spec + spec="$(python3 - <<'PY' "$out" +import re +import sys + +text = sys.argv[1] +best = None +for line in text.splitlines(): + m = re.search(r'\[\d+\]\s+(Capture screen\s+\d+)', line) + if m: + best = m.group(1) + break +print(best or "") +PY +)" + if [[ -z "$spec" ]]; then + echo "unable to auto-detect avfoundation screen device" >&2 + exit 2 + fi + printf '%s' "$spec" +} + +resolve_screen_spec() { + local raw="$1" + if [[ "$raw" == "auto" ]]; then + detect_screen_index + return + fi + if [[ "$raw" =~ ^[0-9]+$ ]]; then + local out + out="$(ffmpeg -f avfoundation -list_devices true -i "" 2>&1 || true)" + local spec + spec="$(python3 - <<'PY' "$out" "$raw" +import re +import sys + +text = sys.argv[1] +idx = sys.argv[2] +name = "" +for line in text.splitlines(): + m = re.search(rf'\[{re.escape(idx)}\]\s+(.+)$', line) + if m: + candidate = m.group(1).strip() + if "Capture screen" not in candidate: + continue + name = candidate + break +print(name) +PY +)" + if [[ -z "$spec" ]]; then + echo "unable to resolve avfoundation device index ${raw}" >&2 + exit 2 + fi + printf '%s' "$spec" + return + fi + printf '%s' "$raw" +} + +emit_event() { + local events_file="$1" + local event_type="$2" + local step_name="$3" + local extra_json="$4" + + python3 - <<'PY' "$events_file" "$event_type" "$step_name" "$extra_json" +import json +import sys +from datetime import datetime, timezone + +path = sys.argv[1] +event_type = sys.argv[2] +step_name = sys.argv[3] +extra_raw = sys.argv[4] + +event = { + "at": datetime.now(timezone.utc).isoformat(), + "type": event_type, + "step": step_name, +} +if extra_raw: + try: + extra = json.loads(extra_raw) + if isinstance(extra, dict): + event.update(extra) + except Exception: + event["extra_raw"] = extra_raw + +with open(path, "a", encoding="utf-8") as f: + f.write(json.dumps(event, ensure_ascii=True) + "\n") +PY +} + +vt_enabled() { + [[ "${VISUAL_TRUTH:-0}" == "1" ]] +} + +subcmd="${1:-}" +if [[ -z "$subcmd" ]]; then + usage + exit 2 +fi +shift || true + +case "$subcmd" in + doctor) + require ffmpeg + require python3 + ffmpeg -version + ffmpeg -f avfoundation -list_devices true -i "" 2>&1 | sed -n '1,120p' || true + + spec="$(detect_screen_index)" + tmp_dir="${VISUAL_TRUTH_DIR:-/tmp/automation_logs}/vt-doctor-$$-$(date -u +%Y%m%dT%H%M%SZ)" + mkdir -p "$tmp_dir" + tmp_video="${tmp_dir}/probe.mp4" + tmp_log="${tmp_dir}/probe.ffmpeg.log" + + ffmpeg -nostdin -hide_banner -loglevel error -y \ + -f avfoundation -framerate 10 -pixel_format nv12 -i "$spec" \ + -t 2 -vf format=yuv420p -c:v libx264 -preset ultrafast -crf 23 \ + "$tmp_video" 2>"$tmp_log" & + pid=$! + for _ in {1..60}; do + if kill -0 "$pid" 2>/dev/null; then + sleep 0.1 + else + break + fi + done + if kill -0 "$pid" 2>/dev/null; then + kill -INT "$pid" 2>/dev/null || true + for _ in {1..20}; do + if kill -0 "$pid" 2>/dev/null; then + sleep 0.1 + else + break + fi + done + fi + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" 2>/dev/null || true + sleep 0.2 + fi + if kill -0 "$pid" 2>/dev/null; then + kill -KILL "$pid" 2>/dev/null || true + fi + wait "$pid" 2>/dev/null || true + + bytes=0 + if [[ -f "$tmp_video" ]]; then + bytes="$(wc -c <"$tmp_video" 2>/dev/null || echo 0)" + fi + if [[ "${bytes:-0}" -gt 4096 ]]; then + echo "[vt] doctor probe OK: ${tmp_video} (${bytes} bytes)" + exit 0 + fi + echo "[vt] doctor probe FAILED: no usable video produced" >&2 + echo "[vt] hint: macOS System Settings -> Privacy & Security -> Screen Recording -> enable your terminal app, then restart it" >&2 + echo "[vt] probe log: ${tmp_log}" >&2 + exit 1 + ;; + + step) + step_name="" + while [[ $# -gt 0 ]]; do + case "$1" in + --name) + step_name="${2:-}" + shift 2 + ;; + --) + shift + break + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown arg: $1" >&2 + usage + exit 2 + ;; + esac + done + + if [[ -z "$step_name" ]]; then + echo "missing --name" >&2 + exit 2 + fi + if [[ $# -lt 1 ]]; then + echo "missing command after --" >&2 + exit 2 + fi + + if ! vt_enabled; then + "$@" + exit $? + fi + + require ffmpeg + require python3 + + base_dir="${VISUAL_TRUTH_DIR:-/tmp/automation_logs}" + session="${VISUAL_TRUTH_SESSION:-$(date -u +%Y%m%dT%H%M%SZ)-$$}" + safe_step="$(sanitize_name "$step_name")" + step_dir="${base_dir}/${session}/${safe_step}" + mkdir -p "${step_dir}" + + events_file="${base_dir}/${session}/events.jsonl" + touch "${events_file}" + + ts="$(date -u +%Y%m%dT%H%M%SZ)" + video_file="${step_dir}/${ts}.mp4" + cmd_log="${step_dir}/${ts}.cmd.log" + ffmpeg_log="${step_dir}/${ts}.ffmpeg.log" + meta_file="${step_dir}/${ts}.meta.json" + + screen_spec="$(resolve_screen_spec "${VISUAL_TRUTH_SCREEN_SPEC:-${VISUAL_TRUTH_SCREEN_INDEX:-auto}}")" + fps="${VISUAL_TRUTH_FPS:-30}" + min_s="${VISUAL_TRUTH_MIN_SECONDS:-5}" + pre_s="${VISUAL_TRUTH_PRE_SECONDS:-0}" + post_s="${VISUAL_TRUTH_POST_SECONDS:-0}" + + python3 - <<'PY' "$meta_file" "$step_name" "$screen_spec" "$fps" "$min_s" "$pre_s" "$post_s" "$video_file" "$cmd_log" "$ffmpeg_log" +import json +import sys +from datetime import datetime, timezone + +def to_int(value, default): + try: + return int(value) + except Exception: + return default + +def to_float(value, default): + try: + return float(value) + except Exception: + return default + +path = sys.argv[1] +data = { + "created_at": datetime.now(timezone.utc).isoformat(), + "step": sys.argv[2], + "screen_spec": sys.argv[3], + "fps": int(sys.argv[4]), + "min_seconds": to_int(sys.argv[5], 5), + "pre_seconds": to_float(sys.argv[6], 0.0), + "post_seconds": to_float(sys.argv[7], 0.0), + "video": sys.argv[8], + "command_log": sys.argv[9], + "ffmpeg_log": sys.argv[10], +} +with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(data, indent=2, ensure_ascii=True)) +PY + + emit_event "$events_file" "step.start" "$step_name" "{\"video\":\"${video_file}\",\"cmd_log\":\"${cmd_log}\",\"ffmpeg_log\":\"${ffmpeg_log}\"}" + + rec_started_epoch="$(date +%s)" + + ffmpeg -nostdin -hide_banner -loglevel error -y \ + -f avfoundation -framerate "$fps" -pixel_format nv12 -i "$screen_spec" \ + -vf format=yuv420p -c:v libx264 -preset ultrafast -crf 23 \ + "$video_file" 2>"$ffmpeg_log" & + rec_pid=$! + + handshake_ok="false" + hb_bytes=0 + for _ in {1..40}; do + if ! kill -0 "$rec_pid" 2>/dev/null; then + break + fi + if [[ -f "$video_file" ]]; then + hb_bytes="$(wc -c <"$video_file" 2>/dev/null || echo 0)" + if [[ "${hb_bytes:-0}" -gt 4096 ]]; then + handshake_ok="true" + break + fi + fi + sleep 0.1 + done + if [[ "$handshake_ok" != "true" ]]; then + if kill -0 "$rec_pid" 2>/dev/null; then + kill -INT "$rec_pid" 2>/dev/null || true + sleep 0.2 + fi + if kill -0 "$rec_pid" 2>/dev/null; then + kill -TERM "$rec_pid" 2>/dev/null || true + sleep 0.2 + fi + if kill -0 "$rec_pid" 2>/dev/null; then + kill -KILL "$rec_pid" 2>/dev/null || true + fi + wait "$rec_pid" 2>/dev/null || true + + emit_event "$events_file" "step.error" "$step_name" "{\"error\":\"recording_failed\",\"detail\":\"recorder_not_writing\",\"bytes\":${hb_bytes:-0}}" + echo "recording failed: recorder did not produce a usable mp4 (check macOS Screen Recording permission for your terminal app)" >&2 + echo "ffmpeg_log=${ffmpeg_log}" >&2 + exit 4 + fi + + if [[ "$pre_s" != "0" ]]; then + sleep "$pre_s" + fi + + set +e + ( "$@" ) 2>&1 | tee "$cmd_log" + cmd_status=${PIPESTATUS[0]} + set -e + + if [[ "$post_s" != "0" ]]; then + sleep "$post_s" + fi + + if [[ "$min_s" =~ ^[0-9]+$ ]] && (( min_s > 0 )); then + now_epoch="$(date +%s)" + elapsed=$(( now_epoch - rec_started_epoch )) + if (( elapsed < min_s )); then + sleep $(( min_s - elapsed )) + fi + fi + + if kill -0 "$rec_pid" 2>/dev/null; then + kill -INT "$rec_pid" 2>/dev/null || true + for _ in {1..40}; do + if kill -0 "$rec_pid" 2>/dev/null; then + sleep 0.1 + else + break + fi + done + fi + if kill -0 "$rec_pid" 2>/dev/null; then + kill -TERM "$rec_pid" 2>/dev/null || true + sleep 0.2 + fi + if kill -0 "$rec_pid" 2>/dev/null; then + kill -KILL "$rec_pid" 2>/dev/null || true + fi + wait "$rec_pid" 2>/dev/null || true + + if [[ ! -f "$video_file" ]]; then + emit_event "$events_file" "step.error" "$step_name" "{\"error\":\"recording_failed\",\"detail\":\"missing_video_file\"}" + echo "recording failed: missing video file (check macOS Screen Recording permission for your terminal app)" >&2 + echo "ffmpeg_log=${ffmpeg_log}" >&2 + exit 4 + fi + video_bytes="$(wc -c <"$video_file" 2>/dev/null || echo 0)" + if [[ "${video_bytes:-0}" -lt 50000 ]]; then + emit_event "$events_file" "step.error" "$step_name" "{\"error\":\"recording_failed\",\"detail\":\"video_too_small\",\"bytes\":${video_bytes:-0}}" + echo "recording failed: video too small (check macOS Screen Recording permission for your terminal app)" >&2 + echo "ffmpeg_log=${ffmpeg_log}" >&2 + exit 4 + fi + + emit_event "$events_file" "step.end" "$step_name" "{\"status\":${cmd_status}}" + + if [[ $cmd_status -ne 0 ]]; then + emit_event "$events_file" "step.error" "$step_name" "{\"error\":\"command_failed\",\"status\":${cmd_status}}" + exit $cmd_status + fi + + if [[ "${VISUAL_TRUTH_VALIDATE:-0}" == "1" ]]; then + set +e + python3 "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/vt_validate.py" \ + --video "$video_file" \ + --task "$step_name" \ + --out "${step_dir}/${ts}.validation.json" + validate_status=$? + set -e + if [[ $validate_status -ne 0 && "${VISUAL_TRUTH_VALIDATE_STRICT:-1}" == "1" ]]; then + emit_event "$events_file" "step.error" "$step_name" "{\"error\":\"validation_failed\",\"status\":${validate_status}}" + exit $validate_status + fi + fi + + ;; + + *) + usage + exit 2 + ;; +esac diff --git a/scripts/visual_truth/vt_validate.py b/scripts/visual_truth/vt_validate.py new file mode 100755 index 0000000..6ed30df --- /dev/null +++ b/scripts/visual_truth/vt_validate.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 + +import argparse +import base64 +import json +import os +import re +import shlex +import subprocess +import sys +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + + +def now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def read_env(key: str, default: str = "") -> str: + return os.environ.get(key, default).strip() + + +def run_upload_cmd(upload_cmd: str, video_path: str) -> str: + argv = shlex.split(upload_cmd) + argv.append(video_path) + proc = subprocess.run(argv, capture_output=True, text=True) + if proc.returncode != 0: + raise RuntimeError( + f"upload cmd failed rc={proc.returncode} stderr={proc.stderr.strip()[:400]}" + ) + url = (proc.stdout or "").strip().splitlines()[-1].strip() if proc.stdout else "" + if not url: + raise RuntimeError("upload cmd returned empty url") + return url + + +def nim_chat_completion( + base_url: str, + api_key: str, + model: str, + prompt: str, + video_url: str, + media_type: str, + video_fps: float | None, +) -> dict: + url = base_url.rstrip("/") + "/chat/completions" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}", + } + payload: dict = { + "model": model, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {}, + ], + } + ], + } + + media_type = (media_type or "").strip().lower() + if media_type == "video_url": + payload["messages"][0]["content"][1] = { + "type": "video_url", + "video_url": {"url": video_url}, + } + else: + payload["messages"][0]["content"][1] = { + "type": "file", + "url": video_url, + } + if video_fps is not None: + payload["extra_body"] = { + "media_io_kwargs": {"video": {"fps": float(video_fps)}} + } + + req = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers=headers, + method="POST", + ) + with urllib.request.urlopen(req, timeout=180) as resp: + raw = resp.read().decode("utf-8") + return json.loads(raw) + + +def extract_message_content(resp: dict) -> str: + choices = resp.get("choices") or [] + if not choices: + return "" + msg = (choices[0] or {}).get("message") or {} + content = msg.get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for item in content: + if isinstance(item, dict) and item.get("type") == "text": + parts.append(str(item.get("text", ""))) + return "\n".join(parts).strip() + return "" + + +def transcode_for_validation(source: Path, dest: Path) -> None: + cmd = [ + "ffmpeg", + "-nostdin", + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + str(source), + "-vf", + "scale=1280:-2,fps=4", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "30", + "-pix_fmt", + "yuv420p", + str(dest), + ] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + raise RuntimeError( + f"ffmpeg transcode failed rc={proc.returncode} stderr={proc.stderr.strip()[:400]}" + ) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--video", required=True) + ap.add_argument("--task", required=True) + ap.add_argument("--out", required=True) + args = ap.parse_args() + + out_path = Path(args.out) + out_path.parent.mkdir(parents=True, exist_ok=True) + + validate_enabled = read_env("VISUAL_TRUTH_VALIDATE", "0") == "1" + allow_upload = read_env("VISUAL_TRUTH_ALLOW_UPLOAD", "0") == "1" + allow_base64 = read_env("VISUAL_TRUTH_ALLOW_BASE64", "0") == "1" + strict = read_env("VISUAL_TRUTH_VALIDATE_STRICT", "1") == "1" + + result: dict = { + "at": now_iso(), + "validated": False, + "task": args.task, + "video": str(args.video), + "status": "skipped" if not validate_enabled else "pending", + } + + if not validate_enabled: + out_path.write_text( + json.dumps(result, indent=2, ensure_ascii=True), encoding="utf-8" + ) + return 0 + + base_url = read_env("NIM_BASE_URL", "https://integrate.api.nvidia.com/v1") + api_key = read_env("NIM_API_KEY", "") or read_env("NVIDIA_API_KEY", "") + model = read_env("NIM_MODEL", "nvidia/cosmos-reason2-8b") + media_type = read_env("VISUAL_TRUTH_NIM_MEDIA_TYPE", "file") + video_fps_raw = read_env("VISUAL_TRUTH_VALIDATE_FPS", "4") + try: + video_fps = float(video_fps_raw) if video_fps_raw else None + except Exception: + video_fps = None + + if not api_key: + result["status"] = "error" + result["error"] = "missing NIM_API_KEY or NVIDIA_API_KEY" + out_path.write_text( + json.dumps(result, indent=2, ensure_ascii=True), encoding="utf-8" + ) + return 2 if strict else 0 + + video_path = Path(args.video) + if not video_path.exists(): + result["status"] = "error" + result["error"] = "video not found" + out_path.write_text( + json.dumps(result, indent=2, ensure_ascii=True), encoding="utf-8" + ) + return 2 if strict else 0 + + prompt = ( + "Analysiere diesen Videoabschnitt des gesamten Desktops.\n" + f"Task: {args.task}\n" + "Pruefe:\n" + "1. Wurde die Interaktion korrekt ausgefuehrt?\n" + "2. Erscheinen Fehlermeldungen im Terminal oder im Browser?\n\n" + "Antworte genau mit einem der folgenden Formate:\n" + "VALIDATED\n" + "ERROR:\n" + ) + + try: + upload_cmd = read_env("VISUAL_TRUTH_UPLOAD_CMD", "") + video_url = "" + if allow_upload and upload_cmd: + video_url = run_upload_cmd(upload_cmd, str(video_path)) + result["upload"] = {"method": "cmd", "cmd": upload_cmd} + elif allow_base64 and allow_upload: + max_bytes = int( + read_env("VISUAL_TRUTH_MAX_BASE64_BYTES", "8000000") or "8000000" + ) + encode_path = video_path + size = encode_path.stat().st_size + if size > max_bytes: + candidate = video_path.with_name(video_path.stem + ".validate.mp4") + transcode_for_validation(video_path, candidate) + encode_path = candidate + size = encode_path.stat().st_size + result["upload"] = { + "method": "base64", + "bytes": size, + "transcoded": True, + "source": str(video_path), + "derived": str(candidate), + } + if size > max_bytes: + raise RuntimeError( + f"video too large for base64 after transcode ({size} bytes > {max_bytes}); set VISUAL_TRUTH_UPLOAD_CMD" + ) + blob = encode_path.read_bytes() + b64 = base64.b64encode(blob).decode("ascii") + video_url = "data:video/mp4;base64," + b64 + result.setdefault("upload", {"method": "base64", "bytes": size}) + else: + raise RuntimeError( + "validation requires VISUAL_TRUTH_ALLOW_UPLOAD=1 and either VISUAL_TRUTH_UPLOAD_CMD or VISUAL_TRUTH_ALLOW_BASE64=1" + ) + + resp = nim_chat_completion( + base_url, api_key, model, prompt, video_url, media_type, video_fps + ) + content = extract_message_content(resp) + result["status"] = "completed" + result["nim"] = { + "base_url": base_url, + "model": model, + "media_type": media_type, + "video_fps": video_fps, + } + result["response"] = { + "content": content, + } + + verdict = (content or "").strip() + validated = False + if re.search(r"(?im)^\s*VALIDATED\s*$", verdict): + validated = True + if re.search(r"(?im)^\s*STATUS\s*:\s*VALIDATED\s*$", verdict): + validated = True + if validated: + result["validated"] = True + out_path.write_text( + json.dumps(result, indent=2, ensure_ascii=True), encoding="utf-8" + ) + return 0 + + result["validated"] = False + status_error = re.search(r"(?im)^\s*STATUS\s*:\s*(ERROR\s*:[^\n]+)", verdict) + if status_error: + result["error"] = status_error.group(1).strip()[:800] + else: + error_line = re.search(r"(?im)^\s*(ERROR\s*:[^\n]+)", verdict) + if error_line: + result["error"] = error_line.group(1).strip()[:800] + else: + result["error"] = verdict[:800] if verdict else "empty response" + out_path.write_text( + json.dumps(result, indent=2, ensure_ascii=True), encoding="utf-8" + ) + return 3 if strict else 0 + + except Exception as exc: + result["status"] = "error" + result["error"] = str(exc)[:800] + out_path.write_text( + json.dumps(result, indent=2, ensure_ascii=True), encoding="utf-8" + ) + return 2 if strict else 0 + + +if __name__ == "__main__": + raise SystemExit(main())