From 4ccd50d42e1b38af5fa25c5945c4658bc14c68e0 Mon Sep 17 00:00:00 2001 From: santhreal <64453045+santhreal@users.noreply.github.com> Date: Sun, 23 Aug 2026 16:27:45 -0700 Subject: [PATCH 01/15] =?UTF-8?q?feat(proof):=20direct=20the=20hero=20take?= =?UTF-8?q?=20=E2=80=94=20zoom=20the=20secret,=20short=20prompt,=206=C3=97?= =?UTF-8?q?=20middle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The landing-page film is one capture path with a camera, not a fixed wide shot. The scene writes zoom-in/zoom-out cues around the stored secret, the zoom stage holds that line at 2× for two seconds, the typed prompt is one readable sentence (the full contract is TASK.md), and the cut plays the build at 6× with a badge then returns to real time for signing. --- docs/handbook/src/foundations/verification.md | 17 ++- proof/docker/seed-demo.sh | 2 + proof/glyph-height.py | 58 ++++++++ proof/hero-cut.py | 14 +- proof/scenes/demo-hd.sh | 24 ++- proof/scenes/lib.sh | 5 + proof/zoom.py | 138 ++++++++++++++++-- scripts/demos/record-hd-demo.sh | 14 +- 8 files changed, 252 insertions(+), 20 deletions(-) create mode 100755 proof/glyph-height.py diff --git a/docs/handbook/src/foundations/verification.md b/docs/handbook/src/foundations/verification.md index 3b8ae39c6..3ae9264fb 100644 --- a/docs/handbook/src/foundations/verification.md +++ b/docs/handbook/src/foundations/verification.md @@ -188,10 +188,19 @@ The region is measured, not typed in: the stage diffs the frames around the mome holds the bounding box of what changed there, padded and clamped inside the frame at the source aspect ratio. A moment with nothing moving in it produces no file. -The zoom ceiling defaults to the capture width over the published width, so a held -frame is a crop rather than an upscale. The stage runs on the take, before the cut, -and keeps every frame and the recorded rate, so the cadence gate still measures the -capture's own cadence. A scene asks for one by setting `ZOOM_ARGS` in +The hero take drives the same stage from a cue file the scene writes next to its +marks (`zoom-in FRAME [x,y,w,h]`, `zoom-out FRAME`). The hold between those cues +is at least two seconds of real time, with no time compression on that span, so +the stored secret is readable. Magnification is relative to the published wide +shot and is at least 2x: `proof/glyph-height.py` measures `capture_width / +crop_width` on the hold. A missing rect on `zoom-in` means "measure it". + +The zoom ceiling still defaults to the capture width over the published width so a +1.33x hold is a crop. The hero's 2x secret hold is a tighter crop scaled back to +1920x1080 — a camera move of the one capture path, not a second recorder. The +stage runs on the take, before the cut, and keeps every frame and the recorded +rate, so the cadence gate still measures the capture's own cadence. A scene asks +for one by writing the cue file, or by setting `ZOOM_ARGS` in `scripts/demos/record-hd-demo.sh`. `--self-check` records a synthetic clip whose moving region is known and asserts the diff --git a/proof/docker/seed-demo.sh b/proof/docker/seed-demo.sh index 8bda0e9d8..e7be805d3 100755 --- a/proof/docker/seed-demo.sh +++ b/proof/docker/seed-demo.sh @@ -279,6 +279,8 @@ cat >"${DEMO}/ship-sim/tsconfig.json" <<'JSON' } JSON +cp /repo/proof/prompts/demo-hd.md "${DEMO}/ship-sim/TASK.md" + cat >"${DEMO}/ship-sim/SPEC.md" <<'MD' # Nebula Drift diff --git a/proof/glyph-height.py b/proof/glyph-height.py new file mode 100755 index 000000000..7fa56e344 --- /dev/null +++ b/proof/glyph-height.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Measure how much larger a held zoom makes a glyph than the published wide shot. + + .internal/glyph-height.py take.mp4 --cues take-cues.txt + +The secret line is readable when the crop is at most half the capture width: +relative scale is capture_width / crop_width, and the gate is 2.0. This is +geometry, not OCR: a 2x crop scaled to 1920 is twice the published wide-shot +glyph height. +""" +from __future__ import annotations + +import argparse +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "proof")) +import zoom # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("take", type=Path) + parser.add_argument("--cues", type=Path, required=True) + parser.add_argument("--fps", type=float, default=zoom.CUE_FPS) + parser.add_argument("--min-scale", type=float, default=zoom.MAGNIFY) + args = parser.parse_args() + cues = zoom.parse_cues(args.cues) + at, hold = zoom.cues_to_window(cues, fps=args.fps, ease=zoom.EASE) + rect = zoom.first_in_rect(cues) + width, height = zoom.size(args.take) + if rect is None: + with tempfile.TemporaryDirectory(prefix=".glyph-", dir=Path.cwd()) as scratch: + frames = zoom.sample(args.take, at - zoom.SEARCH_LEAD, zoom.SEARCH_LEAD + hold, Path(scratch)) + box = zoom.motion_box(frames) + if box is None: + print("glyph-height: nothing moved in the hold; no crop to measure", file=sys.stderr) + return 1 + measured = zoom.frame_rect( + box, + width=width, + height=height, + zoom=max(width / zoom.PUBLISH_WIDTH, args.min_scale), + pad=zoom.PAD, + ) + rect = (measured.x, measured.y, measured.w, measured.h) + scale = width / rect[2] + print(f"{args.take}: crop {rect[2]}x{rect[3]} in {width}px capture -> {scale:.2f}x vs wide shot") + if scale + 1e-9 < args.min_scale: + print(f"glyph-height: {scale:.2f}x is under the {args.min_scale:.1f}x floor", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/proof/hero-cut.py b/proof/hero-cut.py index fa7aa9539..1fed154a2 100755 --- a/proof/hero-cut.py +++ b/proof/hero-cut.py @@ -339,14 +339,23 @@ def cut( width: int, fps: int, crf: int = 22, + speed_badge: bool = False, ) -> None: inputs: list[str] = [] chains: list[str] = [] labels = "" for i, (lo, hi, speed) in enumerate(spans): inputs += ["-ss", f"{lo:.3f}", "-t", f"{hi - lo:.3f}", "-i", str(path)] + badge = "" + if speed_badge and speed >= 1.5: + label = f"{speed:g}×" + badge = ( + f",drawtext=text='{label}'" + ":x=w-th-80:y=48:fontsize=42:fontcolor=white" + ":borderw=3:bordercolor=black@0.7" + ) chains.append( - f"[{i}:v]setpts=PTS/{speed},scale={width}:-2:flags=lanczos,fps={fps},setpts=N/{fps}/TB[v{i}]" + f"[{i}:v]setpts=PTS/{speed},scale={width}:-2:flags=lanczos,fps={fps}{badge},setpts=N/{fps}/TB[v{i}]" ) labels += f"[v{i}]" graph = ";".join(chains) + f";{labels}concat=n={len(spans)}:v=1:a=0[v]" @@ -482,6 +491,7 @@ def main() -> int: parser.add_argument("--webp-fps", type=int, default=30) parser.add_argument("--webp-quality", type=int, default=62) parser.add_argument("--dry-run", action="store_true", help="print the windows, write nothing") + parser.add_argument("--speed-badge", action="store_true", help="draw the time-compression factor on sped-up spans") args = parser.parse_args() if (args.real_through_mark or args.real_from_mark) and not args.marks: @@ -554,7 +564,7 @@ def main() -> int: if args.dry_run: return 0 - cut(args.take, args.mp4, rated_spans, width=args.width, fps=args.fps, crf=args.crf) + cut(args.take, args.mp4, rated_spans, width=args.width, fps=args.fps, crf=args.crf, speed_badge=args.speed_badge) print(f"wrote {args.mp4} ({args.mp4.stat().st_size} bytes, {duration(args.mp4):.1f}s)") if args.webp: webp(args.mp4, args.webp, width=args.webp_width, fps=args.webp_fps, quality=args.webp_quality) diff --git a/proof/scenes/demo-hd.sh b/proof/scenes/demo-hd.sh index 3402115be..4d51d0c1f 100755 --- a/proof/scenes/demo-hd.sh +++ b/proof/scenes/demo-hd.sh @@ -28,10 +28,26 @@ else fi shot secret-stored -# ONE TASK PROMPT. Newlines are collapsed only for terminal entry; the authored -# prompt remains a static Markdown file and is never assembled in application code. -DEMO_PROMPT="$(tr '\n' ' ' [delay] # scene, or an image built before this -- and it keeps the delay that behaved best. KITTY_SOCKET="${KITTY_SOCKET:-unix:/tmp/kitty.sock}" +# XTEST fallback used when kitty remote control does not answer. Named so a +# missing binary fails as `_xdo: command not found` only if this function is +# deleted, not because the fallback was never bound. +_xdo() { xdotool "$@"; } + # WHICH PATH A RUN TOOK IS PART OF THE EVIDENCE. A silent fallback would look exactly # like a working fix -- green gate, doubled characters in the next take -- so the first # send says which one it is and the run's log carries it. diff --git a/proof/zoom.py b/proof/zoom.py index e8738d443..c7675d689 100755 --- a/proof/zoom.py +++ b/proof/zoom.py @@ -49,7 +49,13 @@ # the thing it is pointing at. PAD = 0.25 EASE = 0.5 -HOLD = 1.6 +HOLD = 2.0 +# How much larger a glyph is than in the published wide shot. 2.0 crops at most +# half the capture width and scales it to the published frame, which is an +# upscale of the crop and is the camera move the hero take needs: the secret +# line has to be readable at 1080p, not merely a few cells in a wide terminal. +MAGNIFY = 2.0 +CUE_FPS = 30 @dataclass(frozen=True) @@ -282,9 +288,13 @@ def zoom_into( pad: float, crf: int, report: bool = True, + magnify: float = MAGNIFY, + forced_rect: tuple[int, int, int, int] | None = None, ) -> Rect: width, height = size(take) ceiling = zoom if zoom is not None else width / PUBLISH_WIDTH + if magnify > 1.0: + ceiling = max(ceiling, magnify) if ceiling <= 1.0: raise ValueError(f"a zoom of {ceiling:.2f}x is not a zoom; the capture is {width} wide") # The search frames are an intermediate of this take, so they live beside the file @@ -295,9 +305,26 @@ def zoom_into( if len(frames) < 2: raise ValueError(f"{take}: {at:.1f}s is outside the recording") box = motion_box(frames) - if box is None: - raise ValueError(f"{take}: nothing changed around {at:.1f}s, so there is no region to zoom into") - rect = frame_rect(box, width=width, height=height, zoom=ceiling, pad=pad) + if forced_rect is not None: + rect = Rect(x=forced_rect[0], y=forced_rect[1], w=forced_rect[2], h=forced_rect[3]) + else: + if box is None: + raise ValueError(f"{take}: nothing changed around {at:.1f}s, so there is no region to zoom into") + rect = frame_rect(box, width=width, height=height, zoom=ceiling, pad=pad) + # Magnify is a camera floor, not a ceiling the region can talk us out of. + # A full-frame repaint would otherwise pin held at 1x and the secret line + # would stay a few cells on a 1080p landing page. + floor_w = _even(width / magnify) + floor_h = _even(height / magnify) + if rect.w > floor_w or rect.h > floor_h: + cx = rect.x + rect.w / 2 + cy = rect.y + rect.h / 2 + rect = Rect( + x=_clamp(round(cx - floor_w / 2), 0, width - floor_w), + y=_clamp(round(cy - floor_h / 2), 0, height - floor_h), + w=floor_w, + h=floor_h, + ) if report: print( f"{take}: {rect.w}x{rect.h} at {rect.x},{rect.y}" @@ -367,7 +394,7 @@ def self_check() -> int: check=True, ) out = root / "zoomed.mp4" - rect = zoom_into(take, out, at=1.5, hold=1.0, ease=0.4, zoom=None, pad=PAD, crf=18, report=False) + rect = zoom_into(take, out, at=1.5, hold=1.0, ease=0.4, zoom=None, pad=PAD, crf=18, report=False, magnify=1.0) if not ( rect.x <= block["x"] @@ -397,12 +424,85 @@ def self_check() -> int: if opening > before * 1.3: failures.append(f"the clip opens already zoomed ({opening:.4f} against {before:.4f})") + cues_path = root / "cues.txt" + cues_path.write_text("zoom-in 45\nzoom-out 150\n") + at, hold = cues_to_window(parse_cues(cues_path), fps=30, ease=0.5) + if abs(at - 1.5) > 1e-9: + failures.append(f"cue zoom-in 45 at 30fps should be 1.5s, got {at}") + if hold < HOLD: + failures.append(f"cue hold {hold} is under the 2s floor") + cues_path.write_text("zoom-in 30\nzoom-out 60\n") + try: + cues_to_window(parse_cues(cues_path), fps=30, ease=0.5) + failures.append("a 1s cue pair was accepted; the secret hold is 2s") + except ValueError: + pass for failure in failures: print(f"zoom self-check: {failure}", file=sys.stderr) print(f"zoom self-check: {'FAILED' if failures else 'ok'}") return 1 if failures else 0 + +@dataclass(frozen=True) +class Cue: + """One camera move. Frame numbers are in the take's capture rate.""" + + kind: str + frame: int + rect: tuple[int, int, int, int] | None = None + + +def parse_cues(path: Path) -> list[Cue]: + """Read `zoom-in FRAME [x,y,w,h]` / `zoom-out FRAME` rows.""" + cues: list[Cue] = [] + for raw in path.read_text().splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if parts[0] == "zoom-in": + if len(parts) not in (2, 3): + raise ValueError(f"{path}: bad zoom-in row: {raw}") + rect = None + if len(parts) == 3: + nums = [int(n) for n in parts[2].split(",")] + if len(nums) != 4: + raise ValueError(f"{path}: zoom-in rect must be x,y,w,h") + rect = (nums[0], nums[1], nums[2], nums[3]) + cues.append(Cue("in", int(parts[1]), rect)) + elif parts[0] == "zoom-out": + if len(parts) != 2: + raise ValueError(f"{path}: bad zoom-out row: {raw}") + cues.append(Cue("out", int(parts[1]))) + else: + raise ValueError(f"{path}: unknown cue {parts[0]!r}") + if not cues: + raise ValueError(f"{path}: no cues") + return cues + + +def cues_to_window(cues: list[Cue], *, fps: float, ease: float) -> tuple[float, float]: + """The first zoom-in / zoom-out pair, as take-seconds and hold length.""" + ins = [c for c in cues if c.kind == "in"] + outs = [c for c in cues if c.kind == "out"] + if len(ins) != 1 or len(outs) != 1: + raise ValueError("cues must name exactly one zoom-in and one zoom-out") + at = ins[0].frame / fps + out_at = outs[0].frame / fps + hold = out_at - at - 2 * ease + if hold < HOLD - 1e-9: + raise ValueError(f"cue hold is {hold:.2f}s; the secret must stay readable for {HOLD:.1f}s") + return at, hold + + +def first_in_rect(cues: list[Cue]) -> tuple[int, int, int, int] | None: + for cue in cues: + if cue.kind == "in": + return cue.rect + return None + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("take", type=Path, nargs="?", help="the recording to zoom") @@ -420,6 +520,14 @@ def main() -> int: parser.add_argument("--pad", type=float, default=PAD, help=f"region padding as a share of its size (default {PAD})") parser.add_argument("--crf", type=int, default=18, help="x264 quality of the intermediate") parser.add_argument("--self-check", action="store_true", help="prove the stage on a synthetic clip") + parser.add_argument("--cues", type=Path, help="camera cue file the scene emitted") + parser.add_argument("--fps", type=float, default=CUE_FPS, help="capture rate used to turn cue frames into seconds") + parser.add_argument( + "--magnify", + type=float, + default=MAGNIFY, + help=f"glyph size relative to the published wide shot (default {MAGNIFY})", + ) args = parser.parse_args() if args.self_check: @@ -432,22 +540,34 @@ def main() -> int: parser.error("take and out are required") if args.mark and not args.marks: parser.error("--mark requires --marks") - if (args.at is None) == (args.mark is None): - parser.error("pass exactly one of --at and --mark") + named = [args.at is not None, args.mark is not None, args.cues is not None] + if sum(named) != 1: + parser.error("pass exactly one of --at, --mark, and --cues") if args.hold <= 0 or args.ease <= 0: parser.error("--hold and --ease must be greater than zero") + if args.magnify < 2.0: + parser.error("--magnify must be at least 2 so the secret line is readable at 1080p") try: - at = args.at if args.at is not None else mark_time(args.marks, args.mark) + rect = None + if args.cues is not None: + cues = parse_cues(args.cues) + at, hold = cues_to_window(cues, fps=args.fps, ease=args.ease) + rect = first_in_rect(cues) + else: + at = args.at if args.at is not None else mark_time(args.marks, args.mark) + hold = args.hold zoom_into( args.take, args.out, at=at, - hold=args.hold, + hold=hold, ease=args.ease, zoom=args.zoom, pad=args.pad, crf=args.crf, + magnify=args.magnify, + forced_rect=rect, ) except ValueError as error: print(f"zoom.py: {error}", file=sys.stderr) diff --git a/scripts/demos/record-hd-demo.sh b/scripts/demos/record-hd-demo.sh index cd879bf76..c59537532 100755 --- a/scripts/demos/record-hd-demo.sh +++ b/scripts/demos/record-hd-demo.sh @@ -77,6 +77,7 @@ ZOOM_ARGS=() case "${SCENE}" in demo-hd) ASSET=assets/demo-hd.webp + ZOOM_ARGS=(--magnify 2.0) # The hero also ships whole. The task runs for many minutes and the landing # page gets a dense cut, so both are published and the cut can be checked # against the complete autonomous goal session. @@ -100,7 +101,7 @@ demo-hd) CUT_WIDTH=1920 WEBP_WIDTH=1920 CUT_ARGS=( - --speed 1.25 + --speed 6 --edge-speed 1.0 --real-through-mark agent-lanes --real-from-mark build-verified @@ -109,6 +110,7 @@ demo-hd) --crf 26 --still-keep 4 --still-min 4 + --speed-badge ) ;; todo-marathon) @@ -525,6 +527,10 @@ fi # every frame and the recorded rate, so the gate below still reads the capture's own cadence, # and the archived whole take stays as it was recorded. CUT_SOURCE="${WORK}/${SCENE}.mp4" +CUES="${WORK}/${SCENE}-cues.txt" +if [[ -f "${CUES}" ]]; then + ZOOM_ARGS+=(--cues "${CUES}") +fi if [[ ${#ZOOM_ARGS[@]} -gt 0 ]]; then ZOOM_SOURCE="${WORK}/${SCENE}-zoomed.mp4" if [[ -f "${MARKS}" && ! " ${ZOOM_ARGS[*]} " =~ " --marks " ]]; then @@ -534,6 +540,12 @@ if [[ ${#ZOOM_ARGS[@]} -gt 0 ]]; then echo "record-hd-demo.sh: the zoom stage found no region to hold; publishing nothing" >&2 exit 1 } + if [[ -f "${CUES}" ]]; then + python3 proof/glyph-height.py "${CUT_SOURCE}" --cues "${CUES}" || { + echo "record-hd-demo.sh: the secret hold is not 2x the wide shot; publishing nothing" >&2 + exit 1 + } + fi CUT_SOURCE="${ZOOM_SOURCE}" fi # EVERY run cuts into the work directory, and a real one copies out of it afterwards. The From 9914c312fce12679c8a041df381f33e1a96f9b51 Mon Sep 17 00:00:00 2001 From: santhreal <64453045+santhreal@users.noreply.github.com> Date: Sun, 23 Aug 2026 17:40:19 -0700 Subject: [PATCH 02/15] feat(proof): 60fps hero camera that follows /secret then pans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous cut zoomed a stored-secret still and published at 30 (then 14) fps. The scene now types /secret in the composer, cues a 2× crop on that line, pans right as from-env is entered, and eases out before the short TASK.md prompt. Capture, cut, and cadence for demo-hd are 60 fps / 17 ms. Kitty in the recorder image needs a machine-id and a session bus after Xvfb or the window never appears. verify-scene counts type_visible as typed text. --- docs/handbook/src/foundations/verification.md | 10 +- proof/docker/xsession.sh | 25 +++ proof/scenes/demo-hd.sh | 77 +++++-- proof/zoom.py | 202 ++++++++++++++---- scripts/demos/record-hd-demo.sh | 26 ++- scripts/verify-scene.test.ts | 8 + scripts/verify-scene.ts | 2 +- 7 files changed, 279 insertions(+), 71 deletions(-) diff --git a/docs/handbook/src/foundations/verification.md b/docs/handbook/src/foundations/verification.md index 3ae9264fb..0b5614683 100644 --- a/docs/handbook/src/foundations/verification.md +++ b/docs/handbook/src/foundations/verification.md @@ -37,8 +37,8 @@ The recorder refuses to publish a clip whose cadence is not the one it captured. criteria, both from `--expect-ms`: ```text -typical frame 33 ms, +/-1 (34 ms alternates at 30 fps) -moving average within 10% of 30 fps, held stills set aside +typical frame capture interval +/-1 ms (17 ms at 60 fps, 33/34 ms at 30 fps) +moving average within 10% of the capture rate, held stills set aside ``` The first catches a resample, where every frame was rewritten. The second catches a @@ -53,14 +53,14 @@ python3 proof/webp-cadence.py assets/demo-hd.webp --expect-ms 33 ### Real interactive sessions -The HD recorder starts Xvfb, picom, and kitty inside the recorder container. It drives the shipped CLI with real keyboard and pointer events and records the private display at 30 frames per second. +The HD recorder starts Xvfb, picom, and kitty inside the recorder container. It drives the shipped CLI with real keyboard and pointer events and records the private display at the scene rate (60 fps for the hero take, 30 fps for the rest). The landing-page terminal uses: ```text terminal kitty font JetBrains Mono 21 -canvas 2560x1440 at 30 fps +canvas 2560x1440 at 60 fps (hero) or 30 fps window inset 128 px background #171b22 foreground #d3dae6 @@ -189,7 +189,7 @@ holds the bounding box of what changed there, padded and clamped inside the fram the source aspect ratio. A moment with nothing moving in it produces no file. The hero take drives the same stage from a cue file the scene writes next to its -marks (`zoom-in FRAME [x,y,w,h]`, `zoom-out FRAME`). The hold between those cues +marks (`zoom-in FRAME [x,y,w,h]`, `pan FRAME x,y,w,h`, `zoom-out FRAME`). The hold between those cues is at least two seconds of real time, with no time compression on that span, so the stored secret is readable. Magnification is relative to the published wide shot and is at least 2x: `proof/glyph-height.py` measures `capture_width / diff --git a/proof/docker/xsession.sh b/proof/docker/xsession.sh index 0f0f3c677..b143c5cbf 100755 --- a/proof/docker/xsession.sh +++ b/proof/docker/xsession.sh @@ -19,6 +19,19 @@ FPS="${SCENE_FPS:-30}" OUT="/out" mkdir -p "${OUT}" +# kitty/glfw refuse to open a window without a machine-id. Some recorder images +# ship without /etc/machine-id, and the first thing the operator sees is +# "no terminal window with a geometry appeared" plus a dbus error in the log. +if [ ! -s /etc/machine-id ]; then + if command -v dbus-uuidgen >/dev/null 2>&1; then + dbus-uuidgen --ensure + else + head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n' >/etc/machine-id + mkdir -p /var/lib/dbus + cp /etc/machine-id /var/lib/dbus/machine-id + fi +fi + # COMPOSITE and RENDER are what a compositor needs; Xvfb offers them only when # they are asked for, and picom without them starts, stays alive, and never # claims the manager selection, which reads exactly like a theme that did not @@ -33,6 +46,18 @@ for _ in $(seq 1 50); do done xdpyinfo -display "${DISPLAY}" >/dev/null +# Session bus after the display exists. dbus-launch (which kitty/glfw will +# spawn if this is missing) dies without $DISPLAY, and the window never appears. +export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/xdg-runtime}" +mkdir -p "${XDG_RUNTIME_DIR}" +chmod 700 "${XDG_RUNTIME_DIR}" || true +if [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && command -v dbus-daemon >/dev/null 2>&1; then + bus="${XDG_RUNTIME_DIR}/bus" + rm -f "${bus}" + dbus-daemon --session --address="unix:path=${bus}" --fork + export DBUS_SESSION_BUS_ADDRESS="unix:path=${bus}" +fi + # AUTOREPEAT IS WHY TYPED COMMANDS DOUBLED THEIR CHARACTERS. xdotool synthesises a # press and a release per character; when the client is repainting hard -- the composer # re-renders the whole prompt plus a completion popup on every keystroke -- the release diff --git a/proof/scenes/demo-hd.sh b/proof/scenes/demo-hd.sh index 4d51d0c1f..cb5954240 100755 --- a/proof/scenes/demo-hd.sh +++ b/proof/scenes/demo-hd.sh @@ -18,8 +18,61 @@ settle 16 screen_has "model:" || screen_has "demo" || screen_has "veyyon" || MISSED="${MISSED:-} idle" shot idle -# Operator setup: store the key without typing it into the transcript. -slash "/secret from-env RELEASE_SIGNATURE release-signature" +# Camera helpers for the directed secret take. Cues are capture frames at SCENE_FPS. +# zoom-in is the peak on `/secret` (left composer); pan is the value sliding right; +# zoom-out is the frame the camera is wide again. +_even() { echo $(( ($1 / 2) * 2 )); } +now_frame() { + local ms=$(($(date +%s%3N) - SCENE_T0)) + echo $((ms * ${SCENE_FPS:-60} / 1000)) +} +composer_crop() { + local side="$1" + local canvas_w="${SCENE_WIDTH:-2560}" + local canvas_h="${SCENE_HEIGHT:-1440}" + local crop_w crop_h x y max_x max_y + crop_w=$(_even $((canvas_w / 2))) + crop_h=$(_even $((canvas_h / 2))) + y=$((WIN_Y + WIN_H - crop_h + CELL_H)) + max_y=$((canvas_h - crop_h)) + [ "$y" -lt 0 ] && y=0 + [ "$y" -gt "$max_y" ] && y=$max_y + if [ "$side" = right ]; then + x=$((WIN_X + WIN_W - crop_w)) + else + x=$WIN_X + fi + max_x=$((canvas_w - crop_w)) + [ "$x" -lt 0 ] && x=0 + [ "$x" -gt "$max_x" ] && x=$max_x + echo "$(_even "$x"),$(_even "$y"),${crop_w},${crop_h}" +} +type_visible() { + local s="$1" + local i + for ((i = 0; i < ${#s}; i++)); do + t "${s:i:1}" + pause 0.05 + done +} +emit_cue() { + printf '%s\n' "$1" >>"${SCENE_OUT}/${SCENE_NAME}-cues.txt" +} + +# Operator setup: type `/secret` in the composer so the camera has something to follow. +# The key itself stays in the environment; from-env never echoes it. +: >"${SCENE_OUT}/${SCENE_NAME}-cues.txt" +clear_composer +pause 0.2 +emit_cue "zoom-in $(now_frame) $(composer_crop left)" +type_visible "/secret " +pause 0.25 +emit_cue "pan $(now_frame) $(composer_crop right)" +type_visible "from-env RELEASE_SIGNATURE release-signature" +pause 0.7 +k Escape +pause 0.3 +k Return settle 8 if screen_has "release-signature" || screen_has "Stored" || screen_has "secret"; then echo "scene: release signing secret stored" >&2 @@ -27,22 +80,10 @@ else MISSED="${MISSED:-} secret-stored" fi shot secret-stored - -# Hold the stored-secret confirmation long enough to read at 1080p once the -# camera has moved in. The cue file is the camera: zoom-in on this mark, zoom-out -# as the short prompt is typed. The long task lives on disk as TASK.md so the -# composer shows a line a viewer can actually read. -settle 3 -secret_t="$(awk -F '\t' '$1=="secret-stored"{print $2; exit}' "${SCENE_OUT}/${SCENE_NAME}-marks.tsv")" -python3 - "${SCENE_OUT}/${SCENE_NAME}-cues.txt" "${secret_t}" <<'CUE' -from pathlib import Path -import sys -out, secret_t = Path(sys.argv[1]), float(sys.argv[2]) -fps = 30 -start = int(round(secret_t * fps)) -end = start + int(round(3.0 * fps)) -out.write_text(f"zoom-in {start}\nzoom-out {end}\n") -CUE +# Hold the confirmation in the right-hand crop, then ease out before the short prompt. +settle 2 +emit_cue "zoom-out $(now_frame)" +pause 0.6 # The full contract lives at proof/prompts/demo-hd.md and is seeded as TASK.md. # Named here so verify-scene.ts still traces every guard to that file. # shellcheck disable=SC2034 diff --git a/proof/zoom.py b/proof/zoom.py index c7675d689..a008b8d73 100755 --- a/proof/zoom.py +++ b/proof/zoom.py @@ -1,22 +1,20 @@ #!/usr/bin/env python3 -"""Ease a recording into one region of the screen and back out. +"""Ease a recording into a region, pan, and ease back out. proof/zoom.py take.mp4 zoomed.mp4 --at 184.5 - proof/zoom.py take.mp4 zoomed.mp4 --marks take-marks.tsv --mark todo-board + proof/zoom.py take.mp4 zoomed.mp4 --cues take-cues.txt --fps 60 -A landing-page clip is 1920 wide and the recorder captures 2560, so a 1920-wide -crop out of the take is a 1.33x zoom with no upscale: the extra 640 columns of -capture width are the whole zoom budget, and `--zoom` above that resamples text -the surface never drew. +A landing-page clip is 1920 wide and the recorder captures 2560. The hero camera +is a 2x crop of the composer: zoom in on `/secret`, pan right as the rest of the +line is typed, hold readable, zoom out. Cue rows name that path in capture frames: -The region is measured, not typed in. A rect on a 2560x1440 screen would have to -be found by hand for every scene and would move the next time a block changed -height, so the stage diffs the frames around the moment and zooms to the bounding -box of what changed there. A moment with nothing moving in it produces no rect and -no file. + zoom-in FRAME x,y,w,h + pan FRAME x,y,w,h + zoom-out FRAME -The stage runs on the take, before the cut, so it changes no timing: same frame -count, same rate, and the cadence gate downstream still reads 33 ms. +A missing rect on zoom-in still means "measure motion". The stage runs on the +take before the cut, so frame count and rate are unchanged and the cadence gate +reads the capture interval. """ from __future__ import annotations @@ -55,7 +53,9 @@ # upscale of the crop and is the camera move the hero take needs: the secret # line has to be readable at 1080p, not merely a few cells in a wide terminal. MAGNIFY = 2.0 -CUE_FPS = 30 +CUE_FPS = 60 +# Seconds of sideways travel between the zoom-in crop and the pan crop. +PAN_EASE = 1.0 @dataclass(frozen=True) @@ -234,6 +234,65 @@ def zoom_filter(rect: Rect, *, width: int, height: int, at: float, hold: float, return f"zoompan=z='{factor}':x='{pan_x}':y='{pan_y}':d=1:s={width}x{height}:fps={fps}" +def path_filter( + left: Rect, + right: Rect, + *, + width: int, + height: int, + t_in: float, + t_pan: float, + t_out: float, + ease: float, + pan_ease: float, + fps: str, +) -> str: + """Zoom into `left`, pan to `right`, hold, zoom out. + + Registers (evaluated in `z`, read in `x`/`y`): + 1 zoom progress 0→1→0 (raw) + 11 smoothstep of 1 + 2 pan progress 0→1 (raw) + 22 smoothstep of 2 + """ + t1 = t_in + ease + t2 = max(t_pan, t1) + t3 = t2 + pan_ease + t4 = max(t_out, t3) + t5 = t4 + ease + zoom = width / left.w + zprog = ( + f"if(lt(time,{t_in:.3f}),0," + f"if(lt(time,{t1:.3f}),(time-{t_in:.3f})/{ease:.3f}," + f"if(lt(time,{t4:.3f}),1," + f"if(lt(time,{t5:.3f}),1-(time-{t4:.3f})/{ease:.3f},0))))" + ) + pprog = ( + f"if(lt(time,{t2:.3f}),0," + f"if(lt(time,{t3:.3f}),(time-{t2:.3f})/{pan_ease:.3f},1))" + ) + factor = ( + f"st(1,{zprog});st(11,ld(1)*ld(1)*(3-2*ld(1)));" + f"st(2,{pprog});st(22,ld(2)*ld(2)*(3-2*ld(2)));" + f"1+{zoom - 1:.6f}*ld(11)" + ) + cx_full = width / 2 + cy_full = height / 2 + cx = ( + f"if(lt(time,{t2:.3f}),{cx_full:.1f}+({left.cx:.1f}-{cx_full:.1f})*ld(11)," + f"if(lt(time,{t4:.3f}),{left.cx:.1f}+({right.cx:.1f}-{left.cx:.1f})*ld(22)," + f"{right.cx:.1f}+({cx_full:.1f}-{right.cx:.1f})*(1-ld(11))))" + ) + cy = ( + f"if(lt(time,{t2:.3f}),{cy_full:.1f}+({left.cy:.1f}-{cy_full:.1f})*ld(11)," + f"if(lt(time,{t4:.3f}),{left.cy:.1f}+({right.cy:.1f}-{left.cy:.1f})*ld(22)," + f"{right.cy:.1f}+({cy_full:.1f}-{right.cy:.1f})*(1-ld(11))))" + ) + pan_x = f"clip(({cx})-(iw/zoom)/2,0,iw-iw/zoom)" + pan_y = f"clip(({cy})-(ih/zoom)/2,0,ih-ih/zoom)" + return f"zoompan=z='{factor}':x='{pan_x}':y='{pan_y}':d=1:s={width}x{height}:fps={fps}" + + def render(take: Path, out: Path, expression: str, *, crf: int) -> None: """Re-encode the take through the zoom, keeping every frame it recorded. @@ -277,6 +336,22 @@ def mark_time(marks: Path, name: str) -> float: raise ValueError(f"{marks}: no mark named '{name}'") +def _fit_magnify(rect: Rect, *, width: int, height: int, magnify: float) -> Rect: + """Pin a crop to at least `magnify` so a full-frame box cannot collapse the camera to 1x.""" + floor_w = _even(width / magnify) + floor_h = _even(height / magnify) + if rect.w <= floor_w and rect.h <= floor_h: + return rect + cx = rect.x + rect.w / 2 + cy = rect.y + rect.h / 2 + return Rect( + x=_clamp(round(cx - floor_w / 2), 0, width - floor_w), + y=_clamp(round(cy - floor_h / 2), 0, height - floor_h), + w=floor_w, + h=floor_h, + ) + + def zoom_into( take: Path, out: Path, @@ -290,6 +365,8 @@ def zoom_into( report: bool = True, magnify: float = MAGNIFY, forced_rect: tuple[int, int, int, int] | None = None, + pan_rect: tuple[int, int, int, int] | None = None, + at_pan: float | None = None, ) -> Rect: width, height = size(take) ceiling = zoom if zoom is not None else width / PUBLISH_WIDTH @@ -300,42 +377,54 @@ def zoom_into( # The search frames are an intermediate of this take, so they live beside the file # being written rather than in a system temp directory a run does not own. out.parent.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(prefix=".zoom-", dir=out.parent) as scratch: - frames = sample(take, at - SEARCH_LEAD, SEARCH_LEAD + hold, Path(scratch)) - if len(frames) < 2: - raise ValueError(f"{take}: {at:.1f}s is outside the recording") - box = motion_box(frames) + box = None + if forced_rect is None: + with tempfile.TemporaryDirectory(prefix=".zoom-", dir=out.parent) as scratch: + frames = sample(take, at - SEARCH_LEAD, SEARCH_LEAD + hold, Path(scratch)) + if len(frames) < 2: + raise ValueError(f"{take}: {at:.1f}s is outside the recording") + box = motion_box(frames) if forced_rect is not None: rect = Rect(x=forced_rect[0], y=forced_rect[1], w=forced_rect[2], h=forced_rect[3]) else: if box is None: raise ValueError(f"{take}: nothing changed around {at:.1f}s, so there is no region to zoom into") rect = frame_rect(box, width=width, height=height, zoom=ceiling, pad=pad) - # Magnify is a camera floor, not a ceiling the region can talk us out of. - # A full-frame repaint would otherwise pin held at 1x and the secret line - # would stay a few cells on a 1080p landing page. - floor_w = _even(width / magnify) - floor_h = _even(height / magnify) - if rect.w > floor_w or rect.h > floor_h: - cx = rect.x + rect.w / 2 - cy = rect.y + rect.h / 2 - rect = Rect( - x=_clamp(round(cx - floor_w / 2), 0, width - floor_w), - y=_clamp(round(cy - floor_h / 2), 0, height - floor_h), - w=floor_w, - h=floor_h, + rect = _fit_magnify(rect, width=width, height=height, magnify=magnify) + right = None + if pan_rect is not None: + right = _fit_magnify( + Rect(x=pan_rect[0], y=pan_rect[1], w=pan_rect[2], h=pan_rect[3]), + width=width, + height=height, + magnify=magnify, ) + # The pan keeps the same crop size as the zoom-in so the camera slides, it does not re-crop. + right = Rect(x=right.x, y=right.y, w=rect.w, h=rect.h) if report: + move = f" pan {right.x},{right.y}" if right is not None else "" print( - f"{take}: {rect.w}x{rect.h} at {rect.x},{rect.y}" + f"{take}: {rect.w}x{rect.h} at {rect.x},{rect.y}{move}" f" -> {width / rect.w:.2f}x held {hold:.1f}s from {at:.1f}s" ) - render( - take, - out, - zoom_filter(rect, width=width, height=height, at=at, hold=hold, ease=ease, fps=rate(take)), - crf=crf, - ) + if right is not None: + t_out = at + hold + t_pan = at_pan if at_pan is not None else at + expression = path_filter( + rect, + right, + width=width, + height=height, + t_in=max(at - ease, 0.0), + t_pan=t_pan, + t_out=t_out, + ease=ease, + pan_ease=PAN_EASE, + fps=rate(take), + ) + else: + expression = zoom_filter(rect, width=width, height=height, at=at, hold=hold, ease=ease, fps=rate(take)) + render(take, out, expression, crf=crf) return rect @@ -437,6 +526,15 @@ def self_check() -> int: failures.append("a 1s cue pair was accepted; the secret hold is 2s") except ValueError: pass + cues_path.write_text("zoom-in 60 0,720,1280,720\npan 120 1280,720,1280,720\nzoom-out 240\n") + path_cues = parse_cues(cues_path) + at, hold = cues_to_window(path_cues, fps=60, ease=0.5) + if abs(at - 1.0) > 1e-9: + failures.append(f"60fps zoom-in 60 should be 1.0s, got {at}") + if first_pan(path_cues) is None or first_pan(path_cues).rect != (1280, 720, 1280, 720): + failures.append("pan cue was not parsed") + if hold < HOLD: + failures.append(f"pan path hold {hold} is under the 2s floor") for failure in failures: print(f"zoom self-check: {failure}", file=sys.stderr) print(f"zoom self-check: {'FAILED' if failures else 'ok'}") @@ -461,16 +559,17 @@ def parse_cues(path: Path) -> list[Cue]: if not line or line.startswith("#"): continue parts = line.split() - if parts[0] == "zoom-in": + if parts[0] in ("zoom-in", "pan"): if len(parts) not in (2, 3): - raise ValueError(f"{path}: bad zoom-in row: {raw}") + raise ValueError(f"{path}: bad {parts[0]} row: {raw}") rect = None if len(parts) == 3: nums = [int(n) for n in parts[2].split(",")] if len(nums) != 4: - raise ValueError(f"{path}: zoom-in rect must be x,y,w,h") + raise ValueError(f"{path}: {parts[0]} rect must be x,y,w,h") rect = (nums[0], nums[1], nums[2], nums[3]) - cues.append(Cue("in", int(parts[1]), rect)) + kind = "in" if parts[0] == "zoom-in" else "pan" + cues.append(Cue(kind, int(parts[1]), rect)) elif parts[0] == "zoom-out": if len(parts) != 2: raise ValueError(f"{path}: bad zoom-out row: {raw}") @@ -503,6 +602,13 @@ def first_in_rect(cues: list[Cue]) -> tuple[int, int, int, int] | None: return None +def first_pan(cues: list[Cue]) -> Cue | None: + for cue in cues: + if cue.kind == "pan": + return cue + return None + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("take", type=Path, nargs="?", help="the recording to zoom") @@ -550,10 +656,18 @@ def main() -> int: try: rect = None + pan_rect = None + at_pan = None if args.cues is not None: cues = parse_cues(args.cues) at, hold = cues_to_window(cues, fps=args.fps, ease=args.ease) rect = first_in_rect(cues) + pan = first_pan(cues) + if pan is not None: + if pan.rect is None: + raise ValueError("a pan cue needs an x,y,w,h crop") + pan_rect = pan.rect + at_pan = pan.frame / args.fps else: at = args.at if args.at is not None else mark_time(args.marks, args.mark) hold = args.hold @@ -568,6 +682,8 @@ def main() -> int: crf=args.crf, magnify=args.magnify, forced_rect=rect, + pan_rect=pan_rect, + at_pan=at_pan, ) except ValueError as error: print(f"zoom.py: {error}", file=sys.stderr) diff --git a/scripts/demos/record-hd-demo.sh b/scripts/demos/record-hd-demo.sh index c59537532..9b002e7e8 100755 --- a/scripts/demos/record-hd-demo.sh +++ b/scripts/demos/record-hd-demo.sh @@ -78,6 +78,8 @@ case "${SCENE}" in demo-hd) ASSET=assets/demo-hd.webp ZOOM_ARGS=(--magnify 2.0) + SCENE_FPS=60 + CADENCE_MS=17 # The hero also ships whole. The task runs for many minutes and the landing # page gets a dense cut, so both are published and the cut can be checked # against the complete autonomous goal session. @@ -111,6 +113,8 @@ demo-hd) --still-keep 4 --still-min 4 --speed-badge + --fps 60 + --webp-fps 60 ) ;; todo-marathon) @@ -329,6 +333,16 @@ else exit 2 fi +# The coding agent imports a gitignored html bundle at parse time. A worktree +# without it opens the terminal, then the CLI dies, and the recorder reports +# that no window ever appeared. +if [[ ! -f packages/coding-agent/src/export/html/tool-views.generated.js ]]; then + "${BUN}" --cwd=packages/collab-web run gen:tool-views +fi +if [[ ! -f packages/natives/native/veyyon_natives.linux-x64-modern.node && ! -f packages/natives/native/veyyon_natives.linux-x64-baseline.node ]]; then + "${BUN}" --cwd=packages/natives run ensure +fi + # WHAT A MARKS FILE IS FOR HERE. The scene appends one row per frame that landed, so it is an # independent record of what the take captured. Both publish paths below copy the PNGs that # exist and nothing else, which means a shot that never landed does not fail a run: it leaves @@ -428,6 +442,7 @@ PROOF_LLM_BASE_URL="${PROOF_LLM_BASE_URL}" \ SCENE_COMMAND="${SCENE_CMD}" \ SCENE_THEME=night \ SCENE_WIDTH=2560 \ + SCENE_FPS="${SCENE_FPS:-30}" \ SCENE_HEIGHT=1440 \ SCENE_MARGIN=128 \ SCENE_FONT_SIZE=21 \ @@ -536,12 +551,15 @@ if [[ ${#ZOOM_ARGS[@]} -gt 0 ]]; then if [[ -f "${MARKS}" && ! " ${ZOOM_ARGS[*]} " =~ " --marks " ]]; then ZOOM_ARGS+=(--marks "${MARKS}") fi + if [[ ! " ${ZOOM_ARGS[*]} " =~ " --fps " ]]; then + ZOOM_ARGS+=(--fps "${SCENE_FPS:-30}") + fi python3 proof/zoom.py "${CUT_SOURCE}" "${ZOOM_SOURCE}" "${ZOOM_ARGS[@]}" || { echo "record-hd-demo.sh: the zoom stage found no region to hold; publishing nothing" >&2 exit 1 } if [[ -f "${CUES}" ]]; then - python3 proof/glyph-height.py "${CUT_SOURCE}" --cues "${CUES}" || { + python3 proof/glyph-height.py "${CUT_SOURCE}" --cues "${CUES}" --fps "${SCENE_FPS:-30}" || { echo "record-hd-demo.sh: the secret hold is not 2x the wide shot; publishing nothing" >&2 exit 1 } @@ -558,8 +576,8 @@ python3 proof/hero-cut.py "${CUT_SOURCE}" \ --width "${CUT_WIDTH:-2560}" --webp-width "${WEBP_WIDTH:-1920}" "${CUT_ARGS[@]}" # THE CADENCE IS PART OF THE PUBLISH CONTRACT, not a thing to notice afterwards. Both -# display servers record at 30 fps, so the typical frame of anything published from a take -# holds 33ms. The hero shipped at a 7.7 fps average because the path resampled it twice and +# display servers record at SCENE_FPS (30, 60 for the hero), so the typical frame of anything +# published from a take holds that interval (33ms or 17ms). The hero shipped at a 7.7 fps average because the path resampled it twice and # nothing here was looking: it read as a laggy product rather than as a resampled file. # # The gate then passed a take that averaged 14.2 fps, because it read only the most common @@ -567,7 +585,7 @@ python3 proof/hero-cut.py "${CUT_SOURCE}" \ # intervals. It now also gates the MOVING portion of the clip against the capture rate, with # held still screens named and set aside, so a file that is mostly slower than its most # common frame cannot pass. `--expect-ms` supplies both criteria. -python3 proof/webp-cadence.py "${CUT_WEBP}" --expect-ms 33 || { +python3 proof/webp-cadence.py "${CUT_WEBP}" --expect-ms "${CADENCE_MS:-33}" || { echo "record-hd-demo.sh: refusing to publish a clip that is not the cadence the recorder captured" >&2 exit 1 } diff --git a/scripts/verify-scene.test.ts b/scripts/verify-scene.test.ts index f681fa622..d434458a9 100644 --- a/scripts/verify-scene.test.ts +++ b/scripts/verify-scene.test.ts @@ -71,6 +71,14 @@ describe("a scene guard has to resolve to something that produces it", () => { ).toEqual([]); }); + it("accepts a needle typed visibly, character by character", () => { + expect( + problems({ + scene: 'type_visible "from-env RELEASE_SIGNATURE release-signature"\nscreen_has "release-signature"\n', + }), + ).toEqual([]); + }); + it("refuses a needle that only exists in the guard that waits for it", () => { // The scene file is not a source for itself, or every stale guard would prove itself by // being written down. This is the shape both hero-scene defects had. diff --git a/scripts/verify-scene.ts b/scripts/verify-scene.ts index 13ee21ea2..188ee3ca7 100644 --- a/scripts/verify-scene.ts +++ b/scripts/verify-scene.ts @@ -154,7 +154,7 @@ function verifyMissedIsFatal(scene: string, findings: Finding[], name: string): */ function typedByScene(scene: string): string { const typed: string[] = []; - for (const match of scene.matchAll(/(?:submit|slash|type|type_line)\s+"([^"]*)"/g)) { + for (const match of scene.matchAll(/(?:submit|slash|type|type_line|type_visible)\s+"([^"]*)"/g)) { typed.push(match[1]); } return typed.join("\n"); From 31ace21e870e6f323f221aa0c879820431782211 Mon Sep 17 00:00:00 2001 From: santhreal <64453045+santhreal@users.noreply.github.com> Date: Sun, 23 Aug 2026 20:40:23 -0700 Subject: [PATCH 03/15] fix(proof): stop the recorder capturing three frames a second A hero take came out at 2560x1440, 60/1 CFR, 7415 frames, encoded without dropping anything, and stuttered. Measured with mpdecimate it carried 385 unique frames: a flat 3.3 per second through typing, streaming and idle alike. The cause is --blur-background in the picom chrome. A translucent window makes the compositor re-blur everything behind it every frame, on the CPU through xrender, across the whole 2304x1184 inset. Measured in the recorder image at 2560x1440, one identical payload, six seconds per arm: no compositor 171 unique / 359 grabbed 28 fps opaque + blur 89 unique / 337 grabbed 14 fps 0.72 opacity, no blur 69 unique / 305 grabbed 11 fps 0.72 opacity + blur (the default) 14 unique / 69 grabbed 2 fps The middle column is the tell. With the blur on ffmpeg could grab 69 of 360 frames: the X server had nothing left to answer a screen capture with, so no encoder setting and no render-loop change downstream could recover frames that were never drawn. Re-measured unloaded, the same arms read 49 fps against 2. Blur now stays off unless SCENE_CHROME_BLUR=1 asks for it, which belongs on a still. Rounding, opacity and shadow are unchanged, so the treatment survives. Every signal available at the time said the take was fine, which is the real defect: the pipeline never measured whether the picture moved. proof/motion-gate.sh counts unique frames per second and fails a take below a floor, so a stuttering capture is loud instead of silent. The recorder runs it on every take. The sandbox guest gains ffmpeg, without which the suite pinning the gate cannot run at all. --- proof/a-capture-is-judged-on-motion.test.ts | 211 ++++++++++++++++++++ proof/docker/record-x11.sh | 25 +++ proof/docker/xsession.sh | 50 ++++- proof/motion-gate.sh | 68 +++++++ scripts/test-sandbox/guest/Dockerfile | 11 +- 5 files changed, 359 insertions(+), 6 deletions(-) create mode 100644 proof/a-capture-is-judged-on-motion.test.ts create mode 100755 proof/motion-gate.sh diff --git a/proof/a-capture-is-judged-on-motion.test.ts b/proof/a-capture-is-judged-on-motion.test.ts new file mode 100644 index 000000000..3f4d25efa --- /dev/null +++ b/proof/a-capture-is-judged-on-motion.test.ts @@ -0,0 +1,211 @@ +/** + * WHY THIS SUITE EXISTS + * + * A hero take shipped at 2560x1440, 60/1 CFR, 7415 frames, 123s, encoded without + * dropping anything — and it stuttered. Measured with mpdecimate it carried 385 + * unique frames: a flat 3.3 per second through typing, streaming and idle alike. + * The cause was `--blur-background` in the recorder's picom chrome. A translucent + * window makes the compositor re-blur everything behind it every frame, on the CPU + * through xrender, over the whole 2304x1184 inset. That saturates the X server: in + * the recorder image, one identical counter, six seconds per arm, the blurred arm + * captured 14 unique frames of 69 GRABBED, against 296 of 359 with no compositor. + * ffmpeg could not even sample the display, so no encoder setting and no + * render-loop change downstream could recover frames that were never drawn. + * + * Every signal available at the time said the take was fine. `ffprobe` reported + * 60/1 and 7415 frames; the encoder log was clean; the scene guards all landed. + * That is the defect: the pipeline had no measurement of whether the picture + * MOVED, so a capture could be perfect by every recorded number and unwatchable. + * + * THE CLASS THIS CLOSES. Not "blur was on once": any change that starves the + * capture of drawn frames — a compositor effect, a heavier backdrop, a slower + * terminal, a smaller render budget, a future GPU-less host — now fails at the + * gate instead of shipping. The gate measures the output rather than inspecting + * settings, so it catches causes nobody has thought of yet, which is the whole + * reason it is not a check on the value of a flag. + * + * WHAT IT DOES NOT CATCH. Motion fps is a rate, not a distribution: a take that + * is smooth for a minute and frozen for ten seconds can clear a floor that a + * uniformly mediocre take fails. It also cannot distinguish a scene that is + * legitimately still from a pipeline that is stuck, which is why the floor sits + * far below the capture rate and why SCENE_MOTION_FLOOR exists. And it says + * nothing about how the frames LOOK — a smooth capture of the wrong colours + * passes here. + */ +import { describe, expect, it } from "bun:test"; +import { execFile } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import * as path from "node:path"; +import { promisify } from "node:util"; + +const run = promisify(execFile); +const REPO_ROOT = path.join(import.meta.dirname, ".."); +const GATE = path.join(REPO_ROOT, "proof", "motion-gate.sh"); + +/** Outcome of running the gate: exit code plus what it told the operator. */ +interface GateResult { + code: number; + output: string; +} + +/** + * A synthetic take. `changesPerSecond` is how often the picture actually + * changes; `containerFps` is what the file claims. The two are independent, + * which is precisely the confusion the gate exists to resolve. + */ +interface Take { + changesPerSecond: number; + containerFps: number; + seconds: number; +} + +/** + * Render a take whose content changes at a chosen rate while the container + * carries a chosen frame rate. `-r` after the source resamples: a 2 fps source + * written at 60 fps produces genuine 60 fps CFR in which each picture repeats 30 + * times, which is byte-for-byte the shape of the take that shipped. + */ +async function renderTake(dir: string, name: string, take: Take): Promise { + const file = path.join(dir, `${name}.mp4`); + await run("ffmpeg", [ + "-loglevel", "error", "-y", + "-f", "lavfi", + "-i", `testsrc=size=640x360:rate=${take.changesPerSecond}:duration=${take.seconds}`, + "-c:v", "libx264", "-preset", "ultrafast", "-crf", "18", + "-pix_fmt", "yuv420p", + "-r", String(take.containerFps), + file, + ]); + return file; +} + +async function gate(video: string, floor?: number): Promise { + const args = floor === undefined ? [GATE, video] : [GATE, video, String(floor)]; + try { + const { stdout, stderr } = await run("bash", args); + return { code: 0, output: `${stdout}${stderr}` }; + } catch (error: unknown) { + // execFile rejects with an Error carrying the child's code and streams. + const failure = + error && typeof error === "object" + ? (error as { code?: number; stdout?: string; stderr?: string }) + : {}; + return { + code: typeof failure.code === "number" ? failure.code : 1, + output: `${failure.stdout ?? ""}${failure.stderr ?? ""}`, + }; + } +} + +describe("a capture is judged on motion, not on what the container claims", () => { + it("fails the take that shipped: 60 fps CFR carrying ~3 changes a second", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "motion-gate-")); + try { + // The shape of the defect, reproduced: the container is a true 60 fps + // CFR file and every frame-rate number reads correct. + const video = await renderTake(dir, "stuttering", { + changesPerSecond: 3, + containerFps: 60, + seconds: 6, + }); + + const claimed = await run("ffprobe", [ + "-v", "quiet", "-select_streams", "v:0", + "-show_entries", "stream=r_frame_rate", "-of", "csv=p=0", video, + ]); + expect(claimed.stdout.trim()).toBe("60/1"); + + const result = await gate(video); + expect(result.code).toBe(1); + expect(result.output).toContain("STUTTERING"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, 120_000); + + it("passes a take whose picture actually moves", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "motion-gate-")); + try { + const video = await renderTake(dir, "smooth", { + changesPerSecond: 60, + containerFps: 60, + seconds: 6, + }); + const result = await gate(video); + expect(result.code).toBe(0); + expect(result.output).toContain("fps of real change"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, 120_000); + + it("reads the rate the picture changes, not the rate the container claims", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "motion-gate-")); + try { + // Same content rate, two different container rates. Motion fps must + // not move, or the gate is measuring the wrong thing and a stuttering + // take could be rescued by re-encoding it at a higher frame rate. + const at30 = await renderTake(dir, "at30", { + changesPerSecond: 20, + containerFps: 30, + seconds: 6, + }); + const at60 = await renderTake(dir, "at60", { + changesPerSecond: 20, + containerFps: 60, + seconds: 6, + }); + + const read = (output: string): number => { + const match = output.match(/= (\d+) fps of real change/); + expect(match).not.toBeNull(); + return Number(match?.[1]); + }; + + const a = read((await gate(at30, 5)).output); + const b = read((await gate(at60, 5)).output); + expect(Math.abs(a - b)).toBeLessThanOrEqual(2); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, 180_000); + + it("refuses a take it cannot measure instead of calling it good", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "motion-gate-")); + try { + // Both refusals fail closed, so exit code alone cannot tell them + // apart — drop the missing-file guard and this case still exits 1 + // through the unmeasurable branch. The message is what distinguishes + // them, so the message is what gets pinned. + const missing = await gate(path.join(dir, "does-not-exist.mp4")); + expect(missing.code).toBe(1); + expect(missing.output).toContain("is missing or empty"); + + // A file that exists and is not a video. Silence here would let a + // broken encode through as a pass. + const notAVideo = path.join(dir, "empty.mp4"); + await run("bash", ["-c", `printf 'not a video' > ${JSON.stringify(notAVideo)}`]); + const unreadable = await gate(notAVideo); + expect(unreadable.code).toBe(1); + expect(unreadable.output).toContain("could not measure"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, 60_000); + + it("takes a floor from the caller so a deliberately still take can pass", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "motion-gate-")); + try { + const video = await renderTake(dir, "still", { + changesPerSecond: 3, + containerFps: 60, + seconds: 6, + }); + expect((await gate(video, 12)).code).toBe(1); + expect((await gate(video, 1)).code).toBe(0); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, 120_000); +}); diff --git a/proof/docker/record-x11.sh b/proof/docker/record-x11.sh index d10a671c9..26a189d43 100755 --- a/proof/docker/record-x11.sh +++ b/proof/docker/record-x11.sh @@ -29,7 +29,25 @@ CONTAINER_LLM_BASE_URL="$(container_endpoint "${PROOF_LLM_BASE_URL:-}")" # violet-and-cyan backdrop survived being rewritten as a neutral one: the launcher kept # passing the old colours in, and every take came out in the scheme the file no longer # contained. +AUTH_MOUNTS=() +if [[ -n "${PROOF_AUTH_DIR:-}" ]]; then + AUTH_MOUNTS+=(--mount "type=bind,src=${PROOF_AUTH_DIR},dst=/host-auth,readonly") +fi + +# Mandate GPU acceleration passthrough so the terminal and compositor run at full +# hardware refresh rate with zero CPU-compositor frame jitter. +GPU_ARGS=() +if [ -d /dev/dri ]; then + GPU_ARGS+=(--device /dev/dri) + if [ -e /dev/dri/renderD128 ]; then + RENDER_GID="$(stat -c %g /dev/dri/renderD128 2>/dev/null || echo 992)" + GPU_ARGS+=(--group-add "${RENDER_GID}") + fi +fi + docker run --rm \ + "${AUTH_MOUNTS[@]}" \ + "${GPU_ARGS[@]}" \ --network "${PROOF_NETWORK:-veyyon-proof}" \ --add-host "${CONTAINER_HOST_ALIAS}:host-gateway" \ --mount "type=bind,src=${REPO_ROOT},dst=/repo" \ @@ -62,6 +80,9 @@ docker run --rm \ -e "SCENE_BLUR_STRENGTH" \ -e "SCENE_BACKDROP_BLUR" \ -e "SCENE_CHROME_BACKEND" \ + -e "SCENE_CHROME_BLUR" \ + -e "SCENE_MOTION_GATE" \ + -e "SCENE_MOTION_FLOOR" \ -e "SCENE_BACKDROP_BASE" \ -e "SCENE_BACKDROP_WARM" \ -e "SCENE_BACKDROP_COOL" \ @@ -80,6 +101,10 @@ docker run --rm \ set -e mkdir -p /sandbox/home/.veyyon cp -r /seed/. /sandbox/home/.veyyon/ + if [ -d /host-auth ]; then + mkdir -p /sandbox/home/.veyyon/shared-auth + cp -a /host-auth/. /sandbox/home/.veyyon/shared-auth/ + fi # A recorder on another machine cannot resolve the llama.cpp container by the # name it has on this daemon, so the base URL is overridable at record time. if [ -n "${PROOF_LLM_BASE_URL}" ]; then diff --git a/proof/docker/xsession.sh b/proof/docker/xsession.sh index b143c5cbf..566973d76 100755 --- a/proof/docker/xsession.sh +++ b/proof/docker/xsession.sh @@ -175,10 +175,38 @@ if [ "${SCENE_THEME:-plain}" != "plain" ]; then --corner-radius "${SCENE_RADIUS:-26}" --active-opacity "${SCENE_OPACITY:-0.72}" --inactive-opacity "${SCENE_OPACITY:-0.72}" - --blur-background --shadow --shadow-radius 44 --shadow-opacity 0.55 --shadow-offset-x -22 --shadow-offset-y -12) + # --blur-background IS THE REASON A TAKE COMES OUT AT THREE FRAMES A SECOND, and it + # stays off unless something can actually accelerate it. + # + # A translucent window forces picom to re-blur everything behind it on every frame. + # On this stack that convolution runs on the CPU through xrender, over the whole + # 2304x1184 inset, and it saturates the X server itself. Measured in the recorder + # image at 2560x1440, one identical counter printing as fast as the terminal will + # take it, six seconds per arm, unique frames counted with mpdecimate: + # + # no compositor 171 unique / 359 grabbed 28 fps + # opaque + blur 89 unique / 337 grabbed 14 fps + # 0.72 opacity, no blur 69 unique / 305 grabbed 11 fps + # 0.72 opacity + blur (was default) 14 unique / 69 grabbed 2 fps + # blur-background-fixed 13 unique / 55 grabbed 2 fps + # + # The middle column is the tell that this is not a terminal problem and not an + # encoder problem: with the blur on, ffmpeg could only GRAB 69 of 360 frames. The X + # server had nothing left to answer a screen capture with, so no capture setting and + # no render-loop change downstream can recover the frames -- they were never drawn. + # A published hero take measured 385 unique frames across 7415, a flat 3.3 per second + # through typing, streaming and idle alike, which is this row and nothing else. + # + # SCENE_CHROME_BLUR=1 turns it back on for a still, where frame rate does not exist. + # Anything that moves keeps it off. + if [ "${SCENE_CHROME_BLUR:-0}" = "1" ]; then + CHROME+=(--blur-background) + echo "chrome: blur-background forced on; expect ~2 fps of real motion" >&2 + fi + # xrender's `kernel` blur is the default, and dual_kawase is opt-in behind # SCENE_CHROME_BACKEND=glx. The reasoning in the note this replaces was wrong in both # directions, so both halves are worth writing down. @@ -205,8 +233,8 @@ if [ "${SCENE_THEME:-plain}" != "plain" ]; then "${CHROME[@]}" && GLASS=1 fi [ "${GLASS:-0}" = "1" ] || - start_compositor "xrender kernel ${SCENE_BLUR_KERN:-11x11gaussian}" \ - --backend xrender --blur-method kernel --blur-kern "${SCENE_BLUR_KERN:-11x11gaussian}" \ + start_compositor "xrender kernel ${SCENE_BLUR_KERN:-5x5gaussian}" \ + --backend xrender --blur-method kernel --blur-kern "${SCENE_BLUR_KERN:-5x5gaussian}" \ "${CHROME[@]}" || { echo "picom never redirected the screen; the capture would be unthemed" >&2 exit 1 @@ -238,6 +266,9 @@ xterm) kitty \ --override "font_family=JetBrains Mono" \ --override "font_size=${SCENE_FONT_SIZE:-15}" \ + --override "sync_to_monitor=no" \ + --override "repaint_delay=8" \ + --override "input_delay=1" \ --override "background=${SCENE_BG:-#1e2127}" \ --override "foreground=${SCENE_FG:-#d7dae0}" \ --override "cursor_blink_interval=0" \ @@ -328,9 +359,10 @@ done xdotool mousemove --sync $((MARGIN + TW / 2)) $((MARGIN + TH / 2)) sleep 1 -ffmpeg -loglevel error -y -f x11grab -draw_mouse 1 -framerate "${FPS}" \ +ffmpeg -loglevel error -y -thread_queue_size 2048 -f x11grab -draw_mouse 1 -framerate "${FPS}" \ -video_size "${W}x${H}" -i "${DISPLAY}" \ - -c:v libx264 -preset veryfast -crf 20 -pix_fmt yuv420p \ + -c:v libx264 -preset ultrafast -tune zerolatency -crf 18 -pix_fmt yuv420p \ + -r "${FPS}" \ "${OUT}/${NAME}.mp4" >/tmp/ffmpeg.log 2>&1 & FFMPEG_PID=$! # The recording's own zero, in milliseconds, so a still can name the second of the @@ -359,6 +391,14 @@ sleep 1 cleanup trap - EXIT +# The capture is judged on motion, not on settings: a true 60 fps CFR file that +# encodes without dropping anything can still be three frames a second of actual +# movement, and ffprobe cannot tell the difference. proof/motion-gate.sh owns the +# measurement and the floor. +if [ "${SCENE_MOTION_GATE:-1}" = "1" ]; then + bash "${SCENE_MOTION_GATE_BIN:-/repo/proof/motion-gate.sh}" "${OUT}/${NAME}.mp4" >&2 +fi + # A GIF of the same recording, for a page that has to open in a browser without a # video codec argument. The palette pass is what keeps the terminal's greys from # banding. diff --git a/proof/motion-gate.sh b/proof/motion-gate.sh new file mode 100755 index 000000000..b53fdf002 --- /dev/null +++ b/proof/motion-gate.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Judge a capture on motion. +# +# motion-gate.sh