From a773edd8ce2552363ea3e6552115e5df2aeb4725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ClodoCap=C3=A9o?= <159788250+ClodoCapeo@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:08:58 +0200 Subject: [PATCH] Serialize Vision camera frames and pin corrected presenter --- CLAUDE.md | 4 +- docs/development/code-map.md | 8 ++-- docs/development/render-cadence.md | 5 ++- docs/development/scene-feature-map.md | 2 +- public/vision/ui/mainpresenter.mjs | 22 ++++++---- public/vision/vision-assets.json | 13 +++--- src/engine/README.md | 12 +++--- ...edia-bridge.ts => vision-media-sources.ts} | 41 ------------------- src/engine/vision-presenter.ts | 36 +++++++--------- tests/unit/vision-media-bridge.test.ts | 34 --------------- tests/unit/vision-session.test.ts | 30 ++++++++++++-- 11 files changed, 79 insertions(+), 128 deletions(-) rename src/engine/{vision-media-bridge.ts => vision-media-sources.ts} (51%) delete mode 100644 tests/unit/vision-media-bridge.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 1b92bcc..6ec1f68 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,8 +65,8 @@ The compiled scene renderer is excluded from both Solar builds. - `src/engine/frame-patches.ts` owns scalar coalescing and covering presentation receipts; commands and structure keep their ordered execution. - `src/engine/vision-frames.ts` owns one combined state/media submission in flight - through MessageChannel. `vision-media-bridge.ts` captures the supported WASM - engine dependency for upload before that same submission. Fixed-window cadence + through MessageChannel. The presenter stages camera pixels inside its serialized + Rust/WASM request queue before the same submission. Fixed-window cadence and encoded pixels are qualified in `docs/development/render-cadence.md`. - `src/internal/native-tree.ts` owns draft2 Merkle hashing, checked against the byte-pinned upstream vectors and the real native server. diff --git a/docs/development/code-map.md b/docs/development/code-map.md index 7d78826..f77ba40 100644 --- a/docs/development/code-map.md +++ b/docs/development/code-map.md @@ -32,12 +32,12 @@ Entries: `src/index.ts`, `src/server/index.ts`, `src/dev-entry.ts`, `src/host-en | [src/engine/sound/graph.ts](../../src/engine/sound/graph.ts) | [src/engine/sound/README.md](../../src/engine/sound/README.md) | `SoundBus`, `SoundGraph`, `createSoundGraph` | src/engine/sound/catalogue.ts | src/engine/sound/automation.ts, src/engine/sound/controller.ts, src/engine/sound/offline.ts, src/engine/sound/voice.ts | tests/unit/sound.test.ts | | [src/engine/sound/offline.ts](../../src/engine/sound/offline.ts) | [src/engine/sound/README.md](../../src/engine/sound/README.md) | `renderSound` | src/engine/sound/assets.ts, src/engine/sound/catalogue.ts, src/engine/sound/graph.ts, src/engine/sound/voice.ts, src/scenes/native-document.ts | src/engine/native-lsdp-runtime.ts | via consuming boundary | | [src/engine/sound/voice.ts](../../src/engine/sound/voice.ts) | [src/engine/sound/README.md](../../src/engine/sound/README.md) | `startVoice` | src/engine/sound/catalogue.ts, src/engine/sound/graph.ts | src/engine/sound/controller.ts, src/engine/sound/offline.ts | tests/unit/sound.test.ts | -| [src/engine/vision-composition-frame.ts](../../src/engine/vision-composition-frame.ts) | [src/engine/README.md](../../src/engine/README.md) | `VisionCompositionOwner`, `VisionCompositionCapture`, `VisionCompositionSurface`, `VisionCompositionCoverageSurface`, `VisionCompositionFrame`, `appearance`, `nullableFields`, `framing`, `invalid`, `acceptVisionCompositionFrame` | none | src/engine/vision-contracts.ts, src/engine/vision-media-bridge.ts, src/engine/vision-presenter.ts, src/scenes/native-composition.ts | tests/bench/native-composition-capture-translation.bench.ts, tests/helpers/vision-composition-coverage.ts, tests/helpers/vision-composition-frame.ts, tests/unit/native-composition-coverage.test.ts, tests/unit/native-composition.test.ts, tests/unit/vision-composition-frame.test.ts | -| [src/engine/vision-contracts.ts](../../src/engine/vision-contracts.ts) | [src/engine/README.md](../../src/engine/README.md) | `VisionLiveMediaSource`, `VisionPresenterMessage`, `VisionMediaSubmitReceipt`, `VisionSceneHandle`, `VisionRenderPackage` | src/engine/vision-composition-frame.ts, src/scenes/verified-font.ts | src/engine/live-media-metrics.ts, src/engine/native-composition-session.ts, src/engine/vision-frames.ts, src/engine/vision-media-bridge.ts, src/engine/vision-presenter.ts | tests/unit/native-composition-session.test.ts, tests/unit/vision-session.test.ts | +| [src/engine/vision-composition-frame.ts](../../src/engine/vision-composition-frame.ts) | [src/engine/README.md](../../src/engine/README.md) | `VisionCompositionOwner`, `VisionCompositionCapture`, `VisionCompositionSurface`, `VisionCompositionCoverageSurface`, `VisionCompositionFrame`, `appearance`, `nullableFields`, `framing`, `invalid`, `acceptVisionCompositionFrame` | none | src/engine/vision-contracts.ts, src/engine/vision-media-sources.ts, src/engine/vision-presenter.ts, src/scenes/native-composition.ts | tests/bench/native-composition-capture-translation.bench.ts, tests/helpers/vision-composition-coverage.ts, tests/helpers/vision-composition-frame.ts, tests/unit/native-composition-coverage.test.ts, tests/unit/native-composition.test.ts, tests/unit/vision-composition-frame.test.ts | +| [src/engine/vision-contracts.ts](../../src/engine/vision-contracts.ts) | [src/engine/README.md](../../src/engine/README.md) | `VisionLiveMediaSource`, `VisionPresenterMessage`, `VisionMediaSubmitReceipt`, `VisionSceneHandle`, `VisionRenderPackage` | src/engine/vision-composition-frame.ts, src/scenes/verified-font.ts | src/engine/live-media-metrics.ts, src/engine/native-composition-session.ts, src/engine/vision-frames.ts, src/engine/vision-media-sources.ts, src/engine/vision-presenter.ts | tests/unit/native-composition-session.test.ts, tests/unit/vision-session.test.ts | | [src/engine/vision-font-assets.ts](../../src/engine/vision-font-assets.ts) | [src/engine/README.md](../../src/engine/README.md) | `loadHostFonts`, `visionFontResponse` | src/scenes/verified-font.ts | src/engine/vision-presenter.ts | via consuming boundary | | [src/engine/vision-frames.ts](../../src/engine/vision-frames.ts) | [src/engine/README.md](../../src/engine/README.md) | `VisionFrame`, `VisionFrames`, `VisionFrames.addPatch`, `VisionFrames.addMedia`, `VisionFrames.pending`, `VisionFrames.schedule`, `VisionFrames.flush`, `VisionFrames.close` | src/engine/vision-contracts.ts | src/engine/vision-presenter.ts | tests/unit/vision-frames.test.ts | -| [src/engine/vision-media-bridge.ts](../../src/engine/vision-media-bridge.ts) | [src/engine/README.md](../../src/engine/README.md) | `MediaEngine`, `EngineFactory`, `VisionMediaBridge`, `VisionMediaBridge.module`, `VisionMediaBridge.upload`, `VisionMediaSources`, `VisionMediaSources.filterOwned`, `VisionMediaSources.update`, `VisionMediaSources.accept`, `VisionMediaSources.publish` | src/engine/vision-composition-frame.ts, src/engine/vision-contracts.ts | src/engine/vision-presenter.ts | tests/unit/vision-media-bridge.test.ts, tests/unit/vision-session.test.ts | -| [src/engine/vision-presenter.ts](../../src/engine/vision-presenter.ts) | [src/engine/README.md](../../src/engine/README.md) | `PresenterEvent`, `VisionPresenter`, `PresenterModule`, `PersistentVisionSession`, `PersistentVisionLifetime`, `visionMaxTextureSize`, `disconnectedError`, `lifetimeFor`, `isCurrentLifetime`, `assertCurrentLifetime`, `awaitLifetime`, `createSessionPresenter`, `listenToSession`, `disconnectVisionSession`, `mountVisionScene`, `prewarmVisionSession`, `mountVisionSceneNow`, `activateVisionScene`, `visibleState`, `mediaSubmitReceipt` | src/engine/canvas-presentation.ts, src/engine/vision-composition-frame.ts, src/engine/vision-contracts.ts, src/engine/vision-font-assets.ts, src/engine/vision-frames.ts, src/engine/vision-media-bridge.ts, src/scenes/native-document.ts, src/scenes/types.ts, src/scenes/verified-font.ts | src/engine/live-media.ts, src/engine/native-lsdp-runtime.ts, src/engine/native-scene-frames.ts | tests/unit/instance-sources.test.ts, tests/unit/live-media.test.ts, tests/unit/vision-session.test.ts | +| [src/engine/vision-media-sources.ts](../../src/engine/vision-media-sources.ts) | [src/engine/README.md](../../src/engine/README.md) | `VisionMediaSources`, `VisionMediaSources.filterOwned`, `VisionMediaSources.update`, `VisionMediaSources.accept`, `VisionMediaSources.publish` | src/engine/vision-composition-frame.ts, src/engine/vision-contracts.ts | src/engine/vision-presenter.ts | via consuming boundary | +| [src/engine/vision-presenter.ts](../../src/engine/vision-presenter.ts) | [src/engine/README.md](../../src/engine/README.md) | `PresenterEvent`, `VisionPresenter`, `PresenterModule`, `PersistentVisionSession`, `PersistentVisionLifetime`, `visionMaxTextureSize`, `disconnectedError`, `lifetimeFor`, `isCurrentLifetime`, `assertCurrentLifetime`, `awaitLifetime`, `createSessionPresenter`, `listenToSession`, `disconnectVisionSession`, `mountVisionScene`, `prewarmVisionSession`, `mountVisionSceneNow`, `activateVisionScene`, `visibleState`, `mediaSubmitReceipt` | src/engine/canvas-presentation.ts, src/engine/vision-composition-frame.ts, src/engine/vision-contracts.ts, src/engine/vision-font-assets.ts, src/engine/vision-frames.ts, src/engine/vision-media-sources.ts, src/scenes/native-document.ts, src/scenes/types.ts, src/scenes/verified-font.ts | src/engine/live-media.ts, src/engine/native-lsdp-runtime.ts, src/engine/native-scene-frames.ts | tests/unit/instance-sources.test.ts, tests/unit/live-media.test.ts, tests/unit/vision-session.test.ts | | [src/host-entry.ts](../../src/host-entry.ts) | [README.md](../../README.md) | `SolarHostConfig`, `localFonts` | src/mount.ts, src/scenes/canvas.ts, src/scenes/installation-fonts.ts, src/scenes/local-images.ts, src/scenes/local-library.ts, src/scenes/verified-font.ts, src/types/index.ts | declared entry | via consuming boundary | | [src/index.ts](../../src/index.ts) | [README.md](../../README.md) | entry | src/mount.ts, src/scenes/browser-store.ts, src/scenes/cache.ts, src/scenes/canvas.ts, src/scenes/startup.ts, src/scenes/types.ts, src/types/index.ts | declared entry | tests/unit/mount.test.ts, tests/unit/scene-source.test.ts | | [src/internal/native-digest.ts](../../src/internal/native-digest.ts) | [src/internal/README.md](../../src/internal/README.md) | `workerDigest` | @noble/hashes/sha256 | src/engine/native-worker.ts | tests/unit/native-digest.test.ts | diff --git a/docs/development/render-cadence.md b/docs/development/render-cadence.md index bb5b906..ddc6db2 100644 --- a/docs/development/render-cadence.md +++ b/docs/development/render-cadence.md @@ -12,8 +12,9 @@ a browser RAF heartbeat counts as a new rendered mutation state. scalar values through a microtask. `src/engine/vision-frames.ts` owns one actual submission in flight, scheduled by MessageChannel rather than a 60 Hz RAF. Camera-only work schedules the same immediate MessageChannel task as state. -Arrivals before that task share a submission without a fixed timer delay. `vision-media-bridge.ts` uploads the latest owned -camera image before the same state submission. Errors reject the covered callers; +Arrivals before that task share a submission without a fixed timer delay. The +presenter stages the latest owned camera image inside its serialized Vision +queue before the same state submission. Errors reject the covered callers; scene transitions and ordered commands drain pending presentation first. Vision retains glyph tessellation (1024 entries, 8 MiB), byte-identical shader diff --git a/docs/development/scene-feature-map.md b/docs/development/scene-feature-map.md index 74c48fc..3692383 100644 --- a/docs/development/scene-feature-map.md +++ b/docs/development/scene-feature-map.md @@ -16,7 +16,7 @@ does not resolve or compile a RenderBundle. | Active exact-version child LSML graph, package asset verification, bounded HTTP image localization and Vision transfer | `src/scenes/instances.ts`, `src/scenes/instance-images.ts`, `src/scenes/native-document.ts`, `src/scenes/cache.ts`, `src/scenes/local-library.ts`, `src/engine/native-scene-frames.ts` | `tests/unit/instance-sources.test.ts`, `tests/unit/scene-source.test.ts`, `tests/unit/scene-cache.test.ts`, `tests/unit/scene-library.test.ts`; private Chrome/Vision WASM qualification; separated Canvas package assets remain pinned SHA-256 paths | | Native binary WebSocket snapshots, exact selectors, source fetch and distinct reception/presentation receipts | `src/engine/native-lsdp-runtime.ts`, `vendor/lsdp-native-browser/` | `tests/unit/native-lsdp-runtime.test.ts`; `scripts/test-native-lsdp.mjs` | | Scalar coalescing with every mutation retained in covering presentation receipts | `src/engine/frame-patches.ts` | `tests/unit/frame-patches.test.ts`; blocked-frame native subscriber proof | -| Combined state/camera submission without a RAF cap, immediate coalesced task and owned bitmap lifecycle | `src/engine/vision-frames.ts`, `src/engine/vision-media-bridge.ts` | `tests/unit/vision-frames.test.ts`, `tests/unit/vision-media-bridge.test.ts`; fixed-window CEF cadence and encoded counter | +| Combined state/camera submission without a RAF cap, immediate coalesced task and owned bitmap lifecycle | `src/engine/vision-frames.ts`, `src/engine/vision-media-sources.ts` | `tests/unit/vision-frames.test.ts`, `tests/unit/vision-session.test.ts`; fixed-window CEF cadence and encoded counter | | Atomic LSML operations, text-value projection and temporary render packages with pinned assets/provenance | `src/scenes/native-document.ts` | `tests/unit/native-document.test.ts` | | Native draft2 Merkle hashes and portable JSON conformance | `src/internal/native-tree.ts` | upstream vectors in `tests/unit/native-document.test.ts`; Rust interoperability | | Atomic path copies and incremental native integrity without full-scene rehash per leaf | `src/internal/native-state.ts`, `src/internal/native-tree.ts` | `tests/unit/native-state.test.ts`; runtime and reception consumers | diff --git a/public/vision/ui/mainpresenter.mjs b/public/vision/ui/mainpresenter.mjs index e3baab1..1a72d2f 100644 --- a/public/vision/ui/mainpresenter.mjs +++ b/public/vision/ui/mainpresenter.mjs @@ -492,6 +492,7 @@ export function createMainThreadPresenter(canvas, dependencies = {}) { const applyStartedAt = clock.now(); let rawMetadata; try { + if (request.frames !== undefined) await stageLiveFrames(request.frames, ticket); await registerPatchImages(request.images, ticket); rawMetadata = await engine.apply_patch(request.patchJson); ensureActive(ticket); @@ -500,6 +501,8 @@ export function createMainThreadPresenter(canvas, dependencies = {}) { engine.prune_unreferenced_image_assets?.(); syncUploadedAssets(engine); throw withContext(error, "patch", { ...receipt, engineApplyCallMs: elapsedSince(applyStartedAt) }); + } finally { + for (const frame of request.frames ?? []) frame?.bitmap?.close?.(); } const engineApplyCallMs = elapsedSince(applyStartedAt); @@ -536,13 +539,7 @@ export function createMainThreadPresenter(canvas, dependencies = {}) { throw withContext(new Error("A live media update must contain at least one frame."), "media-frames", receipt); } try { - for (const frame of request.frames) { - if (!frame || typeof frame.path !== "string" || !frame.bitmap || typeof frame.bitmap.close !== "function") { - throw new Error("A live media frame needs a path and an owned ImageBitmap."); - } - await engine.update_live_image(frame.path, frame.bitmap); - ensureActive(ticket); - } + await stageLiveFrames(request.frames, ticket); if (latestSize) await submit(request.seq, latestSize, receipt, ticket); lastAppliedSequence = request.seq; emit({ type: "media-applied", seq: request.seq, mediaSources: readMediaSources(engine), timing: receipt }); @@ -553,6 +550,17 @@ export function createMainThreadPresenter(canvas, dependencies = {}) { } } + async function stageLiveFrames(frames, ticket) { + if (!Array.isArray(frames)) throw new Error("Live media frames must be an array."); + for (const frame of frames) { + if (!frame || typeof frame.path !== "string" || !frame.bitmap || typeof frame.bitmap.close !== "function") { + throw new Error("A live media frame needs a path and an owned ImageBitmap."); + } + await engine.update_live_image(frame.path, frame.bitmap); + ensureActive(ticket); + } + } + async function clearLiveFrames(request, receipt, ticket) { if (!Array.isArray(request.paths) || request.paths.some((path) => typeof path !== "string" || path.length === 0)) { throw withContext(new Error("A live media clear must contain valid image paths."), "media-clear", receipt); diff --git a/public/vision/vision-assets.json b/public/vision/vision-assets.json index 96fca2b..856034b 100644 --- a/public/vision/vision-assets.json +++ b/public/vision/vision-assets.json @@ -2,12 +2,12 @@ "schema": "solar.vision-assets.v1", "upstream": "Lumencast/lumencast-vision", "base_revision": "6df746efb087e032d4ca3466cbedfc972e4a2bbf", - "source_revision": "2c03ae158f5d4d4ea93a02a278976e5b09a5d80d", - "source_state": "Vision measures intrinsic image dimensions from admitted bytes and preserves ordered capture-adjacent paint islands. Previously published x-zab.capture nodes without id derive a qualified legacy source alias from the logical source kind and device ref; authored ids remain authoritative. Solar pins the generated JS and WASM from the signed Vision commit; physical cockpit opening is qualified separately in Prism.", + "source_revision": "1e14d95af94052187a20c59c89871aa1a3dda43b", + "source_state": "Vision measures intrinsic image dimensions from admitted bytes and preserves ordered capture-adjacent paint islands. The main presenter serializes camera bitmap staging with scene loads and patches on its one Rust/WASM engine. Previously published x-zab.capture nodes without id derive a qualified legacy source alias from the logical source kind and device ref; authored ids remain authoritative. Solar pins the presenter, generated JS and WASM from the Vision source commit; physical cockpit opening is qualified separately in Prism.", "files": { "ui/mainpresenter.mjs": { - "sha256": "1046dafdf9d1544dd15e17d24b57dcf8e32fdf74a140285b5f200ca544ccf508", - "bytes": 43272 + "sha256": "8d22a413bd596c4ead6a7378e7c972264f5358b9fef467efdfaff90e4987cf41", + "bytes": 43624 }, "ui/fonts.mjs": { "sha256": "d49ed45181a8aa714f675aaa24f5439e9547b274d6eba6cd2bb995c981765c5f", @@ -26,11 +26,12 @@ "repository": "Lumencast/lumencast-vision", "engine_branch": "main", "engine_base": "6df746efb087e032d4ca3466cbedfc972e4a2bbf", - "engine_commit": "2c03ae158f5d4d4ea93a02a278976e5b09a5d80d", - "engine_tree": "1b37ed890ce0c748f8157968e5153ec7349a62eb", + "engine_commit": "1e14d95af94052187a20c59c89871aa1a3dda43b", + "engine_tree": "2b4d1d742ca220947eb88f6aff29aa5004d203f1", "build_command": "scripts/vision/build-web.ps1 (cargo release locked; wasm-bindgen 0.2.129)", "source_files": { "crates/vision-web/src/lib.rs": "b52284cfaa11345bf826591889dd10895daf5295", + "crates/vision-web/ui/mainpresenter.mjs": "1a72d2fd949d1be41b1c06b91a0c197d29ad18b9", "crates/vision-gpu/src/renderer/contribution.rs": "9d97bc50ccf29cc7f587a5bd63e6996272468d64", "crates/vision-gpu/src/renderer/contribution/encode.rs": "33ec6b786e78416748d82993025f0ce0fc551c76", "crates/vision-gpu/src/renderer/contribution/draw.rs": "0d5eb955022279e4db9d62a5c9d6dabda9aae1e6", diff --git a/src/engine/README.md b/src/engine/README.md index 6a93bdc..d5b1c08 100644 --- a/src/engine/README.md +++ b/src/engine/README.md @@ -249,9 +249,11 @@ instead of the host's 60 Hz RAF. Pending scalar values and the latest owned bitmap per camera can share one actual frame; every caller waits for completion. Camera-only work schedules the same immediate task as state; arrivals before that task share a submission without a fixed timer delay. Superseded and consumed -bitmaps are released, and failure rejects all covered callers. The supported -`loadWasm` dependency captures the real engine in `vision-media-bridge.ts`, -allowing camera upload before the same patch submission. This scheduling +bitmaps are released, and failure rejects all covered callers. +Camera pixels accompanying a patch enter the presenter's serialized queue before +its state mutation and one GPU submission; no host-side engine call can overlap +scene admission. `vision-media-sources.ts` publishes frame-coherent media +descriptors. This scheduling policy is not evidence that the engine achieves 120 completed mutation frames/s; fixed-window CEF measurements establish that separately. The retained Vision candidate, independent encoded-counter checks and fixed @@ -404,8 +406,8 @@ local capture paths explicitly delegated by that receipt to Main/Pulsar from Vision raster inputs. No physical capture resolver/upload is started for those delegated paths; rejected frames retain the previous accepted media list. Canonical rendering keeps its existing immediate media descriptor updates. -The accepted/pending descriptor state lives in `VisionMediaSources` beside the -existing WASM media bridge. The presenter calls `accept` only after identity, +The accepted/pending descriptor state lives in `VisionMediaSources`. The +presenter calls `accept` only after identity, revision and canvas validation; this is media ownership, without an extra render, wait, engine or transition/readiness step. diff --git a/src/engine/vision-media-bridge.ts b/src/engine/vision-media-sources.ts similarity index 51% rename from src/engine/vision-media-bridge.ts rename to src/engine/vision-media-sources.ts index 9383fdd..4c7058f 100644 --- a/src/engine/vision-media-bridge.ts +++ b/src/engine/vision-media-sources.ts @@ -1,47 +1,6 @@ import type { VisionCompositionFrame } from "./vision-composition-frame"; import type { VisionLiveMediaSource } from "./vision-contracts"; -interface MediaEngine { - update_live_image(path: string, bitmap: ImageBitmap): void; -} -type EngineFactory = (...args: unknown[]) => Promise; - -/** Capture the real engine through Vision's supported dependency injection point. */ -export class VisionMediaBridge { - private engine: MediaEngine | null = null; - - module(wasm: Record): Record { - const Engine = wasm.VisionGpuEngine as Record; - return { - ...wasm, - VisionGpuEngine: new Proxy(Engine, { - get: (target, property) => { - const method = Reflect.get(target, property); - if ( - property !== "create_html_canvas" && - property !== "create_html_canvas_with_fonts" && - property !== "create_empty_html_canvas" - ) - return method; - return async (...args: unknown[]) => { - const engine = await (method as EngineFactory).apply(target, args); - if (typeof engine.update_live_image !== "function") - throw new Error("Vision live image API is unavailable."); - this.engine = engine; - return engine; - }; - }, - }), - }; - } - - upload(frames: Array<{ path: string; bitmap: ImageBitmap }>): void { - if (!this.engine) throw new Error("Vision engine is disconnected."); - for (const { path, bitmap } of frames) - this.engine.update_live_image(path, bitmap); - } -} - /** Publish only frame-coherent raster inputs; native paths stay owned by Pulsar. */ export class VisionMediaSources { private accepted: readonly VisionLiveMediaSource[] = []; diff --git a/src/engine/vision-presenter.ts b/src/engine/vision-presenter.ts index a53fc1f..71400cb 100644 --- a/src/engine/vision-presenter.ts +++ b/src/engine/vision-presenter.ts @@ -19,7 +19,7 @@ import type { SceneRenderDelivery } from "../scenes/types"; import { visionImageValue, visionTextPatch } from "../scenes/native-document"; import { CanvasPresentation } from "./canvas-presentation"; import { VisionFrames } from "./vision-frames"; -import { VisionMediaBridge, VisionMediaSources } from "./vision-media-bridge"; +import { VisionMediaSources } from "./vision-media-sources"; import { acceptVisionCompositionFrame, type VisionCompositionFrame, @@ -51,7 +51,6 @@ interface PersistentVisionSession { canvas: HTMLCanvasElement; presentation: CanvasPresentation; presenter: VisionPresenter | null; - bridge: VisionMediaBridge; lifetime: PersistentVisionLifetime; sequence: number; owner: object; @@ -186,17 +185,14 @@ function awaitLifetime( function createSessionPresenter( module: PresenterModule, canvas: HTMLCanvasElement, - bridge: VisionMediaBridge, ): VisionPresenter { return module.createMainThreadPresenter(canvas, { loadWasm: async () => - bridge.module( - await import( - /* @vite-ignore */ new URL( - "vision/pkg/lumencast_vision_web.js", - document.baseURI, - ).href - ), + import( + /* @vite-ignore */ new URL( + "vision/pkg/lumencast_vision_web.js", + document.baseURI, + ).href ), }); } @@ -388,7 +384,6 @@ async function mountVisionSceneNow( let resizeObserver: ResizeObserver | null = null; let removeWindowResize: (() => void) | null = null; let termination: Promise | null = null; - const mediaBridge = existing?.bridge ?? new VisionMediaBridge(); let frameQueue: VisionFrames | null = null; const waiters = new Map< number, @@ -405,7 +400,6 @@ async function mountVisionSceneNow( canvas, presentation, presenter: null, - bridge: mediaBridge, lifetime: lifetime!, sequence: 0, owner, @@ -583,7 +577,7 @@ async function mountVisionSceneNow( )) as PresenterModule; if (!presenter) presenter = persistent - ? createSessionPresenter(presenterModule, canvas, mediaBridge) + ? createSessionPresenter(presenterModule, canvas) : presenterModule.createMainThreadPresenter(canvas, { fetch: async ( input: RequestInfo | URL, @@ -606,13 +600,11 @@ async function mountVisionSceneNow( return fetch(input, init); }, loadWasm: async () => - mediaBridge.module( - await import( - /* @vite-ignore */ new URL( - "vision/pkg/lumencast_vision_web.js", - document.baseURI, - ).href - ), + import( + /* @vite-ignore */ new URL( + "vision/pkg/lumencast_vision_web.js", + document.baseURI, + ).href ), }); if (session?.closed) { @@ -802,11 +794,11 @@ async function mountVisionSceneNow( await presenter!.flush(); if (disposed) throw new Error("Vision scene is disconnected."); if (Object.keys(frame.patch).length > 0) { - // Upload camera pixels before the state patch's one real GPU render. - mediaBridge.upload(frame.frames); + // Stage camera pixels inside the serialized presenter, before one GPU submission. const acknowledgement = await requestFrame({ type: "patch", patchJson: JSON.stringify(frame.patch), + ...(frame.frames.length > 0 ? { frames: frame.frames } : {}), ...(frame.images ? { images: frame.images } : {}), }); return mediaSubmitReceipt(acknowledgement); diff --git a/tests/unit/vision-media-bridge.test.ts b/tests/unit/vision-media-bridge.test.ts deleted file mode 100644 index 6a4ec2f..0000000 --- a/tests/unit/vision-media-bridge.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { VisionMediaBridge } from "../../src/engine/vision-media-bridge"; - -describe("real Vision engine media bridge", () => { - it("keeps the engine and factory arguments intact and uploads through its real public API", async () => { - const engine = { update_live_image: vi.fn() }; - const Engine = { - create_html_canvas_with_fonts: vi.fn(async () => engine), - unrelated: "preserved", - }; - const bridge = new VisionMediaBridge(); - const wasm = bridge.module({ VisionGpuEngine: Engine, default: "init" }); - const wrapped = wasm.VisionGpuEngine as typeof Engine; - const image = {} as ImageBitmap; - expect(await wrapped.create_html_canvas_with_fonts()).toBe(engine); - expect(Engine.create_html_canvas_with_fonts).toHaveBeenCalledOnce(); - expect(wrapped.unrelated).toBe("preserved"); - expect(wasm.default).toBe("init"); - bridge.upload([{ path: "live", bitmap: image }]); - expect(engine.update_live_image).toHaveBeenCalledWith("live", image); - }); - - it("does not bypass missing engine media capabilities", async () => { - const bridge = new VisionMediaBridge(); - expect(() => bridge.upload([])).toThrow("disconnected"); - const module = bridge.module({ - VisionGpuEngine: { create_html_canvas: async () => ({}) }, - }); - const Engine = module.VisionGpuEngine as { - create_html_canvas: () => Promise; - }; - await expect(Engine.create_html_canvas()).rejects.toThrow("unavailable"); - }); -}); diff --git a/tests/unit/vision-session.test.ts b/tests/unit/vision-session.test.ts index 4abd2e8..a6af827 100644 --- a/tests/unit/vision-session.test.ts +++ b/tests/unit/vision-session.test.ts @@ -7,7 +7,6 @@ import { } from "../../src/engine/vision-presenter"; import { nativeManifest } from "../helpers/native-source"; import { compositionFrame } from "../helpers/vision-composition-frame"; -import { VisionMediaBridge } from "../../src/engine/vision-media-bridge"; import { LiveMediaController } from "../../src/engine/live-media"; import type { VisionLiveMediaSource, @@ -28,6 +27,7 @@ const fake = vi.hoisted(() => ({ failPreloadFonts: 0, failLoads: 0, requests: [] as string[], + sent: [] as Record[], composition: null as unknown, canvasSizeOverride: null as [number, number] | null, refuseNativeMode: false, @@ -62,6 +62,7 @@ vi.mock("http://localhost:3000/vision/ui/mainpresenter.mjs", () => ({ [canvas.width, canvas.height] = fake.canvasSizeOverride; } fake.requests.push(type); + fake.sent.push(request); const rejectInitEmpty = type === "init-empty" && fake.failInitEmpty > 0; if (rejectInitEmpty) fake.failInitEmpty--; const silenceInitEmpty = @@ -183,6 +184,7 @@ afterEach(() => { fake.failPreloadFonts = 0; fake.failLoads = 0; fake.requests.length = 0; + fake.sent.length = 0; fake.composition = null; fake.canvasSizeOverride = null; fake.refuseNativeMode = false; @@ -302,7 +304,6 @@ it("retries font admission on the same initialized engine", async () => { it("enables native atlas mode before loading and returns only the matching submitted frame", async () => { const { target, delivery, pack } = setup(); // This suite mocks the presenter rather than constructing a WASM media engine. - vi.spyOn(VisionMediaBridge.prototype, "upload").mockImplementation(() => {}); fake.composition = compositionFrame(); const scene = await mountVisionScene( target, @@ -446,6 +447,29 @@ it("reports only the presenter's render-call timing with its frame-submitted med scene.dispose(); }); +it("sends coalesced camera pixels with the patch through the serialized presenter", async () => { + const { target, delivery, pack } = setup(); + const scene = await mountVisionScene( + target, + delivery, + {}, + vi.fn(), + vi.fn(), + pack, + true, + ); + const bitmap = { close: vi.fn() } as unknown as ImageBitmap; + const patch = scene.applyPatch({ opacity: 0.5 }); + const media = scene.updateMedia([{ path: "camera.live", bitmap }]); + + await Promise.all([patch, media]); + + const sent = fake.sent.filter((request) => request.type === "patch"); + expect(sent).toHaveLength(1); + expect(sent[0]?.frames).toEqual([{ path: "camera.live", bitmap }]); + scene.dispose(); +}); + it("queues first scene admission behind unfinished installation font preload", async () => { const { target, delivery, pack } = setup(); fake.heldTypes.add("preload-fonts"); @@ -791,7 +815,6 @@ function emitPresenter(message: VisionPresenterMessage): void { it("delegates native captures only after their matching frame and keeps peer raster inputs", async () => { const { target, delivery, pack } = setup(); // The presenter is mocked; media ownership is checked without a WASM upload engine. - vi.spyOn(VisionMediaBridge.prototype, "upload").mockImplementation(() => {}); fake.composition = compositionFrame(); const resolveCaptureDevice = vi.fn(async () => ({ deviceId: "must-not-open", @@ -860,7 +883,6 @@ it("delegates native captures only after their matching frame and keeps peer ras it("retains accepted native media when the next receipt is rejected", async () => { const { target, delivery, pack } = setup(); // The presenter is mocked; media ownership is checked without a WASM upload engine. - vi.spyOn(VisionMediaBridge.prototype, "upload").mockImplementation(() => {}); fake.composition = compositionFrame(); const onMedia = vi.fn(); const scene = await mountVisionScene(