diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs index 338beaa80..905a6a5e5 100644 --- a/crates/compositor-view-napi/src/lib.rs +++ b/crates/compositor-view-napi/src/lib.rs @@ -204,6 +204,10 @@ pub struct FramePacket { /// Le métrage passe de ses coins à l'image par leur homographie (caméra réelle), pas par /// leur interpolation bilinéaire. pub footage_projective: bool, + /// Clip actif de la scène quand la frame a été composée (`FramePosition`). + pub clip_index: u32, + /// Temps source de la frame écran composée, en secondes. + pub source_time_sec: f64, } /// Renvoie la dernière frame readback du thread de rendu SI elle est plus récente que @@ -243,7 +247,7 @@ pub fn read_frame(id: i32, since_gen: f64) -> Result> { v.latest_frame_since(since_gen.max(0.0) as u64) } }; - Ok(slot.map(|(gen, w, h, pixels, footage)| { + Ok(slot.map(|(gen, w, h, pixels, footage, position)| { debug_assert_eq!(pixels.len(), (w as usize) * (h as usize) * 4); FramePacket { gen: gen as f64, @@ -252,6 +256,8 @@ pub fn read_frame(id: i32, since_gen: f64) -> Result> { data: Buffer::from(pixels), footage: footage_corners(footage), footage_projective: footage.is_some_and(|q| q.projective), + clip_index: position.clip_index, + source_time_sec: position.source_time_sec, } })) } @@ -276,6 +282,8 @@ pub struct SharedFramePacket { pub height: u32, pub footage: Option>, pub footage_projective: bool, + pub clip_index: u32, + pub source_time_sec: f64, } /// Livrer les frames de la vue par textures partagées plutôt que par `read_frame`. Rend @@ -314,6 +322,8 @@ pub fn read_shared_frame(id: i32, since_gen: f64) -> Result Scene { /// Dernière frame readback vers CPU, prête pour le napi `read_frame`. /// -/// `(gen, w, h, vec)` où `vec.len() == w*h*4` octets RGBA8 tightly-packed (R, G, B, A -/// en mémoire — cf. `Compositor::readback_resized`). `gen` est une génération monotone +/// `(gen, w, h, vec, métrage, position)` où `vec.len() == w*h*4` octets RGBA8 tightly-packed +/// (R, G, B, A en mémoire — cf. `Compositor::readback_resized`). `gen` est une génération monotone /// (≥ 1, `0` réservé à « le consommateur n'a encore rien vu ») incrémentée à CHAQUE /// publication, càd uniquement quand une nouvelle frame a réellement été composée (le /// thread de rendu ne republie pas une frame identique — cf. `stepped || first`). Elle /// est l'IDENTITÉ de la frame : le consommateur (`read_frame`) ne repaie le clone + l'IPC /// que lorsqu'elle change. `None` = "aucune frame composée pour l'instant" (toutes les /// lectures avant la 1re frame composée retournent `None` côté napi, jamais un buffer vide). -pub type LatestFrame = (u64, u32, u32, Vec, Option); +pub type LatestFrame = ( + u64, + u32, + u32, + Vec, + Option, + FramePosition, +); + +/// Où en est la vue quand elle compose une frame : le clip actif de la scène et le temps +/// source de la frame écran. Voyage avec chaque frame publiée, quel que soit le transport, +/// pour que l'app compare la position RÉELLE de la vue à sa propre tête de lecture au lieu +/// de la deviner à partir de l'horloge murale. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FramePosition { + /// Index du clip actif dans `scene.clips`. + pub clip_index: u32, + /// pts de la frame écran composée, en secondes du fichier source. + pub source_time_sec: f64, +} /// État partagé thread appelant → thread de rendu (commandes sans blocage). struct Shared { @@ -1840,7 +1859,11 @@ unsafe fn render_thread( if stepped || first || publish_pending { if pw > 0 && ph > 0 { - match publish_shared(&shared, &gpu, &comp, &mut ring) { + let position = FramePosition { + clip_index: active_clip_index as u32, + source_time_sec: player.screen_time_sec(), + }; + match publish_shared(&shared, &gpu, &comp, &mut ring, position) { SharedPublish::Published => { first = false; publish_pending = false; @@ -1874,7 +1897,7 @@ unsafe fn render_thread( // l'ancienne dérivation tant que le slot n'est pas vidé. let next_gen = shared.frame_gen.fetch_add(1, Ordering::Relaxed) + 1; if let Ok(mut slot) = shared.latest_frame.lock() { - *slot = Some((next_gen, rw, rh, rgba, comp.footage_quad())); + *slot = Some((next_gen, rw, rh, rgba, comp.footage_quad(), position)); } first = false; publish_pending = false; @@ -1918,6 +1941,7 @@ unsafe fn publish_shared( gpu: &Gpu, comp: &Compositor, ring: &mut Ring, + position: FramePosition, ) -> SharedPublish { if !shared.shared_frames.load(Ordering::Relaxed) { return SharedPublish::Off; @@ -1967,6 +1991,7 @@ unsafe fn publish_shared( width, height, footage: comp.footage_quad(), + position, }); } SharedPublish::Published @@ -1976,7 +2001,13 @@ unsafe fn publish_shared( } #[cfg(not(windows))] -unsafe fn publish_shared(_: &Shared, _: &Gpu, _: &Compositor, _: &mut Ring) -> SharedPublish { +unsafe fn publish_shared( + _: &Shared, + _: &Gpu, + _: &Compositor, + _: &mut Ring, + _: FramePosition, +) -> SharedPublish { SharedPublish::Off } @@ -2113,7 +2144,7 @@ pub fn run_standalone(screen: &str, webcam: &str, cursor_json: &str) -> Result<( // standalone n'affiche pas réellement les pixels ici (l'embed Electron est // le consumer réel). On imprime juste une frame de temps en temps pour // confirmer que la chaîne fonctionne. - if let Some((_gen, fw, fh, _pixels, _)) = view.latest_frame() { + if let Some((_gen, fw, fh, _pixels, _, _)) = view.latest_frame() { if (fw, fh) != (w, h) { // garde-fou : la staging de readback suit `set_rect` côté thread // de rendu, donc ce serait une désynchro transitoire — acceptable. diff --git a/crates/compositor/src/shared_frames.rs b/crates/compositor/src/shared_frames.rs index 421897131..a6b4f8415 100644 --- a/crates/compositor/src/shared_frames.rs +++ b/crates/compositor/src/shared_frames.rs @@ -13,6 +13,7 @@ //! macOS (IOSurface) et Linux (dmabuf) restent sur le readback. use crate::frame_geometry::FootageQuad; +use crate::live::FramePosition; use std::time::{Duration, Instant}; /// Cases de l'anneau : une prête, une en transit vers le renderer (tenue jusqu'à ce que @@ -37,6 +38,7 @@ pub struct SharedFrame { pub width: u32, pub height: u32, pub footage: Option, + pub position: FramePosition, } /// Une case prise par JS : la génération qu'elle porte et quand elle est partie. @@ -244,7 +246,15 @@ mod tests { use super::*; fn frame(gen: u64, slot: u32) -> SharedFrame { - SharedFrame { gen, slot, handle: 0, width: 2, height: 2, footage: None } + SharedFrame { + gen, + slot, + handle: 0, + width: 2, + height: 2, + footage: None, + position: FramePosition { clip_index: 0, source_time_sec: 0.0 }, + } } /// Toutes les cases prises par JS, la case `n` portant la génération `n + 1`. diff --git a/electron/native-bridge/services/compositorViewService.ts b/electron/native-bridge/services/compositorViewService.ts index 51e562979..6dbfb1461 100644 --- a/electron/native-bridge/services/compositorViewService.ts +++ b/electron/native-bridge/services/compositorViewService.ts @@ -703,6 +703,8 @@ export class CompositorViewService { height: frame.height, footage: frame.footage ?? null, footageProjective: frame.footageProjective ?? false, + clipIndex: frame.clipIndex, + sourceTimeSec: frame.sourceTimeSec, }; const api = this.sharedTextureApi(); let imported: Electron.SharedTextureImported | undefined; diff --git a/electron/native/compositor-view/addon.d.ts b/electron/native/compositor-view/addon.d.ts index 631d70fdf..e87b830b5 100644 --- a/electron/native/compositor-view/addon.d.ts +++ b/electron/native/compositor-view/addon.d.ts @@ -39,6 +39,10 @@ export interface NativeFramePacket { footage?: number[] | null; /** The footage maps from those corners by their homography (real camera), not bilinearly. */ footageProjective?: boolean; + /** Where the view was when it composed this frame: the active clip in `scene.clips` and the + * screen frame's time in its source file. Absent from an older `.node`. */ + clipIndex?: number; + sourceTimeSec?: number; } /** A preview frame left in a shared GPU texture instead of copied into RAM (Windows, hardware @@ -53,6 +57,10 @@ export interface NativeSharedFramePacket { height: number; footage?: number[] | null; footageProjective?: boolean; + /** Where the view was when it composed this frame: the active clip in `scene.clips` and the + * screen frame's time in its source file. Absent from an older `.node`. */ + clipIndex?: number; + sourceTimeSec?: number; } export interface ExportStats { diff --git a/src/components/ai-edition/NativeCompositorOverlay.test.tsx b/src/components/ai-edition/NativeCompositorOverlay.test.tsx new file mode 100644 index 000000000..619bd84b3 --- /dev/null +++ b/src/components/ai-edition/NativeCompositorOverlay.test.tsx @@ -0,0 +1,215 @@ +// @vitest-environment jsdom +/** + * How the overlay steers the native view while the transport plays. + * + * The view runs its own clock and crosses clip boundaries by itself. The overlay used to + * re-send the clip at every cut anyway, which made the view seek back to a place it had just + * left, or drop the next clip it had preloaded: a hitch at every cut of an edited take. Now it + * reads where the view is from the position each frame reports, and only steps in when the + * view is somewhere else. + */ +import { act, cleanup, render } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AxcutClip, AxcutDocument } from "@/lib/ai-edition/schema"; +import { axcutSchemaVersion } from "@/lib/ai-edition/schema"; +import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { publishNativePosition } from "@/native/nativeSync"; + +const native = vi.hoisted(() => ({ + setActiveClip: vi.fn(async () => ({ ok: true })), + setNativePlaying: vi.fn(), +})); + +vi.mock("@/native", () => ({ + pushAllNativeParams: vi.fn(), + setActiveClip: native.setActiveClip, + setCurrentNativeViewId: vi.fn(), + setNativePlaying: native.setNativePlaying, + setNativeScene: vi.fn(), + subscribeNativeCompositor: () => () => undefined, + useIsCpuCompositor: () => false, + useNativeCompositorView: () => ({ viewId: 7, error: null }), +})); + +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: () => (key: string) => key, +})); + +import { NativeCompositorOverlay } from "./NativeCompositorOverlay"; + +/** One take cut in two: source 0-5 s, then 7-12 s after a 2 s cut. */ +const FIRST: AxcutClip = { + id: "clip_a", + assetId: "asset_take", + sourceStartSec: 0, + sourceEndSec: 5, + timelineStartSec: 0, + timelineEndSec: 5, + wordRefs: [], + origin: "system", + reason: "", +}; +const SECOND: AxcutClip = { + ...FIRST, + id: "clip_b", + sourceStartSec: 7, + sourceEndSec: 12, + timelineStartSec: 5, + timelineEndSec: 10, +}; + +function makeDocument(): AxcutDocument { + return { + schemaVersion: axcutSchemaVersion, + project: { + id: "proj_sync", + title: "Sync", + createdAt: "2026-10-02T08:00:00.000Z", + updatedAt: "2026-10-02T08:00:00.000Z", + primaryAssetId: "asset_take", + }, + assets: [ + { + id: "asset_take", + kind: "video", + label: "take", + originalPath: "/take.mp4", + cameraTrack: null, + }, + ], + transcript: null, + transcripts: [], + timeline: { + clips: [FIRST, SECOND], + gaps: [], + trimRanges: [], + muteRanges: [], + speedRanges: [], + captionRanges: [], + }, + annotations: [], + zoomRanges: [], + audioTracks: [], + legacyEditor: null, + }; +} + +function setPlayhead(currentTimeSec: number, playing: boolean) { + act(() => { + useProjectStore.setState({ currentTimeSec, playing }); + }); +} + +describe("NativeCompositorOverlay while playing", () => { + let now = 1000; + + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(performance, "now").mockImplementation(() => now); + useProjectStore.setState({ + projectId: "proj_sync", + document: makeDocument(), + revision: 1, + status: "ready", + error: null, + sourceDurationSec: 12, + currentTimeSec: 4.9, + playing: true, + dirty: false, + lastSavedAt: new Date(), + }); + }); + + afterEach(() => { + cleanup(); + vi.restoreAllMocks(); + publishNativePosition(null); + useProjectStore.getState().clear(); + }); + + async function mountAtFirstClip() { + render(); + // The first mount sends the starting clip, and its reply resumes the view: let both + // settle, so only what follows is under test. + await act(async () => { + await Promise.resolve(); + }); + native.setActiveClip.mockClear(); + native.setNativePlaying.mockClear(); + } + + it("lets the view cross a cut by itself", async () => { + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.99 }, now); + + setPlayhead(5.02, true); + + expect(native.setActiveClip).not.toHaveBeenCalled(); + expect(native.setNativePlaying).not.toHaveBeenCalled(); + }); + + it("follows a jump to another clip at once, without pausing the view", async () => { + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.9 }, now); + + setPlayhead(8, true); + + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 1, 10); + expect(native.setNativePlaying).not.toHaveBeenCalledWith(false); + }); + + it("re-anchors a view that stays behind, once the gap holds", async () => { + await mountAtFirstClip(); + // The view stalled 0.4 s behind the playhead, inside the first clip. + publishNativePosition({ clipIndex: 0, sourceTimeSec: 2 }, now); + setPlayhead(2.4, true); + expect(native.setActiveClip).not.toHaveBeenCalled(); + + now += 120; + publishNativePosition({ clipIndex: 0, sourceTimeSec: 2.12 }, now); + setPlayhead(2.52, true); + + expect(native.setActiveClip).toHaveBeenCalledTimes(1); + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 0, 2.52); + }); + + // At 16× a frame that takes 30 ms to arrive shows the playhead 0.48 s of programme ago. + it("leaves alone a view keeping pace inside a speed region", async () => { + useProjectStore.setState({ + document: { + ...makeDocument(), + legacyEditor: { speedRegions: [{ id: "speed_a", startMs: 0, endMs: 12000, speed: 16 }] }, + }, + currentTimeSec: 0.2, + }); + await mountAtFirstClip(); + + for (let frame = 0; frame < 8; frame++) { + now += 30; + publishNativePosition({ clipIndex: 0, sourceTimeSec: 0.2 + frame * 0.48 }, now); + setPlayhead(0.2 + (frame + 1) * 0.48, true); + } + + expect(native.setActiveClip).not.toHaveBeenCalled(); + }); + + it("still sends the clip on a change while paused", async () => { + useProjectStore.setState({ playing: false }); + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.9 }, now); + + setPlayhead(6, false); + + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 1, 8); + }); + + // An addon that reports no position cannot be read, so it is driven as before. + it("pauses the view across the swap when it reports no position", async () => { + await mountAtFirstClip(); + + setPlayhead(5.02, true); + + expect(native.setNativePlaying).toHaveBeenCalledWith(false); + expect(native.setActiveClip).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/components/ai-edition/NativeCompositorOverlay.tsx b/src/components/ai-edition/NativeCompositorOverlay.tsx index e4af64102..d60bbcd20 100644 --- a/src/components/ai-edition/NativeCompositorOverlay.tsx +++ b/src/components/ai-edition/NativeCompositorOverlay.tsx @@ -1,9 +1,11 @@ import { useEffect, useMemo, useRef, useSyncExternalStore } from "react"; import { useScopedT } from "@/contexts/I18nContext"; +import { readSpeedRegions } from "@/lib/ai-edition/document/timeline"; import { noteUiProbeClipSwitch } from "@/lib/ai-edition/perf/uiFrameProbe"; import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { assetCameraSource } from "@/lib/ai-edition/timeline/camera"; +import { findActiveSpeedRegion, type SpeedRegion } from "@/lib/ai-edition/timeline/speed"; import { resolveNativePosition } from "@/lib/ai-edition/timeline/timelineMap"; import { pushAllNativeParams, @@ -15,6 +17,14 @@ import { useIsCpuCompositor, useNativeCompositorView, } from "@/native"; +import { + type DriftWatch, + getNativePosition, + IDLE_DRIFT_WATCH, + NATIVE_DRIFT_TOLERANCE_SEC, + nativeLeadSec, + watchDrift, +} from "@/native/nativeSync"; import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription"; import { getWebcamNativeSize, @@ -183,12 +193,27 @@ export function NativeCompositorOverlay() { const activeClipId = activeClip?.id ?? null; const activeClipIndex = activePosition?.clipIndex ?? null; const activeSourceTimeSec = activePosition?.sourceTimeSec ?? null; + // Matched on the raw ruler, like `currentTimeSec` and the regions' pills. + const speedRegions = useMemo( + () => (document ? readSpeedRegions(document) : []), + [document], + ); + const playbackSpeed = findActiveSpeedRegion(speedRegions, currentTimeSec * 1000)?.speed ?? 1; const pendingTargetClipIdRef = useRef(null); const playing = useProjectStore((s) => s.playing); + const wasPlayingRef = useRef(playing); - // Change les décodeurs screen/webcam uniquement quand le playhead entre dans un autre clip. + // Change les décodeurs screen/webcam quand le playhead entre dans un autre clip, à l'arrêt. + // + // En lecture, la vue enchaîne seule les clips, préchargement compris. Lui renvoyer le clip à + // chaque coupe la faisait rechercher une position qu'elle avait déjà, ou jeter le + // préchargement qu'elle s'apprêtait à utiliser : un recul de quelques images et un arrêt à + // chaque coupe. Elle n'est recalée que si elle est réellement ailleurs (guet de dérive + // ci-dessous), d'après la position que porte chaque frame. useEffect(() => { + const justPaused = wasPlayingRef.current && !playing; + wasPlayingRef.current = playing; if ( viewId === null || !document || @@ -199,7 +224,13 @@ export function NativeCompositorOverlay() { ) { return; } - if (previousActiveClipIdRef.current === activeClipId) { + const native = getNativePosition(); + const clipChanged = previousActiveClipIdRef.current !== activeClipId; + // Mise en pause pendant que la vue était sur un autre clip que la tête de lecture : elle + // y retourne, sans quoi le recalage en temps (`setNativeTime`) chercherait dans le + // mauvais fichier. + const nativeElsewhere = justPaused && native !== null && native.clipIndex !== activeClipIndex; + if (!clipChanged && !nativeElsewhere) { return; } const asset = document.assets.find((candidate) => candidate.id === activeClip.assetId); @@ -210,13 +241,31 @@ export function NativeCompositorOverlay() { const targetClipId = activeClipId; // Sonde de fluidité (diagnostic) : sépare les mesures d'avant et d'après un // franchissement de clip, qui se sont déjà révélées non comparables. - noteUiProbeClipSwitch(previousActiveClipIdRef.current, activeClipId); + if (clipChanged) { + noteUiProbeClipSwitch(previousActiveClipIdRef.current, activeClipId); + } pendingTargetClipIdRef.current = targetClipId; previousActiveClipIdRef.current = targetClipId; + if (playing && native !== null) { + const lead = nativeLeadSec( + native, + { clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec }, + nativeClips, + playbackSpeed, + performance.now(), + ); + // Une coupe franchie en lecture : la vue la franchit d'elle-même, au même endroit du + // programme. Un saut (clic ailleurs sur la timeline) la met loin : suivi tout de suite, + // sans attendre le délai du guet de dérive. + if (lead !== null && Math.abs(lead) <= NATIVE_DRIFT_TOLERANCE_SEC) { + return; + } + } - // Pause native across the decoder swap. Deliberately NOT kept in a variable to - // resume from later — see the `.then` below, which re-reads the live transport. - if (playing) { + // Sans position rapportée (addon antérieur), la vue est encore pilotée comme avant : en + // pause pendant le changement de décodeurs. Deliberately NOT kept in a variable to resume + // from later — see the `.then` below, which re-reads the live transport. + if (playing && native === null) { setNativePlaying(false); } @@ -248,7 +297,74 @@ export function NativeCompositorOverlay() { previousActiveClipIdRef.current = null; } }); - }, [viewId, document, activeClipId, activeClip, activeClipIndex, activeSourceTimeSec, playing]); + }, [ + viewId, + document, + activeClipId, + activeClip, + activeClipIndex, + activeSourceTimeSec, + playing, + nativeClips, + playbackSpeed, + ]); + + // Guet de dérive, en lecture : la vue tourne sur sa propre horloge, et n'est recalée que si + // la position que porte sa dernière frame s'écarte de la tête de lecture et le reste. Couvre + // un saut de l'utilisateur pendant la lecture comme un blocage du thread de rendu. L'ancien + // recalage devinait la dérive à l'horloge murale, à vitesse 1 : dans une région à 2× il + // relançait une recherche dix fois par seconde. + const driftRef = useRef(IDLE_DRIFT_WATCH); + useEffect(() => { + if ( + viewId === null || + !document || + !playing || + !activeClip || + activeClipIndex === null || + activeSourceTimeSec === null + ) { + driftRef.current = { ...driftRef.current, outSinceMs: null }; + return; + } + const now = performance.now(); + const lead = nativeLeadSec( + getNativePosition(), + { clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec }, + nativeClips, + playbackSpeed, + now, + ); + const { watch, resync } = watchDrift(driftRef.current, lead, now); + driftRef.current = watch; + if (!resync) { + return; + } + const asset = document.assets.find((candidate) => candidate.id === activeClip.assetId); + if (!asset?.originalPath) { + return; + } + const camera = assetCameraSource(asset); + setActiveClip( + viewId, + asset.originalPath, + camera.path, + camera.offsetSec, + activeClipIndex, + activeSourceTimeSec, + ).catch((error: unknown) => { + console.warn("[compositor-view] re-anchoring the preview failed:", error); + }); + }, [ + viewId, + document, + playing, + activeClip, + activeClipIndex, + activeSourceTimeSec, + nativeClips, + playbackSpeed, + ]); if (!ready) { return null; diff --git a/src/native/contracts.ts b/src/native/contracts.ts index d13a95f5c..5421b994a 100644 --- a/src/native/contracts.ts +++ b/src/native/contracts.ts @@ -165,6 +165,10 @@ export interface CompositorFramePacket { footage?: number[] | null; /** The footage maps from those corners by their homography (real camera), not bilinearly. */ footageProjective?: boolean; + /** Where the view was when it composed this frame: the active clip in the scene's clips and + * the screen frame's time in its source file. Absent from an older addon. */ + clipIndex?: number; + sourceTimeSec?: number; } /** What travels with a preview frame handed over as a shared GPU texture (Windows): all a @@ -177,6 +181,9 @@ export interface CompositorSharedFrameMeta { height: number; footage: number[] | null; footageProjective: boolean; + /** See {@link CompositorFramePacket.clipIndex}. */ + clipIndex?: number; + sourceTimeSec?: number; } /** `readFrame`'s answer for a frame sent as a shared texture. The texture reached diff --git a/src/native/hooks/useNativeCompositorView.ts b/src/native/hooks/useNativeCompositorView.ts index 94688c2bf..e11ec5938 100644 --- a/src/native/hooks/useNativeCompositorView.ts +++ b/src/native/hooks/useNativeCompositorView.ts @@ -37,6 +37,7 @@ import { } from "../compositorViewClient"; import type { CompositorParamValue, CompositorViewRect } from "../contracts"; import { publishFootageQuad } from "../footageQuadStore"; +import { publishNativePosition } from "../nativeSync"; import { computeDeviceRect, rectsEqual } from "../nativeViewRect"; export interface UseNativeCompositorViewOptions { @@ -245,6 +246,7 @@ export function useNativeCompositorView( sharedTransport = true; lastGen = Math.max(lastGen, meta.gen); publishFootageQuad(meta.footage, meta.footageProjective); + publishNativePosition(meta); const fresh = canvas.dataset.painted === undefined; const drawn = paint(meta.gen, meta.width, meta.height, () => ctx.drawImage(frame, 0, 0)); if (drawn && fresh && !sharedChecked) { @@ -314,6 +316,7 @@ export function useNativeCompositorView( } // Where the footage lies in this frame: a privacy blur's gimbal follows it. publishFootageQuad(frame.footage, frame.footageProjective); + publishNativePosition(frame); // Wrap the received buffer DIRECTLY — no intermediate copy. `data` is a // fresh per-frame Buffer from IPC (never pooled or reused across frames), // so a view over it is valid for the lifetime of this paint, and nothing @@ -399,6 +402,7 @@ export function useNativeCompositorView( disposed = true; unsubscribeShared(); publishFootageQuad(null); + publishNativePosition(null); if (rectRafHandle !== 0) { cancelAnimationFrame(rectRafHandle); } diff --git a/src/native/nativeSync.test.ts b/src/native/nativeSync.test.ts new file mode 100644 index 000000000..3e6842e11 --- /dev/null +++ b/src/native/nativeSync.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from "vitest"; +import { + getNativePosition, + IDLE_DRIFT_WATCH, + NATIVE_DRIFT_PERSIST_MS, + NATIVE_FRAME_MAX_AGE_MS, + NATIVE_RESYNC_COOLDOWN_MS, + nativeLeadSec, + programmeTimeSec, + publishNativePosition, + watchDrift, +} from "./nativeSync"; + +/** Two segments of one recording around a cut: source 0-5 s, then 7-12 s from programme 5 s. */ +const SEGMENTS = [ + { timelineStartSec: 0, sourceStartSec: 0 }, + { timelineStartSec: 5, sourceStartSec: 7 }, +]; + +describe("programmeTimeSec", () => { + it("places a source time on the trim-compressed timeline", () => { + expect(programmeTimeSec(SEGMENTS, { clipIndex: 1, sourceTimeSec: 8 })).toBe(6); + }); + + it("has no answer for a clip the layout does not have", () => { + expect(programmeTimeSec(SEGMENTS, { clipIndex: 4, sourceTimeSec: 8 })).toBeNull(); + }); +}); + +describe("nativeLeadSec", () => { + // Across a cut the source time jumps by the trimmed 2 s; on the programme it does not. + it("sees no gap when the view and the playhead sit on either side of a cut", () => { + const lead = nativeLeadSec( + { clipIndex: 0, sourceTimeSec: 4.99, receivedAtMs: 1000 }, + { clipIndex: 1, sourceTimeSec: 7.02 }, + SEGMENTS, + 1, + 1000, + ); + expect(lead).toBeCloseTo(-0.03, 5); + }); + + it("ages the view's last frame by the time since it arrived", () => { + const lead = nativeLeadSec( + { clipIndex: 1, sourceTimeSec: 8, receivedAtMs: 1000 }, + { clipIndex: 1, sourceTimeSec: 8.05 }, + SEGMENTS, + 1, + 1050, + ); + expect(lead).toBeCloseTo(0, 5); + }); + + // At 16× the playhead covers 0.64 s of programme in the 40 ms a frame takes to arrive and + // be read: in programme seconds that is a view far behind, to the eye it is in step. + it("measures the gap in seconds of playback inside a speed region", () => { + const lead = nativeLeadSec( + { clipIndex: 1, sourceTimeSec: 8, receivedAtMs: 1000 }, + { clipIndex: 1, sourceTimeSec: 8.64 }, + SEGMENTS, + 16, + 1030, + ); + expect(lead).toBeCloseTo(-0.01, 5); + }); + + it("measures a view left behind by a stall", () => { + const lead = nativeLeadSec( + { clipIndex: 0, sourceTimeSec: 2, receivedAtMs: 1000 }, + { clipIndex: 0, sourceTimeSec: 2.4 }, + SEGMENTS, + 1, + 1000, + ); + expect(lead).toBeCloseTo(-0.4, 5); + }); + + // Aging the last frame without bound would move a frozen view along with the playhead. + it("catches a view that froze in step with the playhead", () => { + const frozen = { clipIndex: 0, sourceTimeSec: 2, receivedAtMs: 1000 }; + let state = watchDrift(IDLE_DRIFT_WATCH, null, 1000); + let resyncAtMs: number | null = null; + for (let now = 1000; now <= 2000 && resyncAtMs === null; now += 16) { + const playhead = { clipIndex: 0, sourceTimeSec: 2 + (now - 1000) / 1000 }; + state = watchDrift(state.watch, nativeLeadSec(frozen, playhead, SEGMENTS, 1, now), now); + resyncAtMs = state.resync ? now : null; + } + expect(resyncAtMs).not.toBeNull(); + expect(resyncAtMs).toBeGreaterThan(1000 + NATIVE_FRAME_MAX_AGE_MS); + }); + + it("says nothing without a reported position", () => { + expect(nativeLeadSec(null, { clipIndex: 0, sourceTimeSec: 1 }, SEGMENTS, 1, 0)).toBeNull(); + }); +}); + +describe("watchDrift", () => { + it("leaves a view within tolerance alone", () => { + const { resync, watch } = watchDrift(IDLE_DRIFT_WATCH, 0.1, 1000); + expect(resync).toBe(false); + expect(watch.outSinceMs).toBeNull(); + }); + + // Right at a cut, or after an edit re-indexes the clips, the gap lasts a frame or two. + it("re-anchors only a gap that holds", () => { + let state = watchDrift(IDLE_DRIFT_WATCH, 0.5, 1000); + expect(state.resync).toBe(false); + state = watchDrift(state.watch, 0.5, 1000 + NATIVE_DRIFT_PERSIST_MS - 1); + expect(state.resync).toBe(false); + state = watchDrift(state.watch, 0.5, 1000 + NATIVE_DRIFT_PERSIST_MS); + expect(state.resync).toBe(true); + }); + + it("forgets a gap that closed before it held", () => { + let state = watchDrift(IDLE_DRIFT_WATCH, 0.5, 1000); + state = watchDrift(state.watch, 0.01, 1050); + state = watchDrift(state.watch, 0.5, 1100); + expect(state.resync).toBe(false); + }); + + it("waits for a re-anchor to land before the next", () => { + let state = watchDrift(IDLE_DRIFT_WATCH, 0.5, 0); + state = watchDrift(state.watch, 0.5, NATIVE_DRIFT_PERSIST_MS); + expect(state.resync).toBe(true); + const at = NATIVE_DRIFT_PERSIST_MS; + state = watchDrift(state.watch, 0.5, at + 10); + state = watchDrift(state.watch, 0.5, at + NATIVE_RESYNC_COOLDOWN_MS - 1); + expect(state.resync).toBe(false); + state = watchDrift(state.watch, 0.5, at + NATIVE_RESYNC_COOLDOWN_MS); + expect(state.resync).toBe(true); + }); +}); + +describe("publishNativePosition", () => { + it("keeps the position a frame reports, with when it arrived", () => { + publishNativePosition({ clipIndex: 2, sourceTimeSec: 3.5 }, 42); + expect(getNativePosition()).toEqual({ clipIndex: 2, sourceTimeSec: 3.5, receivedAtMs: 42 }); + }); + + // An addon that predates positions must not be steered on a stale one. + it("forgets the position when a frame reports none", () => { + publishNativePosition({ clipIndex: 2, sourceTimeSec: 3.5 }, 42); + publishNativePosition({}); + expect(getNativePosition()).toBeNull(); + }); +}); diff --git a/src/native/nativeSync.ts b/src/native/nativeSync.ts new file mode 100644 index 000000000..0db0b8bac --- /dev/null +++ b/src/native/nativeSync.ts @@ -0,0 +1,127 @@ +/** + * Where the native preview view is, against where the app's playhead is. + * + * While playing, the view runs its own clock and crosses clip boundaries by itself, so the + * app does not drive it frame by frame. It used to guess whether the view had drifted from + * the wall clock, assuming 1× speed: inside a 2× speed region the guess was 100 ms off every + * 100 ms, and the view was re-seeked ten times a second. Each frame now says where the view + * really is (`clipIndex`, `sourceTimeSec`), and this compares that with the playhead on the + * one timeline both share: programme time, the trim-compressed one, where a cut is no jump. + */ + +/** A segment of the trim-compressed programme, as `resolveVisibleClips` lays them out. */ +export interface ProgrammeSegment { + timelineStartSec: number; + sourceStartSec: number; +} + +/** Where a frame is: the active clip in the scene and the frame's time in its source file. */ +export interface NativePosition { + clipIndex: number; + sourceTimeSec: number; +} + +/** The position the view last reported, and when it reached the renderer. */ +export interface ReportedNativePosition extends NativePosition { + receivedAtMs: number; +} + +/** Beyond this the picture is visibly off the sound: re-anchor the view. Lip sync is noticed + * from ~45 ms when the picture leads and ~125 ms when it lags (ITU-R BT.1359); frames reach + * the renderer up to a frame or two late, so a tighter bound would chase measurement noise. */ +export const NATIVE_DRIFT_TOLERANCE_SEC = 0.15; +/** The gap must hold this long: right at a cut the view and the playhead cross it a frame or + * two apart, and a scene edit re-indexes the clips for a frame before both agree again. */ +export const NATIVE_DRIFT_PERSIST_MS = 100; +/** After a re-anchor the view needs a seek and a first frame before it reports the new place. */ +export const NATIVE_RESYNC_COOLDOWN_MS = 500; +/** A frame older than this is not late but stalled: aging it further would keep a frozen view + * level with the playhead for good. Two frames of the CPU compositor's ~8 fps. */ +export const NATIVE_FRAME_MAX_AGE_MS = 250; + +/** `sourceTimeSec` of segment `clipIndex`, on the programme timeline. `null` for a segment the + * layout does not have (the scene and the document briefly disagree after an edit). */ +export function programmeTimeSec( + segments: readonly ProgrammeSegment[], + position: NativePosition, +): number | null { + const segment = segments[position.clipIndex]; + if (!segment) { + return null; + } + return segment.timelineStartSec + (position.sourceTimeSec - segment.sourceStartSec); +} + +/** How far ahead of the playhead the view is now, in seconds of playback (negative: behind). + * Inside a speed region the programme runs `speed` times faster than playback: a frame or + * two of delay at 16× is a second of programme, and still only a frame or two to the eye. + * The last frame is aged by the wall time since it arrived, up to `NATIVE_FRAME_MAX_AGE_MS`. + * `null` when either position cannot be placed. */ +export function nativeLeadSec( + native: ReportedNativePosition | null, + app: NativePosition, + segments: readonly ProgrammeSegment[], + speed: number, + nowMs: number, +): number | null { + if (!native) { + return null; + } + const nativeSec = programmeTimeSec(segments, native); + const appSec = programmeTimeSec(segments, app); + if (nativeSec === null || appSec === null) { + return null; + } + const ageMs = Math.min(Math.max(0, nowMs - native.receivedAtMs), NATIVE_FRAME_MAX_AGE_MS); + return (nativeSec - appSec) / speed + ageMs / 1000; +} + +export interface DriftWatch { + /** Since when the view has been out of tolerance, `null` while it is in. */ + outSinceMs: number | null; + lastResyncMs: number; +} + +export const IDLE_DRIFT_WATCH: DriftWatch = { + outSinceMs: null, + lastResyncMs: Number.NEGATIVE_INFINITY, +}; + +/** Whether to re-anchor the view now, given its lead. Only for a gap that holds, and not again + * before the last re-anchor has had time to land. */ +export function watchDrift( + watch: DriftWatch, + leadSec: number | null, + nowMs: number, +): { watch: DriftWatch; resync: boolean } { + if (leadSec === null || Math.abs(leadSec) <= NATIVE_DRIFT_TOLERANCE_SEC) { + return { watch: { ...watch, outSinceMs: null }, resync: false }; + } + const outSinceMs = watch.outSinceMs ?? nowMs; + if ( + nowMs - outSinceMs < NATIVE_DRIFT_PERSIST_MS || + nowMs - watch.lastResyncMs < NATIVE_RESYNC_COOLDOWN_MS + ) { + return { watch: { ...watch, outSinceMs }, resync: false }; + } + return { watch: { outSinceMs: null, lastResyncMs: nowMs }, resync: true }; +} + +let reported: ReportedNativePosition | null = null; + +/** Records where the view's latest frame is, as it reaches the renderer. `null` forgets it: the + * view is gone, or its addon predates positions — then nothing here may steer it. */ +export function publishNativePosition( + position: Partial | null | undefined, + receivedAtMs = performance.now(), +): void { + reported = + position && typeof position.clipIndex === "number" && typeof position.sourceTimeSec === "number" + ? { clipIndex: position.clipIndex, sourceTimeSec: position.sourceTimeSec, receivedAtMs } + : null; +} + +/** The position the view last reported, `null` before its first frame or without one. */ +export function getNativePosition(): ReportedNativePosition | null { + return reported; +} diff --git a/src/native/useNativePlaybackSync.test.ts b/src/native/useNativePlaybackSync.test.ts new file mode 100644 index 000000000..4629f2130 --- /dev/null +++ b/src/native/useNativePlaybackSync.test.ts @@ -0,0 +1,85 @@ +// @vitest-environment jsdom +/** + * A jump inside the clip while playing. The overlay catches it from the position each frame + * reports; an addon that reports none leaves it to this hook, against the wall clock at 1×. + */ +import { renderHook } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AxcutClip } from "@/lib/ai-edition/schema"; +import { publishNativePosition } from "./nativeSync"; + +const store = vi.hoisted(() => ({ + setNativePlaying: vi.fn(), + setNativeTime: vi.fn(), +})); + +vi.mock("./nativeCompositorStore", () => ({ + getCurrentNativeViewId: () => 7, + setNativePlaying: store.setNativePlaying, + setNativeTime: store.setNativeTime, + subscribeNativeCompositor: () => () => undefined, +})); + +import { useNativePlaybackSync } from "./useNativePlaybackSync"; + +const CLIPS: AxcutClip[] = [ + { + id: "clip_a", + assetId: "asset_take", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + wordRefs: [], + origin: "system", + reason: "", + }, +]; + +describe("useNativePlaybackSync while playing", () => { + let now = 1000; + + beforeEach(() => { + vi.clearAllMocks(); + now = 1000; + vi.spyOn(performance, "now").mockImplementation(() => now); + }); + + afterEach(() => { + vi.restoreAllMocks(); + publishNativePosition(null); + }); + + /** Plays from `fromSec` for a few frames, the playhead in step with the wall clock. */ + function play(fromSec: number) { + const hook = renderHook(({ time }) => useNativePlaybackSync(true, time, CLIPS, CLIPS), { + initialProps: { time: fromSec }, + }); + for (let frame = 1; frame <= 3; frame++) { + now += 16; + hook.rerender({ time: fromSec + frame * 0.016 }); + } + return hook; + } + + it("seeks the view on a jump inside the clip when the addon reports no position", () => { + const hook = play(1); + expect(store.setNativeTime).not.toHaveBeenCalled(); + + now += 16; + hook.rerender({ time: 6 }); + + expect(store.setNativeTime).toHaveBeenCalledWith(6); + }); + + // The overlay watches the drift then; a guess at 1× here would re-seek inside speed regions. + it("leaves the view to the overlay when the addon reports positions", () => { + publishNativePosition({ clipIndex: 0, sourceTimeSec: 1 }, now); + const hook = play(1); + + now += 16; + hook.rerender({ time: 6 }); + + expect(store.setNativeTime).not.toHaveBeenCalled(); + }); +}); diff --git a/src/native/useNativePlaybackSync.ts b/src/native/useNativePlaybackSync.ts index 64e877b63..7ff2dd81f 100644 --- a/src/native/useNativePlaybackSync.ts +++ b/src/native/useNativePlaybackSync.ts @@ -13,9 +13,10 @@ * over free-run). So discrete seeks are only sent while *paused* — i.e. real * scrub/step interactions. Pausing also re-snaps native to the app playhead. * - * Known POC limitation: during free-run the native clock and the app clock can - * drift (independent tickers); acceptable for the fixture (~6 s loop). A pause - * re-aligns them. + * While playing, the two clocks are compared by `NativeCompositorOverlay`, from the + * position each native frame reports (`nativeSync.ts`). This hook only guesses the drift + * from the wall clock at 1× speed for an addon that reports no position: the guess + * re-seeked the view ten times a second inside a 2× speed region. */ import { useEffect, useMemo, useRef, useSyncExternalStore } from "react"; import type { AxcutClip } from "@/lib/ai-edition/schema"; @@ -26,6 +27,7 @@ import { setNativeTime, subscribeNativeCompositor, } from "./nativeCompositorStore"; +import { getNativePosition } from "./nativeSync"; export function useNativePlaybackSync( playing: boolean, @@ -57,7 +59,8 @@ export function useNativePlaybackSync( setNativePlaying(playing); }, [active, playing]); - // Scrub/step while paused OR periodic resync during playback when drift > 100ms + // Scrub/step while paused, and drift while playing for an addon that reports no position. + // A clip change is `setActiveClip`'s, in the overlay. const lastSyncedSourceTimeRef = useRef(null); const lastSyncedWallTimeRef = useRef(0); const lastActiveClipIdRef = useRef(null); @@ -67,22 +70,21 @@ export function useNativePlaybackSync( return; } const now = performance.now(); - - // When clip changes, let setActiveClip handle the atomic clip-switch-and-seek. if (lastActiveClipIdRef.current !== activeClipId) { lastActiveClipIdRef.current = activeClipId; lastSyncedSourceTimeRef.current = sourceTimeSec; lastSyncedWallTimeRef.current = now; return; } - if (!playing) { setNativeTime(sourceTimeSec); lastSyncedSourceTimeRef.current = sourceTimeSec; lastSyncedWallTimeRef.current = now; return; } - // While playing: periodically verify master clock alignment to prevent drift + if (getNativePosition() !== null) { + return; + } if (lastSyncedSourceTimeRef.current === null || lastSyncedWallTimeRef.current === 0) { lastSyncedSourceTimeRef.current = sourceTimeSec; lastSyncedWallTimeRef.current = now; diff --git a/technical-documentation/architecture/preview.md b/technical-documentation/architecture/preview.md index 8fa2d6fd5..b328307e7 100644 --- a/technical-documentation/architecture/preview.md +++ b/technical-documentation/architecture/preview.md @@ -245,12 +245,18 @@ a new clip). The mapping sits in `clipIndex` + source time. Without this bridge a RAW playhead against a compressed clip list pointed at the wrong clip after a trim — wrong camera, misaligned screen. -- **Drift re-anchor.** During free-run the two clocks can drift; once the - additive error exceeds 100 ms (`Math.abs(sourceTimeSec - expectedSourceTimeSec) > 0.1`, - [`useNativePlaybackSync.ts:94`](../../src/native/useNativePlaybackSync.ts:94)) - the hook re-issues `setNativeTime`. `useNativePlaybackSync:18` calls this a - known limitation acceptable for the ~6 s fixture it's measured on; a pause - resets the drift by construction. +- **Drift, measured rather than guessed.** Every frame carries where the view was when it + composed it (`clipIndex`, `sourceTimeSec`, both transports), and + `NativeCompositorOverlay` compares that with the playhead on the one timeline both share: + programme time, the trim-compressed one, where a cut is no jump + ([`nativeSync.ts`](../../src/native/nativeSync.ts)). The gap is counted in seconds of + playback, so a speed region divides it by its speed, and the last frame is aged by the + time since it arrived, at most 250 ms, past which the view counts as stalled. A gap over + 150 ms that holds for 100 ms re-anchors the view with `setActiveClip`, at most every + 500 ms. That covers a stall of the render thread and a jump by the user while playing. + It replaced a guess from the wall clock at 1× speed, which inside a 2× speed region + re-seeked the view ten times a second; `useNativePlaybackSync` still makes that guess + for an addon that reports no position. The overlay's rect is kept aligned with the DOM via the same primitives used elsewhere in the renderer: @@ -272,12 +278,21 @@ elsewhere in the renderer: Clip changes across the playhead boundary are atomic at the `setActiveClip(viewId, screenPath, webcamPath, webcamOffsetSec, clipIndex, sourceTimeSec)` RPC -([`compositorViewClient.ts:93`](../../src/native/compositorViewClient.ts:93)): -when the playhead crosses into a clip whose `assetId` / `webcamPath` differs -from the previous one, `NativeCompositorOverlay.tsx:175-203` pauses native across -the decoder swap, awaits `setActiveClip`, and re-reads the live transport *now* -(not from a captured `isPlaying`) before resuming — so a user pause that lands -in the middle of a clip transition is honoured, not silently undone. +([`compositorViewClient.ts`](../../src/native/compositorViewClient.ts)), and who sends it +depends on the transport: + +- **Paused** (scrub, step): `NativeCompositorOverlay` sends it whenever the playhead enters + another clip. +- **Playing**: the render thread crosses into the next clip by itself, preloading it ahead + of the cut. The overlay used to send the clip again at every cut, which made the view + seek back to a place it had just left, or drop the clip it had preloaded: a hitch at + every cut of an edited take. It now only sends it when the view is elsewhere — a jump + (a click on the timeline, followed at once) or a gap the drift watch above catches. + On a pause it also brings back a view left on another clip, before `setNativeTime` + seeks in it. +- **An addon that reports no position** is driven as before: native is paused across the + decoder swap, and the live transport is re-read *now* (not from a captured `isPlaying`) + before resuming, so a user pause that lands mid-transition is honoured. ## When the decode clock fails @@ -341,14 +356,9 @@ was thrown away. path and not for the live view. Editing playback is therefore silent against the exported file; users hear audio only when the export runs. There is no flag in this branch that re-routes live audio. -- **Long-recording scrub drift.** `useNativePlaybackSync:18` documents the - accepted-at-fixture-time drift between the app's rAF playhead and the addon's - free-run clock as a known limitation; a pause re-aligns them. A scrub further - than 100 ms past expected position triggers an explicit re-anchor; below that - the two clocks run independently until something forces a sync. Long recordings - measured at the bench in - [engineering/rendering-performance.md](../engineering/rendering-performance.md) - stay below the threshold in practice, but no systematic measurement exists. +- **Drift under 150 ms is left alone.** The view and the app's clock run independently + inside the drift watch's tolerance, and a correction is a seek, not a change of pace: a + view drifting slowly is re-anchored with a visible step rather than eased back. - **Shared textures are Windows-only.** macOS (an `IOSurface`-backed Metal texture) and Linux (a dmabuf exported from Vulkan) still read back: `sharedTexture` imports both, the native halves are not written.