From 52f4f709c86ff98ccca7208488871b8e9a5178f3 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Fri, 2 Oct 2026 10:24:54 +0200 Subject: [PATCH 1/2] fix(preview): stop re-seeking the native view at every cut and in speed regions While playing, the native view runs its own clock and crosses clip boundaries by itself, preloading the next clip ahead of the cut. Two things kept steering it anyway: - The overlay re-sent the clip at every cut, with a pause around it. The view then sought back to a place it had already passed, or threw away the clip it had preloaded and opened it synchronously: a hitch at every cut, which on a transcript-edited take means every few seconds. - The playback sync guessed drift from the wall clock at 1x speed. Inside a 2x speed region that guess was 100 ms off every 100 ms, and the view was re-seeked about ten times a second. Every frame now carries where the view was when it composed it (clip index, source time, both transports). The overlay compares that with the playhead on the trim-compressed programme timeline, where a cut is no jump: a cut crossed in play is left to the view, a jump is followed at once, and a gap over 150 ms that holds for 100 ms re-anchors the view, at most every 500 ms. The wall-clock guess is gone. An addon that reports no position is driven as before. --- crates/compositor-view-napi/src/lib.rs | 12 +- crates/compositor/src/live.rs | 45 +++- crates/compositor/src/shared_frames.rs | 12 +- .../services/compositorViewService.ts | 2 + electron/native/compositor-view/addon.d.ts | 8 + .../NativeCompositorOverlay.test.tsx | 195 ++++++++++++++++++ .../ai-edition/NativeCompositorOverlay.tsx | 110 +++++++++- src/native/contracts.ts | 7 + src/native/hooks/useNativeCompositorView.ts | 4 + src/native/nativeSync.test.ts | 115 +++++++++++ src/native/nativeSync.ts | 120 +++++++++++ src/native/useNativePlaybackSync.ts | 33 +-- .../architecture/preview.md | 47 +++-- 13 files changed, 646 insertions(+), 64 deletions(-) create mode 100644 src/components/ai-edition/NativeCompositorOverlay.test.tsx create mode 100644 src/native/nativeSync.test.ts create mode 100644 src/native/nativeSync.ts diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs index 338beaa80..905a6a5e5 100644 --- a/crates/compositor-view-napi/src/lib.rs +++ b/crates/compositor-view-napi/src/lib.rs @@ -204,6 +204,10 @@ pub struct FramePacket { /// Le métrage passe de ses coins à l'image par leur homographie (caméra réelle), pas par /// leur interpolation bilinéaire. pub footage_projective: bool, + /// Clip actif de la scène quand la frame a été composée (`FramePosition`). + pub clip_index: u32, + /// Temps source de la frame écran composée, en secondes. + pub source_time_sec: f64, } /// Renvoie la dernière frame readback du thread de rendu SI elle est plus récente que @@ -243,7 +247,7 @@ pub fn read_frame(id: i32, since_gen: f64) -> Result> { v.latest_frame_since(since_gen.max(0.0) as u64) } }; - Ok(slot.map(|(gen, w, h, pixels, footage)| { + Ok(slot.map(|(gen, w, h, pixels, footage, position)| { debug_assert_eq!(pixels.len(), (w as usize) * (h as usize) * 4); FramePacket { gen: gen as f64, @@ -252,6 +256,8 @@ pub fn read_frame(id: i32, since_gen: f64) -> Result> { data: Buffer::from(pixels), footage: footage_corners(footage), footage_projective: footage.is_some_and(|q| q.projective), + clip_index: position.clip_index, + source_time_sec: position.source_time_sec, } })) } @@ -276,6 +282,8 @@ pub struct SharedFramePacket { pub height: u32, pub footage: Option>, pub footage_projective: bool, + pub clip_index: u32, + pub source_time_sec: f64, } /// Livrer les frames de la vue par textures partagées plutôt que par `read_frame`. Rend @@ -314,6 +322,8 @@ pub fn read_shared_frame(id: i32, since_gen: f64) -> Result Scene { /// Dernière frame readback vers CPU, prête pour le napi `read_frame`. /// -/// `(gen, w, h, vec)` où `vec.len() == w*h*4` octets RGBA8 tightly-packed (R, G, B, A -/// en mémoire — cf. `Compositor::readback_resized`). `gen` est une génération monotone +/// `(gen, w, h, vec, métrage, position)` où `vec.len() == w*h*4` octets RGBA8 tightly-packed +/// (R, G, B, A en mémoire — cf. `Compositor::readback_resized`). `gen` est une génération monotone /// (≥ 1, `0` réservé à « le consommateur n'a encore rien vu ») incrémentée à CHAQUE /// publication, càd uniquement quand une nouvelle frame a réellement été composée (le /// thread de rendu ne republie pas une frame identique — cf. `stepped || first`). Elle /// est l'IDENTITÉ de la frame : le consommateur (`read_frame`) ne repaie le clone + l'IPC /// que lorsqu'elle change. `None` = "aucune frame composée pour l'instant" (toutes les /// lectures avant la 1re frame composée retournent `None` côté napi, jamais un buffer vide). -pub type LatestFrame = (u64, u32, u32, Vec, Option); +pub type LatestFrame = ( + u64, + u32, + u32, + Vec, + Option, + FramePosition, +); + +/// Où en est la vue quand elle compose une frame : le clip actif de la scène et le temps +/// source de la frame écran. Voyage avec chaque frame publiée, quel que soit le transport, +/// pour que l'app compare la position RÉELLE de la vue à sa propre tête de lecture au lieu +/// de la deviner à partir de l'horloge murale. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FramePosition { + /// Index du clip actif dans `scene.clips`. + pub clip_index: u32, + /// pts de la frame écran composée, en secondes du fichier source. + pub source_time_sec: f64, +} /// État partagé thread appelant → thread de rendu (commandes sans blocage). struct Shared { @@ -1840,7 +1859,11 @@ unsafe fn render_thread( if stepped || first || publish_pending { if pw > 0 && ph > 0 { - match publish_shared(&shared, &gpu, &comp, &mut ring) { + let position = FramePosition { + clip_index: active_clip_index as u32, + source_time_sec: player.screen_time_sec(), + }; + match publish_shared(&shared, &gpu, &comp, &mut ring, position) { SharedPublish::Published => { first = false; publish_pending = false; @@ -1874,7 +1897,7 @@ unsafe fn render_thread( // l'ancienne dérivation tant que le slot n'est pas vidé. let next_gen = shared.frame_gen.fetch_add(1, Ordering::Relaxed) + 1; if let Ok(mut slot) = shared.latest_frame.lock() { - *slot = Some((next_gen, rw, rh, rgba, comp.footage_quad())); + *slot = Some((next_gen, rw, rh, rgba, comp.footage_quad(), position)); } first = false; publish_pending = false; @@ -1918,6 +1941,7 @@ unsafe fn publish_shared( gpu: &Gpu, comp: &Compositor, ring: &mut Ring, + position: FramePosition, ) -> SharedPublish { if !shared.shared_frames.load(Ordering::Relaxed) { return SharedPublish::Off; @@ -1967,6 +1991,7 @@ unsafe fn publish_shared( width, height, footage: comp.footage_quad(), + position, }); } SharedPublish::Published @@ -1976,7 +2001,13 @@ unsafe fn publish_shared( } #[cfg(not(windows))] -unsafe fn publish_shared(_: &Shared, _: &Gpu, _: &Compositor, _: &mut Ring) -> SharedPublish { +unsafe fn publish_shared( + _: &Shared, + _: &Gpu, + _: &Compositor, + _: &mut Ring, + _: FramePosition, +) -> SharedPublish { SharedPublish::Off } @@ -2113,7 +2144,7 @@ pub fn run_standalone(screen: &str, webcam: &str, cursor_json: &str) -> Result<( // standalone n'affiche pas réellement les pixels ici (l'embed Electron est // le consumer réel). On imprime juste une frame de temps en temps pour // confirmer que la chaîne fonctionne. - if let Some((_gen, fw, fh, _pixels, _)) = view.latest_frame() { + if let Some((_gen, fw, fh, _pixels, _, _)) = view.latest_frame() { if (fw, fh) != (w, h) { // garde-fou : la staging de readback suit `set_rect` côté thread // de rendu, donc ce serait une désynchro transitoire — acceptable. diff --git a/crates/compositor/src/shared_frames.rs b/crates/compositor/src/shared_frames.rs index 421897131..a6b4f8415 100644 --- a/crates/compositor/src/shared_frames.rs +++ b/crates/compositor/src/shared_frames.rs @@ -13,6 +13,7 @@ //! macOS (IOSurface) et Linux (dmabuf) restent sur le readback. use crate::frame_geometry::FootageQuad; +use crate::live::FramePosition; use std::time::{Duration, Instant}; /// Cases de l'anneau : une prête, une en transit vers le renderer (tenue jusqu'à ce que @@ -37,6 +38,7 @@ pub struct SharedFrame { pub width: u32, pub height: u32, pub footage: Option, + pub position: FramePosition, } /// Une case prise par JS : la génération qu'elle porte et quand elle est partie. @@ -244,7 +246,15 @@ mod tests { use super::*; fn frame(gen: u64, slot: u32) -> SharedFrame { - SharedFrame { gen, slot, handle: 0, width: 2, height: 2, footage: None } + SharedFrame { + gen, + slot, + handle: 0, + width: 2, + height: 2, + footage: None, + position: FramePosition { clip_index: 0, source_time_sec: 0.0 }, + } } /// Toutes les cases prises par JS, la case `n` portant la génération `n + 1`. diff --git a/electron/native-bridge/services/compositorViewService.ts b/electron/native-bridge/services/compositorViewService.ts index 51e562979..6dbfb1461 100644 --- a/electron/native-bridge/services/compositorViewService.ts +++ b/electron/native-bridge/services/compositorViewService.ts @@ -703,6 +703,8 @@ export class CompositorViewService { height: frame.height, footage: frame.footage ?? null, footageProjective: frame.footageProjective ?? false, + clipIndex: frame.clipIndex, + sourceTimeSec: frame.sourceTimeSec, }; const api = this.sharedTextureApi(); let imported: Electron.SharedTextureImported | undefined; diff --git a/electron/native/compositor-view/addon.d.ts b/electron/native/compositor-view/addon.d.ts index 631d70fdf..e87b830b5 100644 --- a/electron/native/compositor-view/addon.d.ts +++ b/electron/native/compositor-view/addon.d.ts @@ -39,6 +39,10 @@ export interface NativeFramePacket { footage?: number[] | null; /** The footage maps from those corners by their homography (real camera), not bilinearly. */ footageProjective?: boolean; + /** Where the view was when it composed this frame: the active clip in `scene.clips` and the + * screen frame's time in its source file. Absent from an older `.node`. */ + clipIndex?: number; + sourceTimeSec?: number; } /** A preview frame left in a shared GPU texture instead of copied into RAM (Windows, hardware @@ -53,6 +57,10 @@ export interface NativeSharedFramePacket { height: number; footage?: number[] | null; footageProjective?: boolean; + /** Where the view was when it composed this frame: the active clip in `scene.clips` and the + * screen frame's time in its source file. Absent from an older `.node`. */ + clipIndex?: number; + sourceTimeSec?: number; } export interface ExportStats { diff --git a/src/components/ai-edition/NativeCompositorOverlay.test.tsx b/src/components/ai-edition/NativeCompositorOverlay.test.tsx new file mode 100644 index 000000000..43d389e62 --- /dev/null +++ b/src/components/ai-edition/NativeCompositorOverlay.test.tsx @@ -0,0 +1,195 @@ +// @vitest-environment jsdom +/** + * How the overlay steers the native view while the transport plays. + * + * The view runs its own clock and crosses clip boundaries by itself. The overlay used to + * re-send the clip at every cut anyway, which made the view seek back to a place it had just + * left, or drop the next clip it had preloaded: a hitch at every cut of an edited take. Now it + * reads where the view is from the position each frame reports, and only steps in when the + * view is somewhere else. + */ +import { act, cleanup, render } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AxcutClip, AxcutDocument } from "@/lib/ai-edition/schema"; +import { axcutSchemaVersion } from "@/lib/ai-edition/schema"; +import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { publishNativePosition } from "@/native/nativeSync"; + +const native = vi.hoisted(() => ({ + setActiveClip: vi.fn(async () => ({ ok: true })), + setNativePlaying: vi.fn(), +})); + +vi.mock("@/native", () => ({ + pushAllNativeParams: vi.fn(), + setActiveClip: native.setActiveClip, + setCurrentNativeViewId: vi.fn(), + setNativePlaying: native.setNativePlaying, + setNativeScene: vi.fn(), + subscribeNativeCompositor: () => () => undefined, + useIsCpuCompositor: () => false, + useNativeCompositorView: () => ({ viewId: 7, error: null }), +})); + +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: () => (key: string) => key, +})); + +import { NativeCompositorOverlay } from "./NativeCompositorOverlay"; + +/** One take cut in two: source 0-5 s, then 7-12 s after a 2 s cut. */ +const FIRST: AxcutClip = { + id: "clip_a", + assetId: "asset_take", + sourceStartSec: 0, + sourceEndSec: 5, + timelineStartSec: 0, + timelineEndSec: 5, + wordRefs: [], + origin: "system", + reason: "", +}; +const SECOND: AxcutClip = { + ...FIRST, + id: "clip_b", + sourceStartSec: 7, + sourceEndSec: 12, + timelineStartSec: 5, + timelineEndSec: 10, +}; + +function makeDocument(): AxcutDocument { + return { + schemaVersion: axcutSchemaVersion, + project: { + id: "proj_sync", + title: "Sync", + createdAt: "2026-10-02T08:00:00.000Z", + updatedAt: "2026-10-02T08:00:00.000Z", + primaryAssetId: "asset_take", + }, + assets: [ + { + id: "asset_take", + kind: "video", + label: "take", + originalPath: "/take.mp4", + cameraTrack: null, + }, + ], + transcript: null, + transcripts: [], + timeline: { + clips: [FIRST, SECOND], + gaps: [], + trimRanges: [], + muteRanges: [], + speedRanges: [], + captionRanges: [], + }, + annotations: [], + zoomRanges: [], + audioTracks: [], + legacyEditor: null, + }; +} + +function setPlayhead(currentTimeSec: number, playing: boolean) { + act(() => { + useProjectStore.setState({ currentTimeSec, playing }); + }); +} + +describe("NativeCompositorOverlay while playing", () => { + let now = 1000; + + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(performance, "now").mockImplementation(() => now); + useProjectStore.setState({ + projectId: "proj_sync", + document: makeDocument(), + revision: 1, + status: "ready", + error: null, + sourceDurationSec: 12, + currentTimeSec: 4.9, + playing: true, + dirty: false, + lastSavedAt: new Date(), + }); + }); + + afterEach(() => { + cleanup(); + vi.restoreAllMocks(); + publishNativePosition(null); + useProjectStore.getState().clear(); + }); + + async function mountAtFirstClip() { + render(); + // The first mount sends the starting clip, and its reply resumes the view: let both + // settle, so only what follows is under test. + await act(async () => { + await Promise.resolve(); + }); + native.setActiveClip.mockClear(); + native.setNativePlaying.mockClear(); + } + + it("lets the view cross a cut by itself", async () => { + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.99 }, now); + + setPlayhead(5.02, true); + + expect(native.setActiveClip).not.toHaveBeenCalled(); + expect(native.setNativePlaying).not.toHaveBeenCalled(); + }); + + it("follows a jump to another clip at once, without pausing the view", async () => { + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.9 }, now); + + setPlayhead(8, true); + + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 1, 10); + expect(native.setNativePlaying).not.toHaveBeenCalledWith(false); + }); + + it("re-anchors a view that stays behind, once the gap holds", async () => { + await mountAtFirstClip(); + // The view stalled 0.4 s behind the playhead, inside the first clip. + publishNativePosition({ clipIndex: 0, sourceTimeSec: 2 }, now); + setPlayhead(2.4, true); + expect(native.setActiveClip).not.toHaveBeenCalled(); + + now += 120; + publishNativePosition({ clipIndex: 0, sourceTimeSec: 2.12 }, now); + setPlayhead(2.52, true); + + expect(native.setActiveClip).toHaveBeenCalledTimes(1); + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 0, 2.52); + }); + + it("still sends the clip on a change while paused", async () => { + useProjectStore.setState({ playing: false }); + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.9 }, now); + + setPlayhead(6, false); + + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 1, 8); + }); + + // An addon that reports no position cannot be read, so it is driven as before. + it("pauses the view across the swap when it reports no position", async () => { + await mountAtFirstClip(); + + setPlayhead(5.02, true); + + expect(native.setNativePlaying).toHaveBeenCalledWith(false); + expect(native.setActiveClip).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/components/ai-edition/NativeCompositorOverlay.tsx b/src/components/ai-edition/NativeCompositorOverlay.tsx index e4af64102..e98e9e45b 100644 --- a/src/components/ai-edition/NativeCompositorOverlay.tsx +++ b/src/components/ai-edition/NativeCompositorOverlay.tsx @@ -15,6 +15,14 @@ import { useIsCpuCompositor, useNativeCompositorView, } from "@/native"; +import { + type DriftWatch, + getNativePosition, + IDLE_DRIFT_WATCH, + NATIVE_DRIFT_TOLERANCE_SEC, + nativeLeadSec, + watchDrift, +} from "@/native/nativeSync"; import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription"; import { getWebcamNativeSize, @@ -186,9 +194,18 @@ export function NativeCompositorOverlay() { const pendingTargetClipIdRef = useRef(null); const playing = useProjectStore((s) => s.playing); + const wasPlayingRef = useRef(playing); - // Change les décodeurs screen/webcam uniquement quand le playhead entre dans un autre clip. + // Change les décodeurs screen/webcam quand le playhead entre dans un autre clip, à l'arrêt. + // + // En lecture, la vue enchaîne seule les clips, préchargement compris. Lui renvoyer le clip à + // chaque coupe la faisait rechercher une position qu'elle avait déjà, ou jeter le + // préchargement qu'elle s'apprêtait à utiliser : un recul de quelques images et un arrêt à + // chaque coupe. Elle n'est recalée que si elle est réellement ailleurs (guet de dérive + // ci-dessous), d'après la position que porte chaque frame. useEffect(() => { + const justPaused = wasPlayingRef.current && !playing; + wasPlayingRef.current = playing; if ( viewId === null || !document || @@ -199,7 +216,13 @@ export function NativeCompositorOverlay() { ) { return; } - if (previousActiveClipIdRef.current === activeClipId) { + const native = getNativePosition(); + const clipChanged = previousActiveClipIdRef.current !== activeClipId; + // Mise en pause pendant que la vue était sur un autre clip que la tête de lecture : elle + // y retourne, sans quoi le recalage en temps (`setNativeTime`) chercherait dans le + // mauvais fichier. + const nativeElsewhere = justPaused && native !== null && native.clipIndex !== activeClipIndex; + if (!clipChanged && !nativeElsewhere) { return; } const asset = document.assets.find((candidate) => candidate.id === activeClip.assetId); @@ -210,13 +233,30 @@ export function NativeCompositorOverlay() { const targetClipId = activeClipId; // Sonde de fluidité (diagnostic) : sépare les mesures d'avant et d'après un // franchissement de clip, qui se sont déjà révélées non comparables. - noteUiProbeClipSwitch(previousActiveClipIdRef.current, activeClipId); + if (clipChanged) { + noteUiProbeClipSwitch(previousActiveClipIdRef.current, activeClipId); + } pendingTargetClipIdRef.current = targetClipId; previousActiveClipIdRef.current = targetClipId; + if (playing && native !== null) { + const lead = nativeLeadSec( + native, + { clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec }, + nativeClips, + performance.now(), + ); + // Une coupe franchie en lecture : la vue la franchit d'elle-même, au même endroit du + // programme. Un saut (clic ailleurs sur la timeline) la met loin : suivi tout de suite, + // sans attendre le délai du guet de dérive. + if (lead !== null && Math.abs(lead) <= NATIVE_DRIFT_TOLERANCE_SEC) { + return; + } + } - // Pause native across the decoder swap. Deliberately NOT kept in a variable to - // resume from later — see the `.then` below, which re-reads the live transport. - if (playing) { + // Sans position rapportée (addon antérieur), la vue est encore pilotée comme avant : en + // pause pendant le changement de décodeurs. Deliberately NOT kept in a variable to resume + // from later — see the `.then` below, which re-reads the live transport. + if (playing && native === null) { setNativePlaying(false); } @@ -248,7 +288,63 @@ export function NativeCompositorOverlay() { previousActiveClipIdRef.current = null; } }); - }, [viewId, document, activeClipId, activeClip, activeClipIndex, activeSourceTimeSec, playing]); + }, [ + viewId, + document, + activeClipId, + activeClip, + activeClipIndex, + activeSourceTimeSec, + playing, + nativeClips, + ]); + + // Guet de dérive, en lecture : la vue tourne sur sa propre horloge, et n'est recalée que si + // la position que porte sa dernière frame s'écarte de la tête de lecture et le reste. Couvre + // un saut de l'utilisateur pendant la lecture comme un blocage du thread de rendu. L'ancien + // recalage devinait la dérive à l'horloge murale, à vitesse 1 : dans une région à 2× il + // relançait une recherche dix fois par seconde. + const driftRef = useRef(IDLE_DRIFT_WATCH); + useEffect(() => { + if ( + viewId === null || + !document || + !playing || + !activeClip || + activeClipIndex === null || + activeSourceTimeSec === null + ) { + driftRef.current = { ...driftRef.current, outSinceMs: null }; + return; + } + const now = performance.now(); + const lead = nativeLeadSec( + getNativePosition(), + { clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec }, + nativeClips, + now, + ); + const { watch, resync } = watchDrift(driftRef.current, lead, now); + driftRef.current = watch; + if (!resync) { + return; + } + const asset = document.assets.find((candidate) => candidate.id === activeClip.assetId); + if (!asset?.originalPath) { + return; + } + const camera = assetCameraSource(asset); + setActiveClip( + viewId, + asset.originalPath, + camera.path, + camera.offsetSec, + activeClipIndex, + activeSourceTimeSec, + ).catch((error: unknown) => { + console.warn("[compositor-view] re-anchoring the preview failed:", error); + }); + }, [viewId, document, playing, activeClip, activeClipIndex, activeSourceTimeSec, nativeClips]); if (!ready) { return null; diff --git a/src/native/contracts.ts b/src/native/contracts.ts index d13a95f5c..5421b994a 100644 --- a/src/native/contracts.ts +++ b/src/native/contracts.ts @@ -165,6 +165,10 @@ export interface CompositorFramePacket { footage?: number[] | null; /** The footage maps from those corners by their homography (real camera), not bilinearly. */ footageProjective?: boolean; + /** Where the view was when it composed this frame: the active clip in the scene's clips and + * the screen frame's time in its source file. Absent from an older addon. */ + clipIndex?: number; + sourceTimeSec?: number; } /** What travels with a preview frame handed over as a shared GPU texture (Windows): all a @@ -177,6 +181,9 @@ export interface CompositorSharedFrameMeta { height: number; footage: number[] | null; footageProjective: boolean; + /** See {@link CompositorFramePacket.clipIndex}. */ + clipIndex?: number; + sourceTimeSec?: number; } /** `readFrame`'s answer for a frame sent as a shared texture. The texture reached diff --git a/src/native/hooks/useNativeCompositorView.ts b/src/native/hooks/useNativeCompositorView.ts index 94688c2bf..e11ec5938 100644 --- a/src/native/hooks/useNativeCompositorView.ts +++ b/src/native/hooks/useNativeCompositorView.ts @@ -37,6 +37,7 @@ import { } from "../compositorViewClient"; import type { CompositorParamValue, CompositorViewRect } from "../contracts"; import { publishFootageQuad } from "../footageQuadStore"; +import { publishNativePosition } from "../nativeSync"; import { computeDeviceRect, rectsEqual } from "../nativeViewRect"; export interface UseNativeCompositorViewOptions { @@ -245,6 +246,7 @@ export function useNativeCompositorView( sharedTransport = true; lastGen = Math.max(lastGen, meta.gen); publishFootageQuad(meta.footage, meta.footageProjective); + publishNativePosition(meta); const fresh = canvas.dataset.painted === undefined; const drawn = paint(meta.gen, meta.width, meta.height, () => ctx.drawImage(frame, 0, 0)); if (drawn && fresh && !sharedChecked) { @@ -314,6 +316,7 @@ export function useNativeCompositorView( } // Where the footage lies in this frame: a privacy blur's gimbal follows it. publishFootageQuad(frame.footage, frame.footageProjective); + publishNativePosition(frame); // Wrap the received buffer DIRECTLY — no intermediate copy. `data` is a // fresh per-frame Buffer from IPC (never pooled or reused across frames), // so a view over it is valid for the lifetime of this paint, and nothing @@ -399,6 +402,7 @@ export function useNativeCompositorView( disposed = true; unsubscribeShared(); publishFootageQuad(null); + publishNativePosition(null); if (rectRafHandle !== 0) { cancelAnimationFrame(rectRafHandle); } diff --git a/src/native/nativeSync.test.ts b/src/native/nativeSync.test.ts new file mode 100644 index 000000000..224dd067f --- /dev/null +++ b/src/native/nativeSync.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it } from "vitest"; +import { + getNativePosition, + IDLE_DRIFT_WATCH, + NATIVE_DRIFT_PERSIST_MS, + NATIVE_RESYNC_COOLDOWN_MS, + nativeLeadSec, + programmeTimeSec, + publishNativePosition, + watchDrift, +} from "./nativeSync"; + +/** Two segments of one recording around a cut: source 0-5 s, then 7-12 s from programme 5 s. */ +const SEGMENTS = [ + { timelineStartSec: 0, sourceStartSec: 0 }, + { timelineStartSec: 5, sourceStartSec: 7 }, +]; + +describe("programmeTimeSec", () => { + it("places a source time on the trim-compressed timeline", () => { + expect(programmeTimeSec(SEGMENTS, { clipIndex: 1, sourceTimeSec: 8 })).toBe(6); + }); + + it("has no answer for a clip the layout does not have", () => { + expect(programmeTimeSec(SEGMENTS, { clipIndex: 4, sourceTimeSec: 8 })).toBeNull(); + }); +}); + +describe("nativeLeadSec", () => { + // Across a cut the source time jumps by the trimmed 2 s; on the programme it does not. + it("sees no gap when the view and the playhead sit on either side of a cut", () => { + const lead = nativeLeadSec( + { clipIndex: 0, sourceTimeSec: 4.99, receivedAtMs: 1000 }, + { clipIndex: 1, sourceTimeSec: 7.02 }, + SEGMENTS, + 1000, + ); + expect(lead).toBeCloseTo(-0.03, 5); + }); + + it("ages the view's last frame by the time since it arrived", () => { + const lead = nativeLeadSec( + { clipIndex: 1, sourceTimeSec: 8, receivedAtMs: 1000 }, + { clipIndex: 1, sourceTimeSec: 8.05 }, + SEGMENTS, + 1050, + ); + expect(lead).toBeCloseTo(0, 5); + }); + + it("measures a view left behind by a stall", () => { + const lead = nativeLeadSec( + { clipIndex: 0, sourceTimeSec: 2, receivedAtMs: 1000 }, + { clipIndex: 0, sourceTimeSec: 2.4 }, + SEGMENTS, + 1000, + ); + expect(lead).toBeCloseTo(-0.4, 5); + }); + + it("says nothing without a reported position", () => { + expect(nativeLeadSec(null, { clipIndex: 0, sourceTimeSec: 1 }, SEGMENTS, 0)).toBeNull(); + }); +}); + +describe("watchDrift", () => { + it("leaves a view within tolerance alone", () => { + const { resync, watch } = watchDrift(IDLE_DRIFT_WATCH, 0.1, 1000); + expect(resync).toBe(false); + expect(watch.outSinceMs).toBeNull(); + }); + + // Right at a cut, or after an edit re-indexes the clips, the gap lasts a frame or two. + it("re-anchors only a gap that holds", () => { + let state = watchDrift(IDLE_DRIFT_WATCH, 0.5, 1000); + expect(state.resync).toBe(false); + state = watchDrift(state.watch, 0.5, 1000 + NATIVE_DRIFT_PERSIST_MS - 1); + expect(state.resync).toBe(false); + state = watchDrift(state.watch, 0.5, 1000 + NATIVE_DRIFT_PERSIST_MS); + expect(state.resync).toBe(true); + }); + + it("forgets a gap that closed before it held", () => { + let state = watchDrift(IDLE_DRIFT_WATCH, 0.5, 1000); + state = watchDrift(state.watch, 0.01, 1050); + state = watchDrift(state.watch, 0.5, 1100); + expect(state.resync).toBe(false); + }); + + it("waits for a re-anchor to land before the next", () => { + let state = watchDrift(IDLE_DRIFT_WATCH, 0.5, 0); + state = watchDrift(state.watch, 0.5, NATIVE_DRIFT_PERSIST_MS); + expect(state.resync).toBe(true); + const at = NATIVE_DRIFT_PERSIST_MS; + state = watchDrift(state.watch, 0.5, at + 10); + state = watchDrift(state.watch, 0.5, at + NATIVE_RESYNC_COOLDOWN_MS - 1); + expect(state.resync).toBe(false); + state = watchDrift(state.watch, 0.5, at + NATIVE_RESYNC_COOLDOWN_MS); + expect(state.resync).toBe(true); + }); +}); + +describe("publishNativePosition", () => { + it("keeps the position a frame reports, with when it arrived", () => { + publishNativePosition({ clipIndex: 2, sourceTimeSec: 3.5 }, 42); + expect(getNativePosition()).toEqual({ clipIndex: 2, sourceTimeSec: 3.5, receivedAtMs: 42 }); + }); + + // An addon that predates positions must not be steered on a stale one. + it("forgets the position when a frame reports none", () => { + publishNativePosition({ clipIndex: 2, sourceTimeSec: 3.5 }, 42); + publishNativePosition({}); + expect(getNativePosition()).toBeNull(); + }); +}); diff --git a/src/native/nativeSync.ts b/src/native/nativeSync.ts new file mode 100644 index 000000000..84539572a --- /dev/null +++ b/src/native/nativeSync.ts @@ -0,0 +1,120 @@ +/** + * Where the native preview view is, against where the app's playhead is. + * + * While playing, the view runs its own clock and crosses clip boundaries by itself, so the + * app does not drive it frame by frame. It used to guess whether the view had drifted from + * the wall clock, assuming 1× speed: inside a 2× speed region the guess was 100 ms off every + * 100 ms, and the view was re-seeked ten times a second. Each frame now says where the view + * really is (`clipIndex`, `sourceTimeSec`), and this compares that with the playhead on the + * one timeline both share: programme time, the trim-compressed one, where a cut is no jump. + */ + +/** A segment of the trim-compressed programme, as `resolveVisibleClips` lays them out. */ +export interface ProgrammeSegment { + timelineStartSec: number; + sourceStartSec: number; +} + +/** Where a frame is: the active clip in the scene and the frame's time in its source file. */ +export interface NativePosition { + clipIndex: number; + sourceTimeSec: number; +} + +/** The position the view last reported, and when it reached the renderer. */ +export interface ReportedNativePosition extends NativePosition { + receivedAtMs: number; +} + +/** Beyond this the picture is visibly off the sound: re-anchor the view. Lip sync is noticed + * from ~45 ms when the picture leads and ~125 ms when it lags (ITU-R BT.1359); frames reach + * the renderer up to a frame or two late, so a tighter bound would chase measurement noise. */ +export const NATIVE_DRIFT_TOLERANCE_SEC = 0.15; +/** The gap must hold this long: right at a cut the view and the playhead cross it a frame or + * two apart, and a scene edit re-indexes the clips for a frame before both agree again. */ +export const NATIVE_DRIFT_PERSIST_MS = 100; +/** After a re-anchor the view needs a seek and a first frame before it reports the new place. */ +export const NATIVE_RESYNC_COOLDOWN_MS = 500; + +/** `sourceTimeSec` of segment `clipIndex`, on the programme timeline. `null` for a segment the + * layout does not have (the scene and the document briefly disagree after an edit). */ +export function programmeTimeSec( + segments: readonly ProgrammeSegment[], + position: NativePosition, +): number | null { + const segment = segments[position.clipIndex]; + if (!segment) { + return null; + } + return segment.timelineStartSec + (position.sourceTimeSec - segment.sourceStartSec); +} + +/** How far ahead of the playhead the view is now, in programme seconds (negative: behind). + * Its last frame is aged by the wall time since it arrived. `null` when either position + * cannot be placed. */ +export function nativeLeadSec( + native: ReportedNativePosition | null, + app: NativePosition, + segments: readonly ProgrammeSegment[], + nowMs: number, +): number | null { + if (!native) { + return null; + } + const nativeSec = programmeTimeSec(segments, native); + const appSec = programmeTimeSec(segments, app); + if (nativeSec === null || appSec === null) { + return null; + } + return nativeSec + Math.max(0, nowMs - native.receivedAtMs) / 1000 - appSec; +} + +export interface DriftWatch { + /** Since when the view has been out of tolerance, `null` while it is in. */ + outSinceMs: number | null; + lastResyncMs: number; +} + +export const IDLE_DRIFT_WATCH: DriftWatch = { + outSinceMs: null, + lastResyncMs: Number.NEGATIVE_INFINITY, +}; + +/** Whether to re-anchor the view now, given its lead. Only for a gap that holds, and not again + * before the last re-anchor has had time to land. */ +export function watchDrift( + watch: DriftWatch, + leadSec: number | null, + nowMs: number, +): { watch: DriftWatch; resync: boolean } { + if (leadSec === null || Math.abs(leadSec) <= NATIVE_DRIFT_TOLERANCE_SEC) { + return { watch: { ...watch, outSinceMs: null }, resync: false }; + } + const outSinceMs = watch.outSinceMs ?? nowMs; + if ( + nowMs - outSinceMs < NATIVE_DRIFT_PERSIST_MS || + nowMs - watch.lastResyncMs < NATIVE_RESYNC_COOLDOWN_MS + ) { + return { watch: { ...watch, outSinceMs }, resync: false }; + } + return { watch: { outSinceMs: null, lastResyncMs: nowMs }, resync: true }; +} + +let reported: ReportedNativePosition | null = null; + +/** Records where the view's latest frame is, as it reaches the renderer. `null` forgets it: the + * view is gone, or its addon predates positions — then nothing here may steer it. */ +export function publishNativePosition( + position: Partial | null | undefined, + receivedAtMs = performance.now(), +): void { + reported = + position && typeof position.clipIndex === "number" && typeof position.sourceTimeSec === "number" + ? { clipIndex: position.clipIndex, sourceTimeSec: position.sourceTimeSec, receivedAtMs } + : null; +} + +/** The position the view last reported, `null` before its first frame or without one. */ +export function getNativePosition(): ReportedNativePosition | null { + return reported; +} diff --git a/src/native/useNativePlaybackSync.ts b/src/native/useNativePlaybackSync.ts index 64e877b63..3ade18269 100644 --- a/src/native/useNativePlaybackSync.ts +++ b/src/native/useNativePlaybackSync.ts @@ -13,9 +13,10 @@ * over free-run). So discrete seeks are only sent while *paused* — i.e. real * scrub/step interactions. Pausing also re-snaps native to the app playhead. * - * Known POC limitation: during free-run the native clock and the app clock can - * drift (independent tickers); acceptable for the fixture (~6 s loop). A pause - * re-aligns them. + * While playing, the two clocks are compared by `NativeCompositorOverlay`, from the + * position each native frame reports (`nativeSync.ts`), not here: this hook used to + * guess the drift from the wall clock at 1× speed, and re-seeked the view ten times a + * second inside a 2× speed region. */ import { useEffect, useMemo, useRef, useSyncExternalStore } from "react"; import type { AxcutClip } from "@/lib/ai-edition/schema"; @@ -57,43 +58,19 @@ export function useNativePlaybackSync( setNativePlaying(playing); }, [active, playing]); - // Scrub/step while paused OR periodic resync during playback when drift > 100ms - const lastSyncedSourceTimeRef = useRef(null); - const lastSyncedWallTimeRef = useRef(0); + // Scrub/step while paused. A clip change is `setActiveClip`'s, in the overlay. const lastActiveClipIdRef = useRef(null); useEffect(() => { if (!active || sourceTimeSec === null || !activeClipId) { return; } - const now = performance.now(); - - // When clip changes, let setActiveClip handle the atomic clip-switch-and-seek. if (lastActiveClipIdRef.current !== activeClipId) { lastActiveClipIdRef.current = activeClipId; - lastSyncedSourceTimeRef.current = sourceTimeSec; - lastSyncedWallTimeRef.current = now; return; } - if (!playing) { setNativeTime(sourceTimeSec); - lastSyncedSourceTimeRef.current = sourceTimeSec; - lastSyncedWallTimeRef.current = now; - return; - } - // While playing: periodically verify master clock alignment to prevent drift - if (lastSyncedSourceTimeRef.current === null || lastSyncedWallTimeRef.current === 0) { - lastSyncedSourceTimeRef.current = sourceTimeSec; - lastSyncedWallTimeRef.current = now; - return; - } - const wallElapsedSec = (now - lastSyncedWallTimeRef.current) / 1000; - const expectedSourceTimeSec = lastSyncedSourceTimeRef.current + wallElapsedSec; - if (Math.abs(sourceTimeSec - expectedSourceTimeSec) > 0.1) { - setNativeTime(sourceTimeSec); - lastSyncedSourceTimeRef.current = sourceTimeSec; - lastSyncedWallTimeRef.current = now; } }, [active, playing, activeClipId, sourceTimeSec]); } diff --git a/technical-documentation/architecture/preview.md b/technical-documentation/architecture/preview.md index 8fa2d6fd5..6df1823e1 100644 --- a/technical-documentation/architecture/preview.md +++ b/technical-documentation/architecture/preview.md @@ -245,12 +245,15 @@ a new clip). The mapping sits in `clipIndex` + source time. Without this bridge a RAW playhead against a compressed clip list pointed at the wrong clip after a trim — wrong camera, misaligned screen. -- **Drift re-anchor.** During free-run the two clocks can drift; once the - additive error exceeds 100 ms (`Math.abs(sourceTimeSec - expectedSourceTimeSec) > 0.1`, - [`useNativePlaybackSync.ts:94`](../../src/native/useNativePlaybackSync.ts:94)) - the hook re-issues `setNativeTime`. `useNativePlaybackSync:18` calls this a - known limitation acceptable for the ~6 s fixture it's measured on; a pause - resets the drift by construction. +- **Drift, measured rather than guessed.** Every frame carries where the view was when it + composed it (`clipIndex`, `sourceTimeSec`, both transports), and + `NativeCompositorOverlay` compares that with the playhead on the one timeline both share: + programme time, the trim-compressed one, where a cut is no jump + ([`nativeSync.ts`](../../src/native/nativeSync.ts)). A gap over 150 ms that holds for + 100 ms re-anchors the view with `setActiveClip`, at most every 500 ms. That covers a stall + of the render thread and a jump by the user while playing. It replaced a guess from the + wall clock at 1× speed, which inside a 2× speed region re-seeked the view ten times a + second. The overlay's rect is kept aligned with the DOM via the same primitives used elsewhere in the renderer: @@ -272,12 +275,21 @@ elsewhere in the renderer: Clip changes across the playhead boundary are atomic at the `setActiveClip(viewId, screenPath, webcamPath, webcamOffsetSec, clipIndex, sourceTimeSec)` RPC -([`compositorViewClient.ts:93`](../../src/native/compositorViewClient.ts:93)): -when the playhead crosses into a clip whose `assetId` / `webcamPath` differs -from the previous one, `NativeCompositorOverlay.tsx:175-203` pauses native across -the decoder swap, awaits `setActiveClip`, and re-reads the live transport *now* -(not from a captured `isPlaying`) before resuming — so a user pause that lands -in the middle of a clip transition is honoured, not silently undone. +([`compositorViewClient.ts`](../../src/native/compositorViewClient.ts)), and who sends it +depends on the transport: + +- **Paused** (scrub, step): `NativeCompositorOverlay` sends it whenever the playhead enters + another clip. +- **Playing**: the render thread crosses into the next clip by itself, preloading it ahead + of the cut. The overlay used to send the clip again at every cut, which made the view + seek back to a place it had just left, or drop the clip it had preloaded: a hitch at + every cut of an edited take. It now only sends it when the view is elsewhere — a jump + (a click on the timeline, followed at once) or a gap the drift watch above catches. + On a pause it also brings back a view left on another clip, before `setNativeTime` + seeks in it. +- **An addon that reports no position** is driven as before: native is paused across the + decoder swap, and the live transport is re-read *now* (not from a captured `isPlaying`) + before resuming, so a user pause that lands mid-transition is honoured. ## When the decode clock fails @@ -341,14 +353,9 @@ was thrown away. path and not for the live view. Editing playback is therefore silent against the exported file; users hear audio only when the export runs. There is no flag in this branch that re-routes live audio. -- **Long-recording scrub drift.** `useNativePlaybackSync:18` documents the - accepted-at-fixture-time drift between the app's rAF playhead and the addon's - free-run clock as a known limitation; a pause re-aligns them. A scrub further - than 100 ms past expected position triggers an explicit re-anchor; below that - the two clocks run independently until something forces a sync. Long recordings - measured at the bench in - [engineering/rendering-performance.md](../engineering/rendering-performance.md) - stay below the threshold in practice, but no systematic measurement exists. +- **Drift under 150 ms is left alone.** The view and the app's clock run independently + inside the drift watch's tolerance, and a correction is a seek, not a change of pace: a + view drifting slowly is re-anchored with a visible step rather than eased back. - **Shared textures are Windows-only.** macOS (an `IOSurface`-backed Metal texture) and Linux (a dmabuf exported from Vulkan) still read back: `sharedTexture` imports both, the native halves are not written. From c3dd4fbaf288dcb8df81848246afb3b5698d32b2 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Fri, 2 Oct 2026 11:52:24 +0200 Subject: [PATCH 2/2] fix(preview): count native drift in seconds of playback, and catch a frozen view The drift watch compared programme seconds and aged the last frame at 1x: inside a 16x speed region a view in step looked half a second behind and was re-seeked twice a second, and a view that froze in step with the playhead aged along with it and was never caught. The gap is now divided by the region's speed, and a frame is aged by at most 250 ms. An addon that reports no position gets its wall-clock correction back, so a jump inside the clip while playing still reaches it. --- .../NativeCompositorOverlay.test.tsx | 20 +++++ .../ai-edition/NativeCompositorOverlay.tsx | 22 ++++- src/native/nativeSync.test.ts | 33 ++++++- src/native/nativeSync.ts | 15 +++- src/native/useNativePlaybackSync.test.ts | 85 +++++++++++++++++++ src/native/useNativePlaybackSync.ts | 33 ++++++- .../architecture/preview.md | 13 +-- 7 files changed, 206 insertions(+), 15 deletions(-) create mode 100644 src/native/useNativePlaybackSync.test.ts diff --git a/src/components/ai-edition/NativeCompositorOverlay.test.tsx b/src/components/ai-edition/NativeCompositorOverlay.test.tsx index 43d389e62..619bd84b3 100644 --- a/src/components/ai-edition/NativeCompositorOverlay.test.tsx +++ b/src/components/ai-edition/NativeCompositorOverlay.test.tsx @@ -173,6 +173,26 @@ describe("NativeCompositorOverlay while playing", () => { expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 0, 2.52); }); + // At 16× a frame that takes 30 ms to arrive shows the playhead 0.48 s of programme ago. + it("leaves alone a view keeping pace inside a speed region", async () => { + useProjectStore.setState({ + document: { + ...makeDocument(), + legacyEditor: { speedRegions: [{ id: "speed_a", startMs: 0, endMs: 12000, speed: 16 }] }, + }, + currentTimeSec: 0.2, + }); + await mountAtFirstClip(); + + for (let frame = 0; frame < 8; frame++) { + now += 30; + publishNativePosition({ clipIndex: 0, sourceTimeSec: 0.2 + frame * 0.48 }, now); + setPlayhead(0.2 + (frame + 1) * 0.48, true); + } + + expect(native.setActiveClip).not.toHaveBeenCalled(); + }); + it("still sends the clip on a change while paused", async () => { useProjectStore.setState({ playing: false }); await mountAtFirstClip(); diff --git a/src/components/ai-edition/NativeCompositorOverlay.tsx b/src/components/ai-edition/NativeCompositorOverlay.tsx index e98e9e45b..d60bbcd20 100644 --- a/src/components/ai-edition/NativeCompositorOverlay.tsx +++ b/src/components/ai-edition/NativeCompositorOverlay.tsx @@ -1,9 +1,11 @@ import { useEffect, useMemo, useRef, useSyncExternalStore } from "react"; import { useScopedT } from "@/contexts/I18nContext"; +import { readSpeedRegions } from "@/lib/ai-edition/document/timeline"; import { noteUiProbeClipSwitch } from "@/lib/ai-edition/perf/uiFrameProbe"; import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { assetCameraSource } from "@/lib/ai-edition/timeline/camera"; +import { findActiveSpeedRegion, type SpeedRegion } from "@/lib/ai-edition/timeline/speed"; import { resolveNativePosition } from "@/lib/ai-edition/timeline/timelineMap"; import { pushAllNativeParams, @@ -191,6 +193,12 @@ export function NativeCompositorOverlay() { const activeClipId = activeClip?.id ?? null; const activeClipIndex = activePosition?.clipIndex ?? null; const activeSourceTimeSec = activePosition?.sourceTimeSec ?? null; + // Matched on the raw ruler, like `currentTimeSec` and the regions' pills. + const speedRegions = useMemo( + () => (document ? readSpeedRegions(document) : []), + [document], + ); + const playbackSpeed = findActiveSpeedRegion(speedRegions, currentTimeSec * 1000)?.speed ?? 1; const pendingTargetClipIdRef = useRef(null); const playing = useProjectStore((s) => s.playing); @@ -243,6 +251,7 @@ export function NativeCompositorOverlay() { native, { clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec }, nativeClips, + playbackSpeed, performance.now(), ); // Une coupe franchie en lecture : la vue la franchit d'elle-même, au même endroit du @@ -297,6 +306,7 @@ export function NativeCompositorOverlay() { activeSourceTimeSec, playing, nativeClips, + playbackSpeed, ]); // Guet de dérive, en lecture : la vue tourne sur sa propre horloge, et n'est recalée que si @@ -322,6 +332,7 @@ export function NativeCompositorOverlay() { getNativePosition(), { clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec }, nativeClips, + playbackSpeed, now, ); const { watch, resync } = watchDrift(driftRef.current, lead, now); @@ -344,7 +355,16 @@ export function NativeCompositorOverlay() { ).catch((error: unknown) => { console.warn("[compositor-view] re-anchoring the preview failed:", error); }); - }, [viewId, document, playing, activeClip, activeClipIndex, activeSourceTimeSec, nativeClips]); + }, [ + viewId, + document, + playing, + activeClip, + activeClipIndex, + activeSourceTimeSec, + nativeClips, + playbackSpeed, + ]); if (!ready) { return null; diff --git a/src/native/nativeSync.test.ts b/src/native/nativeSync.test.ts index 224dd067f..3e6842e11 100644 --- a/src/native/nativeSync.test.ts +++ b/src/native/nativeSync.test.ts @@ -3,6 +3,7 @@ import { getNativePosition, IDLE_DRIFT_WATCH, NATIVE_DRIFT_PERSIST_MS, + NATIVE_FRAME_MAX_AGE_MS, NATIVE_RESYNC_COOLDOWN_MS, nativeLeadSec, programmeTimeSec, @@ -33,6 +34,7 @@ describe("nativeLeadSec", () => { { clipIndex: 0, sourceTimeSec: 4.99, receivedAtMs: 1000 }, { clipIndex: 1, sourceTimeSec: 7.02 }, SEGMENTS, + 1, 1000, ); expect(lead).toBeCloseTo(-0.03, 5); @@ -43,23 +45,52 @@ describe("nativeLeadSec", () => { { clipIndex: 1, sourceTimeSec: 8, receivedAtMs: 1000 }, { clipIndex: 1, sourceTimeSec: 8.05 }, SEGMENTS, + 1, 1050, ); expect(lead).toBeCloseTo(0, 5); }); + // At 16× the playhead covers 0.64 s of programme in the 40 ms a frame takes to arrive and + // be read: in programme seconds that is a view far behind, to the eye it is in step. + it("measures the gap in seconds of playback inside a speed region", () => { + const lead = nativeLeadSec( + { clipIndex: 1, sourceTimeSec: 8, receivedAtMs: 1000 }, + { clipIndex: 1, sourceTimeSec: 8.64 }, + SEGMENTS, + 16, + 1030, + ); + expect(lead).toBeCloseTo(-0.01, 5); + }); + it("measures a view left behind by a stall", () => { const lead = nativeLeadSec( { clipIndex: 0, sourceTimeSec: 2, receivedAtMs: 1000 }, { clipIndex: 0, sourceTimeSec: 2.4 }, SEGMENTS, + 1, 1000, ); expect(lead).toBeCloseTo(-0.4, 5); }); + // Aging the last frame without bound would move a frozen view along with the playhead. + it("catches a view that froze in step with the playhead", () => { + const frozen = { clipIndex: 0, sourceTimeSec: 2, receivedAtMs: 1000 }; + let state = watchDrift(IDLE_DRIFT_WATCH, null, 1000); + let resyncAtMs: number | null = null; + for (let now = 1000; now <= 2000 && resyncAtMs === null; now += 16) { + const playhead = { clipIndex: 0, sourceTimeSec: 2 + (now - 1000) / 1000 }; + state = watchDrift(state.watch, nativeLeadSec(frozen, playhead, SEGMENTS, 1, now), now); + resyncAtMs = state.resync ? now : null; + } + expect(resyncAtMs).not.toBeNull(); + expect(resyncAtMs).toBeGreaterThan(1000 + NATIVE_FRAME_MAX_AGE_MS); + }); + it("says nothing without a reported position", () => { - expect(nativeLeadSec(null, { clipIndex: 0, sourceTimeSec: 1 }, SEGMENTS, 0)).toBeNull(); + expect(nativeLeadSec(null, { clipIndex: 0, sourceTimeSec: 1 }, SEGMENTS, 1, 0)).toBeNull(); }); }); diff --git a/src/native/nativeSync.ts b/src/native/nativeSync.ts index 84539572a..0db0b8bac 100644 --- a/src/native/nativeSync.ts +++ b/src/native/nativeSync.ts @@ -35,6 +35,9 @@ export const NATIVE_DRIFT_TOLERANCE_SEC = 0.15; export const NATIVE_DRIFT_PERSIST_MS = 100; /** After a re-anchor the view needs a seek and a first frame before it reports the new place. */ export const NATIVE_RESYNC_COOLDOWN_MS = 500; +/** A frame older than this is not late but stalled: aging it further would keep a frozen view + * level with the playhead for good. Two frames of the CPU compositor's ~8 fps. */ +export const NATIVE_FRAME_MAX_AGE_MS = 250; /** `sourceTimeSec` of segment `clipIndex`, on the programme timeline. `null` for a segment the * layout does not have (the scene and the document briefly disagree after an edit). */ @@ -49,13 +52,16 @@ export function programmeTimeSec( return segment.timelineStartSec + (position.sourceTimeSec - segment.sourceStartSec); } -/** How far ahead of the playhead the view is now, in programme seconds (negative: behind). - * Its last frame is aged by the wall time since it arrived. `null` when either position - * cannot be placed. */ +/** How far ahead of the playhead the view is now, in seconds of playback (negative: behind). + * Inside a speed region the programme runs `speed` times faster than playback: a frame or + * two of delay at 16× is a second of programme, and still only a frame or two to the eye. + * The last frame is aged by the wall time since it arrived, up to `NATIVE_FRAME_MAX_AGE_MS`. + * `null` when either position cannot be placed. */ export function nativeLeadSec( native: ReportedNativePosition | null, app: NativePosition, segments: readonly ProgrammeSegment[], + speed: number, nowMs: number, ): number | null { if (!native) { @@ -66,7 +72,8 @@ export function nativeLeadSec( if (nativeSec === null || appSec === null) { return null; } - return nativeSec + Math.max(0, nowMs - native.receivedAtMs) / 1000 - appSec; + const ageMs = Math.min(Math.max(0, nowMs - native.receivedAtMs), NATIVE_FRAME_MAX_AGE_MS); + return (nativeSec - appSec) / speed + ageMs / 1000; } export interface DriftWatch { diff --git a/src/native/useNativePlaybackSync.test.ts b/src/native/useNativePlaybackSync.test.ts new file mode 100644 index 000000000..4629f2130 --- /dev/null +++ b/src/native/useNativePlaybackSync.test.ts @@ -0,0 +1,85 @@ +// @vitest-environment jsdom +/** + * A jump inside the clip while playing. The overlay catches it from the position each frame + * reports; an addon that reports none leaves it to this hook, against the wall clock at 1×. + */ +import { renderHook } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AxcutClip } from "@/lib/ai-edition/schema"; +import { publishNativePosition } from "./nativeSync"; + +const store = vi.hoisted(() => ({ + setNativePlaying: vi.fn(), + setNativeTime: vi.fn(), +})); + +vi.mock("./nativeCompositorStore", () => ({ + getCurrentNativeViewId: () => 7, + setNativePlaying: store.setNativePlaying, + setNativeTime: store.setNativeTime, + subscribeNativeCompositor: () => () => undefined, +})); + +import { useNativePlaybackSync } from "./useNativePlaybackSync"; + +const CLIPS: AxcutClip[] = [ + { + id: "clip_a", + assetId: "asset_take", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + wordRefs: [], + origin: "system", + reason: "", + }, +]; + +describe("useNativePlaybackSync while playing", () => { + let now = 1000; + + beforeEach(() => { + vi.clearAllMocks(); + now = 1000; + vi.spyOn(performance, "now").mockImplementation(() => now); + }); + + afterEach(() => { + vi.restoreAllMocks(); + publishNativePosition(null); + }); + + /** Plays from `fromSec` for a few frames, the playhead in step with the wall clock. */ + function play(fromSec: number) { + const hook = renderHook(({ time }) => useNativePlaybackSync(true, time, CLIPS, CLIPS), { + initialProps: { time: fromSec }, + }); + for (let frame = 1; frame <= 3; frame++) { + now += 16; + hook.rerender({ time: fromSec + frame * 0.016 }); + } + return hook; + } + + it("seeks the view on a jump inside the clip when the addon reports no position", () => { + const hook = play(1); + expect(store.setNativeTime).not.toHaveBeenCalled(); + + now += 16; + hook.rerender({ time: 6 }); + + expect(store.setNativeTime).toHaveBeenCalledWith(6); + }); + + // The overlay watches the drift then; a guess at 1× here would re-seek inside speed regions. + it("leaves the view to the overlay when the addon reports positions", () => { + publishNativePosition({ clipIndex: 0, sourceTimeSec: 1 }, now); + const hook = play(1); + + now += 16; + hook.rerender({ time: 6 }); + + expect(store.setNativeTime).not.toHaveBeenCalled(); + }); +}); diff --git a/src/native/useNativePlaybackSync.ts b/src/native/useNativePlaybackSync.ts index 3ade18269..7ff2dd81f 100644 --- a/src/native/useNativePlaybackSync.ts +++ b/src/native/useNativePlaybackSync.ts @@ -14,9 +14,9 @@ * scrub/step interactions. Pausing also re-snaps native to the app playhead. * * While playing, the two clocks are compared by `NativeCompositorOverlay`, from the - * position each native frame reports (`nativeSync.ts`), not here: this hook used to - * guess the drift from the wall clock at 1× speed, and re-seeked the view ten times a - * second inside a 2× speed region. + * position each native frame reports (`nativeSync.ts`). This hook only guesses the drift + * from the wall clock at 1× speed for an addon that reports no position: the guess + * re-seeked the view ten times a second inside a 2× speed region. */ import { useEffect, useMemo, useRef, useSyncExternalStore } from "react"; import type { AxcutClip } from "@/lib/ai-edition/schema"; @@ -27,6 +27,7 @@ import { setNativeTime, subscribeNativeCompositor, } from "./nativeCompositorStore"; +import { getNativePosition } from "./nativeSync"; export function useNativePlaybackSync( playing: boolean, @@ -58,19 +59,43 @@ export function useNativePlaybackSync( setNativePlaying(playing); }, [active, playing]); - // Scrub/step while paused. A clip change is `setActiveClip`'s, in the overlay. + // Scrub/step while paused, and drift while playing for an addon that reports no position. + // A clip change is `setActiveClip`'s, in the overlay. + const lastSyncedSourceTimeRef = useRef(null); + const lastSyncedWallTimeRef = useRef(0); const lastActiveClipIdRef = useRef(null); useEffect(() => { if (!active || sourceTimeSec === null || !activeClipId) { return; } + const now = performance.now(); if (lastActiveClipIdRef.current !== activeClipId) { lastActiveClipIdRef.current = activeClipId; + lastSyncedSourceTimeRef.current = sourceTimeSec; + lastSyncedWallTimeRef.current = now; return; } if (!playing) { setNativeTime(sourceTimeSec); + lastSyncedSourceTimeRef.current = sourceTimeSec; + lastSyncedWallTimeRef.current = now; + return; + } + if (getNativePosition() !== null) { + return; + } + if (lastSyncedSourceTimeRef.current === null || lastSyncedWallTimeRef.current === 0) { + lastSyncedSourceTimeRef.current = sourceTimeSec; + lastSyncedWallTimeRef.current = now; + return; + } + const wallElapsedSec = (now - lastSyncedWallTimeRef.current) / 1000; + const expectedSourceTimeSec = lastSyncedSourceTimeRef.current + wallElapsedSec; + if (Math.abs(sourceTimeSec - expectedSourceTimeSec) > 0.1) { + setNativeTime(sourceTimeSec); + lastSyncedSourceTimeRef.current = sourceTimeSec; + lastSyncedWallTimeRef.current = now; } }, [active, playing, activeClipId, sourceTimeSec]); } diff --git a/technical-documentation/architecture/preview.md b/technical-documentation/architecture/preview.md index 6df1823e1..b328307e7 100644 --- a/technical-documentation/architecture/preview.md +++ b/technical-documentation/architecture/preview.md @@ -249,11 +249,14 @@ a new clip). The mapping sits in composed it (`clipIndex`, `sourceTimeSec`, both transports), and `NativeCompositorOverlay` compares that with the playhead on the one timeline both share: programme time, the trim-compressed one, where a cut is no jump - ([`nativeSync.ts`](../../src/native/nativeSync.ts)). A gap over 150 ms that holds for - 100 ms re-anchors the view with `setActiveClip`, at most every 500 ms. That covers a stall - of the render thread and a jump by the user while playing. It replaced a guess from the - wall clock at 1× speed, which inside a 2× speed region re-seeked the view ten times a - second. + ([`nativeSync.ts`](../../src/native/nativeSync.ts)). The gap is counted in seconds of + playback, so a speed region divides it by its speed, and the last frame is aged by the + time since it arrived, at most 250 ms, past which the view counts as stalled. A gap over + 150 ms that holds for 100 ms re-anchors the view with `setActiveClip`, at most every + 500 ms. That covers a stall of the render thread and a jump by the user while playing. + It replaced a guess from the wall clock at 1× speed, which inside a 2× speed region + re-seeked the view ten times a second; `useNativePlaybackSync` still makes that guess + for an addon that reports no position. The overlay's rect is kept aligned with the DOM via the same primitives used elsewhere in the renderer: