From d0f3136b154544aa4f65584d1d128ee123723df8 Mon Sep 17 00:00:00 2001 From: NoahSeeger Date: Tue, 22 Sep 2026 23:40:33 +0200 Subject: [PATCH 1/5] fix: ignore hidden cursor telemetry in auto-zoom --- electron/media/cursorSidecar.test.ts | 16 ++- electron/media/cursorSidecar.ts | 1 + src/components/video-editor/types.ts | 1 + .../timeline/zoom-suggestions.test.ts | 99 +++++++++++++++++++ .../ai-edition/timeline/zoom-suggestions.ts | 45 ++++++--- src/native/contracts.ts | 1 + 6 files changed, 151 insertions(+), 12 deletions(-) diff --git a/electron/media/cursorSidecar.test.ts b/electron/media/cursorSidecar.test.ts index 1c0e46a4d..88c7ed0da 100644 --- a/electron/media/cursorSidecar.test.ts +++ b/electron/media/cursorSidecar.test.ts @@ -149,7 +149,9 @@ describe("readCursorTelemetryFile", () => { const result = await readCursorTelemetryFile(video, {}); expect(result.success).toBe(true); - expect(result.samples).toEqual([{ timeMs: 10, cx: 0.5, cy: 0.5, interactionType: "click" }]); + expect(result.samples).toEqual([ + { timeMs: 10, cx: 0.5, cy: 0.5, visible: true, interactionType: "click" }, + ]); // This projection feeds the ai-edition auto-zoom detector, whose click // candidates ARE the recorded interactions (issue #699). It used to strip // the field — locked by a test, even — and the detector never saw a click @@ -158,6 +160,18 @@ describe("readCursorTelemetryFile", () => { // the extra field. }); + it("keeps hidden cursor state for auto-zoom", async () => { + const video = path.join(dir, "hidden.mp4"); + await writeSidecar(video, { + samples: [{ timeMs: 10, cx: 0.5, cy: 0.5, visible: false, interactionType: "click" }], + }); + + const result = await readCursorTelemetryFile(video, {}); + + expect(result.success).toBe(true); + expect(result.samples[0].visible).toBe(false); + }); + it("keeps every click kind, not only the plain left click", async () => { const video = path.join(dir, "kinds.mp4"); await writeSidecar(video, { diff --git a/electron/media/cursorSidecar.ts b/electron/media/cursorSidecar.ts index a47f63f5b..0a588e0ae 100644 --- a/electron/media/cursorSidecar.ts +++ b/electron/media/cursorSidecar.ts @@ -242,6 +242,7 @@ export async function readCursorTelemetryFile( timeMs: sample.timeMs, cx: sample.cx, cy: sample.cy, + visible: sample.visible, interactionType: sample.interactionType, })), }; diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index 312c9a70d..dda4abbf7 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -263,6 +263,7 @@ export interface CursorTelemetryPoint { timeMs: number; cx: number; cy: number; + visible?: boolean; interactionType?: "move" | "click" | "double-click" | "right-click" | "middle-click" | "mouseup"; cursorType?: | "arrow" diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts index f36b1cc32..accf82f68 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts @@ -52,6 +52,105 @@ describe("detectZoomDwellCandidates", () => { }); describe("buildAutoZoomSuggestions", () => { + it("ignores a click made while the cursor is hidden", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [{ ...click(2000, 0.5, 0.5), visible: false }], + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toEqual([]); + }); + + it("ignores a visible mouse-down when the cursor hides during that drag", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(1000, 0.5, 0.5), visible: true }, + { timeMs: 1016, cx: 0.5, cy: 0.5, visible: false, interactionType: "move" }, + { timeMs: 1600, cx: 0.7, cy: 0.5, visible: false, interactionType: "mouseup" }, + ], + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toEqual([]); + }); + + it("keeps a click released before the cursor later hides", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(1000, 0.5, 0.5), visible: true }, + { timeMs: 1050, cx: 0.5, cy: 0.5, visible: true, interactionType: "mouseup" }, + { timeMs: 1400, cx: 0.5, cy: 0.5, visible: false, interactionType: "move" }, + ], + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toHaveLength(1); + expect(suggestions[0].focus).toEqual({ cx: 0.5, cy: 0.5 }); + }); + + it("keeps a later visible click after a hidden drag", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(1000, 0.5, 0.5), visible: true }, + { timeMs: 1100, cx: 0.6, cy: 0.5, visible: false, interactionType: "move" }, + { timeMs: 1600, cx: 0.7, cy: 0.5, visible: true, interactionType: "mouseup" }, + { ...click(4000, 0.2, 0.8), visible: true }, + ], + totalMs: 6000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toHaveLength(1); + expect(suggestions[0].focus).toEqual({ cx: 0.2, cy: 0.8 }); + }); + + it("does not join visible samples across a hidden interval into a dwell", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { timeMs: 1000, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 1200, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 1400, cx: 0.5, cy: 0.5, visible: false }, + { timeMs: 1800, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 2000, cx: 0.5, cy: 0.5, visible: true }, + ], + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toEqual([]); + }); + + it("ignores a dwell recorded while the cursor is hidden", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: dwell(2000, 0.5, 0.5).map((sample) => ({ + ...sample, + visible: false, + })), + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toEqual([]); + }); + + it("keeps a visible dwell beside hidden cursor samples", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + ...dwell(1000, 0.2, 0.8), + { timeMs: 2000, cx: 0.2, cy: 0.8, visible: false }, + ], + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toHaveLength(1); + expect(suggestions[0].focus.cx).toBeCloseTo(0.2); + expect(suggestions[0].focus.cy).toBeCloseTo(0.8); + }); + it("returns a centered span around each accepted dwell", () => { const telemetry = dwell(2000, 0.5, 0.5); const suggestions = buildAutoZoomSuggestions({ diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.ts b/src/lib/ai-edition/timeline/zoom-suggestions.ts index b82872267..145b892cd 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.ts @@ -30,6 +30,7 @@ function normalizeTelemetrySample( timeMs: Math.max(0, Math.min(sample.timeMs, totalMs)), cx: Math.max(0, Math.min(sample.cx, 1)), cy: Math.max(0, Math.min(sample.cy, 1)), + visible: sample.visible, interactionType: sample.interactionType, }; } @@ -92,7 +93,13 @@ export function detectZoomDwellCandidates( }); }; - for (let index = 1; index < samples.length; index += 1) { + for (let index = 0; index < samples.length; index += 1) { + if (samples[index].visible === false) { + pushRunIfDwell(runStart, index); + runStart = index + 1; + continue; + } + if (index <= runStart) continue; const prev = samples[index - 1]; const curr = samples[index]; const distance = Math.hypot(curr.cx - prev.cx, curr.cy - prev.cy); @@ -119,16 +126,32 @@ const CLICK_INTERACTION_TYPES: ReadonlySet - sample.interactionType !== undefined && CLICK_INTERACTION_TYPES.has(sample.interactionType), - ) - .map((sample) => ({ - centerTimeMs: sample.timeMs, - focus: { cx: sample.cx, cy: sample.cy }, - strength: 0, - })); + const candidates: ZoomDwellCandidate[] = []; + let pendingClick: CursorTelemetryPoint | null = null; + let hiddenDuringPress = false; + const finishPress = () => { + if (pendingClick && !hiddenDuringPress) { + candidates.push({ + centerTimeMs: pendingClick.timeMs, + focus: { cx: pendingClick.cx, cy: pendingClick.cy }, + strength: 0, + }); + } + pendingClick = null; + hiddenDuringPress = false; + }; + + for (const sample of samples) { + if (sample.interactionType && CLICK_INTERACTION_TYPES.has(sample.interactionType)) { + finishPress(); + pendingClick = sample.visible === false ? null : sample; + } else { + if (sample.visible === false) hiddenDuringPress = true; + if (sample.interactionType === "mouseup") finishPress(); + } + } + finishPress(); + return candidates; } export interface AutoZoomSuggestion { diff --git a/src/native/contracts.ts b/src/native/contracts.ts index a2115e7f6..10164ad18 100644 --- a/src/native/contracts.ts +++ b/src/native/contracts.ts @@ -27,6 +27,7 @@ export interface CursorTelemetryPoint { timeMs: number; cx: number; cy: number; + visible?: boolean; } export interface CursorRecordingSample extends CursorTelemetryPoint { From b9788d579b18a689e31c784f6e89d102cc43411a Mon Sep 17 00:00:00 2001 From: NoahSeeger Date: Wed, 23 Sep 2026 01:31:56 +0200 Subject: [PATCH 2/5] fix: honor recorded cursor visibility in compositor --- crates/compositor/src/cursor.rs | 122 +++++++++++++++++- crates/compositor/src/frame_geometry.rs | 3 + .../timeline/zoom-suggestions.test.ts | 28 ++++ .../ai-edition/timeline/zoom-suggestions.ts | 24 +++- .../testing/manual-e2e-checklist.md | 1 + 5 files changed, 166 insertions(+), 12 deletions(-) diff --git a/crates/compositor/src/cursor.rs b/crates/compositor/src/cursor.rs index fb859ab34..2c8fb019a 100644 --- a/crates/compositor/src/cursor.rs +++ b/crates/compositor/src/cursor.rs @@ -42,6 +42,9 @@ pub struct CursorTrack { /// réellement cliqué. `smoothed()` la garde telle quelle, alors que la piste lissée passe /// ailleurs à cet instant (cf. `pinned_at`). click_points: Vec<(f32, f32)>, + /// CHANGEMENTS de visibilité : `(instant, visible)`, triés. Comme `types`, c'est une + /// fonction en escalier. Une piste ancienne sans champ `visible` reste visible. + visibility: Vec<(f32, bool)>, /// CHANGEMENTS d'état du curseur : (instant, `"arrow"` / `"text"` / `"pointer"` / …), triés. /// Une fonction en escalier, pas une valeur par échantillon : l'état tient sur des secondes /// entières alors que la position est échantillonnée toutes les 33 ms (~30 Hz, cf. @@ -135,6 +138,15 @@ impl CursorTrack { samples: Vec<(f32, f32, f32)>, clicks: Vec, types: Vec<(f32, String)>, + ) -> CursorTrack { + CursorTrack::new_with_visibility(samples, clicks, types, Vec::new()) + } + + fn new_with_visibility( + samples: Vec<(f32, f32, f32)>, + clicks: Vec, + types: Vec<(f32, String)>, + visibility: Vec<(f32, bool)>, ) -> CursorTrack { let follow_samples = smooth_follow_samples(&samples); let click_points = clicks @@ -146,11 +158,23 @@ impl CursorTrack { follow_samples, clicks, click_points, + visibility, types, view_cache: Default::default(), } } + /// Visibilité enregistrée au temps `t`. Les anciens sidecars qui n'ont pas ce champ sont + /// visibles par défaut, comme les lecteurs TypeScript. + pub fn visible_at(&self, t: f32) -> bool { + let i = self.visibility.partition_point(|(tc, _)| *tc <= t); + if i == 0 { + true + } else { + self.visibility[i - 1].1 + } + } + /// État du curseur au temps `t` : la dernière transition à `t` ou avant. `None` avant la /// première (enregistrement sans état tagué → l'appelant retombe sur la flèche). pub fn type_at(&self, t: f32) -> Option<&str> { @@ -171,6 +195,7 @@ impl CursorTrack { let mut samples = Vec::new(); let mut clicks = Vec::new(); let mut types: Vec<(f32, String)> = Vec::new(); + let mut visibility: Vec<(f32, bool)> = Vec::new(); let end = offset_ms + dur_s * 1000.0; for s in arr { let tm = s["timeMs"].as_f64().unwrap_or(-1.0); @@ -181,7 +206,13 @@ impl CursorTrack { let cx = s["cx"].as_f64().unwrap_or(0.0) as f32; let cy = s["cy"].as_f64().unwrap_or(0.0) as f32; samples.push((t, cx, cy)); - if s["interactionType"].as_str() == Some("click") { + let visible = s.get("visible").and_then(|value| value.as_bool()).unwrap_or(true); + if visibility.last().map(|(_, previous)| *previous) != Some(visible) { + // La première valeur de la fenêtre vaut dès son début. Sans cela, un clip dont + // le curseur était déjà masqué à `offset_ms` affichait une frame fugitive. + visibility.push((if visibility.is_empty() { 0.0 } else { t }, visible)); + } + if visible && s["interactionType"].as_str() == Some("click") { clicks.push(t); } // Seules les TRANSITIONS sont retenues — voir `types`. Le helper @@ -204,7 +235,8 @@ impl CursorTrack { samples.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); clicks.sort_by(|a, b| a.partial_cmp(b).unwrap()); types.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); - Ok(CursorTrack::new(samples, clicks, types)) + visibility.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + Ok(CursorTrack::new_with_visibility(samples, clicks, types, visibility)) } /// Position lissée au temps `t`, pour le suivi auto du zoom. La télémétrie brute est @@ -411,10 +443,11 @@ impl CursorTrack { /// bruts (le bounce est temporel, pas positionnel — ne doit pas suivre le lissage). pub fn smoothed(&self, factor: f32) -> CursorTrack { if self.samples.len() < 2 || factor <= 0.0 { - return CursorTrack::new( + return CursorTrack::new_with_visibility( self.samples.clone(), self.clicks.clone(), self.types.clone(), + self.visibility.clone(), ); } const STEP_S: f32 = 1.0 / 240.0; @@ -437,8 +470,33 @@ impl CursorTrack { raw_y.push(cy); } let (stiffness, damping, mass) = cursor_spring_config(factor); - let xs = spring_smooth(&raw_x, stiffness, damping, mass, STEP_S); - let ys = spring_smooth(&raw_y, stiffness, damping, mass, STEP_S); + // Une phase masquée coupe le ressort. Au retour, le curseur repart du premier point + // réellement visible au lieu de traverser la trajectoire cachée avec du retard. + let mut xs = Vec::with_capacity(n); + let mut ys = Vec::with_capacity(n); + let mut run_start = 0; + while run_start < n { + let visible = self.visible_at(times[run_start]); + let mut run_end = run_start + 1; + while run_end < n && self.visible_at(times[run_end]) == visible { + run_end += 1; + } + xs.extend(spring_smooth( + &raw_x[run_start..run_end], + stiffness, + damping, + mass, + STEP_S, + )); + ys.extend(spring_smooth( + &raw_y[run_start..run_end], + stiffness, + damping, + mass, + STEP_S, + )); + run_start = run_end; + } let samples = times .into_iter() .zip(xs) @@ -450,7 +508,9 @@ impl CursorTrack { // cliqués restent ceux de la piste brute. CursorTrack { click_points: self.click_points.clone(), - ..CursorTrack::new(samples, self.clicks.clone(), self.types.clone()) + ..CursorTrack::new_with_visibility( + samples, self.clicks.clone(), self.types.clone(), self.visibility.clone(), + ) } } @@ -755,6 +815,56 @@ mod tests { assert_eq!(smoothed.type_at(0.7), Some("text")); } + #[test] + fn load_preserves_hidden_intervals_and_ignores_hidden_clicks() { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let path = std::env::temp_dir().join(format!( + "openscreen-cursor-visibility-{}-{}.json", std::process::id(), unique + )); + std::fs::write( + &path, + r#"{"samples":[ + {"timeMs":0,"cx":0.1,"cy":0.1,"visible":true}, + {"timeMs":100,"cx":0.2,"cy":0.2,"visible":false,"interactionType":"click"}, + {"timeMs":500,"cx":0.8,"cy":0.8,"visible":false}, + {"timeMs":600,"cx":0.9,"cy":0.9,"visible":true,"interactionType":"click"} + ]}"#, + ) + .expect("write temp sidecar"); + let path_str = path.to_str().expect("utf-8 temp path"); + let track = CursorTrack::load(path_str, 0.0, 1.0).expect("load sidecar"); + let clipped = CursorTrack::load(path_str, 200.0, 0.5).expect("load clipped sidecar"); + let _ = std::fs::remove_file(&path); + + assert!(track.visible_at(0.0)); + assert!(!track.visible_at(0.1)); + assert!(!track.visible_at(0.59)); + assert!(track.visible_at(0.6)); + assert_eq!(track.clicks, vec![0.6], "le clic masqué ne doit pas animer le curseur"); + assert!(!clipped.visible_at(0.0), "la fenêtre commence dans la phase masquée"); + assert!(clipped.visible_at(0.4), "la transition visible garde son temps relatif"); + + let smoothed = track.smoothed(0.5); + assert!(!smoothed.visible_at(0.3), "le lissage garde la phase masquée"); + assert!(smoothed.visible_at(0.6)); + let reappeared = smoothed.at(0.6).expect("position at reappearance"); + assert!( + (reappeared.0 - 0.9).abs() < 0.02 && (reappeared.1 - 0.9).abs() < 0.02, + "le ressort redémarre au point visible, sans traîne: {reappeared:?}" + ); + } + + #[test] + fn legacy_tracks_without_visibility_stay_visible() { + let track = CursorTrack::new(vec![(0.0, 0.1, 0.1), (1.0, 0.9, 0.9)], vec![], vec![]); + assert!(track.visible_at(0.0)); + assert!(track.visible_at(0.5)); + assert!(track.visible_at(10.0)); + } + /// JSON null et une clé `cursorType` absente resetent vers la flèche, /// au lieu de garder le dernier `pointer`/`text`. #[test] diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index d098b2268..beda24f0f 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -3451,6 +3451,9 @@ pub fn cursor_alpha( if !scene.map(|s| s.cursor.show).unwrap_or(cfg.cursor) { return 0.0; } + if !track.visible_at(t) { + return 0.0; + } let idle_alpha = track.opacity_at(t, live.cursor_auto_hide); let zoom_alpha = match scene { Some(s) => { diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts index accf82f68..6569abb66 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts @@ -107,6 +107,34 @@ describe("buildAutoZoomSuggestions", () => { expect(suggestions[0].focus).toEqual({ cx: 0.2, cy: 0.8 }); }); + it("keeps a legacy click without mouseup when the cursor hides much later", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(1000, 0.2, 0.8), visible: true }, + { timeMs: 1050, cx: 0.3, cy: 0.7, visible: true, interactionType: "move" }, + { timeMs: 2000, cx: 0.7, cy: 0.3, visible: false, interactionType: "move" }, + ], + totalMs: 4000, + existingRegions: [], + defaultDurationMs: 1000, + }); + expect(suggestions).toHaveLength(1); + expect(suggestions[0].focus).toEqual({ cx: 0.2, cy: 0.8 }); + }); + + it("rejects a legacy click when the cursor hides on the next sampler tick", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(1000, 0.2, 0.8), visible: true }, + { timeMs: 1033, cx: 0.25, cy: 0.75, visible: false, interactionType: "move" }, + ], + totalMs: 4000, + existingRegions: [], + defaultDurationMs: 1000, + }); + expect(suggestions).toEqual([]); + }); + it("does not join visible samples across a hidden interval into a dwell", () => { const suggestions = buildAutoZoomSuggestions({ cursorTelemetry: [ diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.ts b/src/lib/ai-edition/timeline/zoom-suggestions.ts index 145b892cd..26c7f0946 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.ts @@ -127,10 +127,20 @@ const CLICK_INTERACTION_TYPES: ReadonlySet sample.interactionType === "mouseup"); + // PipeWire and older sidecars have clicks but no mouse-up events. A hidden sample on the next + // sampler tick still identifies a slider press; a much later hidden interval must not erase an + // already completed click merely because that format cannot mark its release. + const LEGACY_PRESS_GRACE_MS = 150; + let pendingClick: CursorTelemetryPoint | null = null; - let hiddenDuringPress = false; + let hiddenAtMs: number | null = null; const finishPress = () => { - if (pendingClick && !hiddenDuringPress) { + if (!pendingClick) return; + const hiddenDuringPress = + hiddenAtMs !== null && + (hasMouseupTelemetry || hiddenAtMs - pendingClick.timeMs <= LEGACY_PRESS_GRACE_MS); + if (!hiddenDuringPress) { candidates.push({ centerTimeMs: pendingClick.timeMs, focus: { cx: pendingClick.cx, cy: pendingClick.cy }, @@ -138,17 +148,19 @@ function detectZoomClickCandidates(samples: CursorTelemetryPoint[]): ZoomDwellCa }); } pendingClick = null; - hiddenDuringPress = false; + hiddenAtMs = null; }; for (const sample of samples) { if (sample.interactionType && CLICK_INTERACTION_TYPES.has(sample.interactionType)) { finishPress(); pendingClick = sample.visible === false ? null : sample; - } else { - if (sample.visible === false) hiddenDuringPress = true; - if (sample.interactionType === "mouseup") finishPress(); + continue; + } + if (pendingClick && sample.visible === false && hiddenAtMs === null) { + hiddenAtMs = sample.timeMs; } + if (sample.interactionType === "mouseup") finishPress(); } finishPress(); return candidates; diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index ea959a411..760959666 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -610,6 +610,7 @@ The mask comes from the native compositor (ONNX Runtime + the vendored selfie-se | Date | Build / tag | Platform | Pass/fail | Notes | |------|-------------|----------|-----------|-------| +| 2026-09-23 | dev `fix/725-hidden-cursor-auto-zoom`; rebuilt compositor addon SHA256 `a0bcd61e17084c010141049a9978121e45be2985449ff1618ab6a3145f8381f6` | Windows 11 Home 26200 | Partial — issue #725 slider slice passed | Human-driven real recording in the Electron dev build: moving a slider whose application cursor becomes hidden kept the reconstructed cursor hidden in the Studio recording and did not create a click-driven automatic zoom for that press. A visible control click still produced the normal automatic zoom. Sidecar inspection independently confirmed the hidden interval (`visible: false`) and that no zoom was centred on the press immediately preceding it. Recording, stop and editor open passed. Not covered: export, packaged build, audio/webcam, tray/HUD, other editor features, macOS or Linux. | | 2026-09-12 | dev `fix/643-gif-export-cancel`, based on `fbe461e9`; rebuilt compositor addon SHA256 `128c89fe65a459b3f17721d610ea9a78e0d3e980e9eb0b5fb2b0e5d018619e15` | Windows | Partial — export cancellation passed | Computer-use OS input, real Electron and native save dialog, imported synthetic 8 s source. GIF 15 FPS / Medium (720p) / loop on: observed 55/120 frames, clicked Cancel while rendering (pending snapshot 104/120), returned to the same options; no final or partial output remained. Retried in that dialog to the same path and received Saved to; resulting GIF is 1280×720, 120 frames, 8,200,202 bytes, fully decodes with ffmpeg (exit 0). Automated real-addon tests separately pass existing-destination preservation, GIF retry, ordinary GIF and MP4 export. Desktop adapter required screenshot-based coordinates and Ctrl+O to reach the editor; HUD reachability is not claimed. An unrelated process error window partly obscured the retry success screenshot; file verification independently passed. Skipped: recording, audio/mic, webcam, tray/HUD, effect/caption comparisons, second viewer, macOS/Linux and release promotion. Task-local profile and evidence retained; test app stopped. | | 2026-07-31 | dev build, `claude/e2e-tests-v1-8-0-474894` (e9578f09) | macOS 26.5, M1 | Partial — 1 defect | Ran launch/HUD, media, modifier anchoring, and export. **Defect: a dangling asset blanks the preview.** Modifier anchoring across a reorder verified in preview and in the exported frames. macOS export produced 1280×720 h264 + AAC at ~2× realtime. Chat sections skipped: no AI provider configured. HUD drag not runnable under computer-use (drop point is the desktop). | | 2026-08-13 | installed `v1.9.5-rc.1` | Windows 11 26200, 1920×1080 @ 100% | Partial — 1 defect | Ran launch/HUD, source selection, recording, stop, editor open. Fragmented MP4 confirmed on the shipped artifact: 48 `moof`+`mdat` pairs over 47.6s, `mvex` present, `mfra` on clean stop. **Defect: a recording that survives a helper kill is thrown away by the app** — killing `wgc-capture.exe` mid-recording leaves a fully decodable 41s file (2460 packets, `ffmpeg -f null -` exit 0) with no `.session.json` and no `.cursor.json`, and stop answers "The recording could not be saved". Fixed in #363, re-verified end to end. Truncation ablation at 60%: plain MP4 unreadable, fragmented plays 29s. **A dev build cannot test any of this** — the prebuilt worktree helper predated the change and silently ran the old path. Editor/export/chat sections not run. | From d0926341766c0b28d8a2a9f75ddd364d5ef0ab39 Mon Sep 17 00:00:00 2001 From: NoahSeeger Date: Wed, 23 Sep 2026 01:52:29 +0200 Subject: [PATCH 3/5] fix: preserve cursor visibility at clip offsets --- crates/compositor/src/cursor.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/compositor/src/cursor.rs b/crates/compositor/src/cursor.rs index 2c8fb019a..3375e712c 100644 --- a/crates/compositor/src/cursor.rs +++ b/crates/compositor/src/cursor.rs @@ -195,7 +195,19 @@ impl CursorTrack { let mut samples = Vec::new(); let mut clicks = Vec::new(); let mut types: Vec<(f32, String)> = Vec::new(); - let mut visibility: Vec<(f32, bool)> = Vec::new(); + let mut visibility_at_offset = true; + let mut visibility_sample_time = f64::NEG_INFINITY; + // A clipped track inherits the last recorded state at its source offset. Using the first + // later sample would reveal a cursor early when that sample is the transition to visible. + for s in arr { + let tm = s["timeMs"].as_f64().unwrap_or(-1.0); + if tm >= 0.0 && tm <= offset_ms && tm >= visibility_sample_time { + visibility_sample_time = tm; + visibility_at_offset = + s.get("visible").and_then(|value| value.as_bool()).unwrap_or(true); + } + } + let mut visibility = vec![(0.0, visibility_at_offset)]; let end = offset_ms + dur_s * 1000.0; for s in arr { let tm = s["timeMs"].as_f64().unwrap_or(-1.0); @@ -208,9 +220,7 @@ impl CursorTrack { samples.push((t, cx, cy)); let visible = s.get("visible").and_then(|value| value.as_bool()).unwrap_or(true); if visibility.last().map(|(_, previous)| *previous) != Some(visible) { - // La première valeur de la fenêtre vaut dès son début. Sans cela, un clip dont - // le curseur était déjà masqué à `offset_ms` affichait une frame fugitive. - visibility.push((if visibility.is_empty() { 0.0 } else { t }, visible)); + visibility.push((t, visible)); } if visible && s["interactionType"].as_str() == Some("click") { clicks.push(t); @@ -829,7 +839,6 @@ mod tests { r#"{"samples":[ {"timeMs":0,"cx":0.1,"cy":0.1,"visible":true}, {"timeMs":100,"cx":0.2,"cy":0.2,"visible":false,"interactionType":"click"}, - {"timeMs":500,"cx":0.8,"cy":0.8,"visible":false}, {"timeMs":600,"cx":0.9,"cy":0.9,"visible":true,"interactionType":"click"} ]}"#, ) @@ -845,6 +854,7 @@ mod tests { assert!(track.visible_at(0.6)); assert_eq!(track.clicks, vec![0.6], "le clic masqué ne doit pas animer le curseur"); assert!(!clipped.visible_at(0.0), "la fenêtre commence dans la phase masquée"); + assert!(!clipped.visible_at(0.39), "l'état masqué tient jusqu'à la transition"); assert!(clipped.visible_at(0.4), "la transition visible garde son temps relatif"); let smoothed = track.smoothed(0.5); From a7f3c32d748086b8ced46cbbddc1c510914427dd Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Sun, 27 Sep 2026 13:38:21 +0200 Subject: [PATCH 4/5] fix(compositor): hide every cursor layer while hidden, not a visible click's impact The recorded visibility now gates `plan_cursor` itself, which every backend and drawing path goes through: flat sprite, 3D model and its shadow, click rings, motion-blur trail. It no longer gates `cursor_alpha`, which also weights the click impact on the screen plane: a visible click whose pointer an app hides on the next tick (dragging a numeric field) snapped the tilted screen back from ~1.8 degrees in one frame. Clicks made while hidden are still dropped at load. The trail no longer samples a hidden phase: on the first frame after the pointer reappeared it drew up to 16 copies from where it had been hidden. `cursor.rs` is back to rustfmt-clean. --- crates/compositor/src/cursor.rs | 52 ++++++++++---- crates/compositor/src/frame_geometry.rs | 93 +++++++++++++++++++++++-- 2 files changed, 128 insertions(+), 17 deletions(-) diff --git a/crates/compositor/src/cursor.rs b/crates/compositor/src/cursor.rs index 3375e712c..d9c104a82 100644 --- a/crates/compositor/src/cursor.rs +++ b/crates/compositor/src/cursor.rs @@ -142,7 +142,7 @@ impl CursorTrack { CursorTrack::new_with_visibility(samples, clicks, types, Vec::new()) } - fn new_with_visibility( + pub(crate) fn new_with_visibility( samples: Vec<(f32, f32, f32)>, clicks: Vec, types: Vec<(f32, String)>, @@ -203,8 +203,10 @@ impl CursorTrack { let tm = s["timeMs"].as_f64().unwrap_or(-1.0); if tm >= 0.0 && tm <= offset_ms && tm >= visibility_sample_time { visibility_sample_time = tm; - visibility_at_offset = - s.get("visible").and_then(|value| value.as_bool()).unwrap_or(true); + visibility_at_offset = s + .get("visible") + .and_then(|value| value.as_bool()) + .unwrap_or(true); } } let mut visibility = vec![(0.0, visibility_at_offset)]; @@ -218,7 +220,10 @@ impl CursorTrack { let cx = s["cx"].as_f64().unwrap_or(0.0) as f32; let cy = s["cy"].as_f64().unwrap_or(0.0) as f32; samples.push((t, cx, cy)); - let visible = s.get("visible").and_then(|value| value.as_bool()).unwrap_or(true); + let visible = s + .get("visible") + .and_then(|value| value.as_bool()) + .unwrap_or(true); if visibility.last().map(|(_, previous)| *previous) != Some(visible) { visibility.push((t, visible)); } @@ -246,7 +251,9 @@ impl CursorTrack { clicks.sort_by(|a, b| a.partial_cmp(b).unwrap()); types.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); visibility.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); - Ok(CursorTrack::new_with_visibility(samples, clicks, types, visibility)) + Ok(CursorTrack::new_with_visibility( + samples, clicks, types, visibility, + )) } /// Position lissée au temps `t`, pour le suivi auto du zoom. La télémétrie brute est @@ -519,7 +526,10 @@ impl CursorTrack { CursorTrack { click_points: self.click_points.clone(), ..CursorTrack::new_with_visibility( - samples, self.clicks.clone(), self.types.clone(), self.visibility.clone(), + samples, + self.clicks.clone(), + self.types.clone(), + self.visibility.clone(), ) } } @@ -832,7 +842,9 @@ mod tests { .map(|d| d.as_nanos()) .unwrap_or(0); let path = std::env::temp_dir().join(format!( - "openscreen-cursor-visibility-{}-{}.json", std::process::id(), unique + "openscreen-cursor-visibility-{}-{}.json", + std::process::id(), + unique )); std::fs::write( &path, @@ -852,13 +864,29 @@ mod tests { assert!(!track.visible_at(0.1)); assert!(!track.visible_at(0.59)); assert!(track.visible_at(0.6)); - assert_eq!(track.clicks, vec![0.6], "le clic masqué ne doit pas animer le curseur"); - assert!(!clipped.visible_at(0.0), "la fenêtre commence dans la phase masquée"); - assert!(!clipped.visible_at(0.39), "l'état masqué tient jusqu'à la transition"); - assert!(clipped.visible_at(0.4), "la transition visible garde son temps relatif"); + assert_eq!( + track.clicks, + vec![0.6], + "le clic masqué ne doit pas animer le curseur" + ); + assert!( + !clipped.visible_at(0.0), + "la fenêtre commence dans la phase masquée" + ); + assert!( + !clipped.visible_at(0.39), + "l'état masqué tient jusqu'à la transition" + ); + assert!( + clipped.visible_at(0.4), + "la transition visible garde son temps relatif" + ); let smoothed = track.smoothed(0.5); - assert!(!smoothed.visible_at(0.3), "le lissage garde la phase masquée"); + assert!( + !smoothed.visible_at(0.3), + "le lissage garde la phase masquée" + ); assert!(smoothed.visible_at(0.6)); let reappeared = smoothed.at(0.6).expect("position at reappearance"); assert!( diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index beda24f0f..cad782a6f 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -3451,9 +3451,6 @@ pub fn cursor_alpha( if !scene.map(|s| s.cursor.show).unwrap_or(cfg.cursor) { return 0.0; } - if !track.visible_at(t) { - return 0.0; - } let idle_alpha = track.opacity_at(t, live.cursor_auto_hide); let zoom_alpha = match scene { Some(s) => { @@ -3482,6 +3479,9 @@ pub fn cursor_plane_point(cut: [f32; 4], uv_max: [f32; 2], p: (f32, f32)) -> Opt /// /// - curseur visible (`cursor_alpha`) : `cursor.show` explicite, la piste étant chargée même /// curseur masqué (le focus auto la suit) ; +/// - curseur caché par l'application (`visible: false`) : ses clics ne sont pas chargés +/// (`CursorTrack::load`). Un clic visible va au bout même si le pointeur disparaît juste après +/// (un glisser sur un champ numérique) : le couper là ferait sauter l'écran en plein creux ; /// - clics dans la fenêtre source du clip actif seulement (cf. `click_impact`) ; /// - vitesse : poids `clamp(2 − vitesse, 0, 1)`. À 100× une frame couvre 3,3 s de source, la /// courbe serait échantillonnée une fois, au hasard : une secousse d'une frame ; @@ -3523,6 +3523,11 @@ fn click_impact_at( pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option { let (rw, rh) = (input.render_px[0], input.render_px[1]); + // Caché par l'application (`visible: false`) : rien de ce plan n'est dessiné, ni le sprite, + // ni le modèle et son ombre, ni l'anneau d'un clic. Les trois backends passent tous par ici. + if !input.track.visible_at(input.t) { + return None; + } let alpha = cursor_alpha(input.scene, input.cfg, &input.live, input.track, input.t); if alpha <= 0.001 { return None; @@ -3613,12 +3618,18 @@ pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option Option = (0..=90).map(|i| (i as f32 / 30.0, 0.6, 0.3)).collect(); + let track = |visibility| -> &'static crate::cursor::CursorTrack { + Box::leak(Box::new(crate::cursor::CursorTrack::new_with_visibility( + samples.clone(), + vec![1.45], + vec![], + visibility, + ))) + }; + let json = zoomed_golden_scene_json() + .replace(r#""rotation":"none""#, r#""rotation":"iso""#) + .replace(r#""cursor":{"#, r#""cursor":{"clickImpact":true,"#); + let scene = Scene::from_json(&json).expect("scène"); + let tilt = |track| { + plan_frame(&FrameGeometryInput { cursor: Some(track), ..golden_input(&scene, &cfg) }) + .zoom_rotation_dyn + }; + let on = tilt(track(vec![])); + assert!(on[1] > 0.75 * crate::regions::CLICK_IMPACT_DEG, "garde : {on:?}"); + assert_eq!(tilt(track(vec![(0.0, true), (1.48, false)])), on); + } + /// L'orbite en focus auto passe par `plan_frame` : l'écran n'est pas incliné, la caméra vise et tourne /// avec le pointeur lu dans le RECADRAGE, la boîte zoome sur son centre sans glisser, la mise au /// point suit la visée, un clic fait reculer l'œil sans presser l'écran, et curseur masqué @@ -8349,6 +8387,51 @@ mod tests { assert_eq!(plan(false).for_backend(true).taps, plan(false).taps, "le sprite plat garde sa traînée"); } + /// Caché par l'application (`visible: false`), le curseur ne dessine rien, sprite plat comme + /// modèle (ombre et anneau compris), et sa traînée ne remonte pas dans la phase cachée à la + /// réapparition : il y filait à droite, il réapparaît là où il avait cliqué. + #[test] + fn a_hidden_phase_draws_no_cursor_and_no_trail_into_it() { + let scene = model_scene(); + let cfg = crate::config::all().pop().expect("cfg"); + let fg = full_frame_geometry(); + let track = crate::cursor::CursorTrack::new_with_visibility( + vec![(0.0, 0.3, 0.6), (1.0, 0.3, 0.6), (1.1, 0.35, 0.6), (1.467, 0.9, 0.6), (1.5, 0.3, 0.6), (3.0, 0.3, 0.6)], + vec![1.0], + vec![], + vec![(0.0, true), (1.1, false), (1.5, true)], + ); + let plan = |model3d: bool, t: f32| { + plan_cursor( + &fg, + &CursorPlanInput { + render_px: [1920.0, 1080.0], + u_max: 1.0, + v_max: 1.0, + cfg: &cfg, + live: LiveParams { + cursor_model3d: model3d, + cursor_bounce_scale: MODEL_CLICK_BOUNCE_REF, + cursor_motion_blur: 1.0, + ..LiveParams::default() + }, + scene: Some(&scene), + track: &track, + t, + }, + ) + }; + for model3d in [false, true] { + assert!(plan(model3d, 1.05).is_some(), "model3d {model3d} : visible avant"); + for t in [1.1, 1.2, 1.4, 1.49] { + assert!(plan(model3d, t).is_none(), "model3d {model3d} : dessiné caché à {t} s"); + } + let back = plan(model3d, 1.505).expect("réapparu"); + assert_eq!(back.taps, 1, "model3d {model3d} : traînée venue de la phase cachée"); + } + assert!(!plan(true, 1.08).expect("visible").impacts.is_empty(), "garde : l'anneau du clic"); + } + /// Sous un angle fixe, l'écran tourne autour du centre de sa boîte ZOOMÉE, réduit de son /// containment : le curseur y perdait ~18 % au point de focus même, pendant que le zoom /// grossissait l'image. Il y garde sa taille à plat, à la perspective de biais près ; ailleurs, From a2acc2104eed48efc524054cbb310d0ec0914d8e Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Sun, 27 Sep 2026 13:38:28 +0200 Subject: [PATCH 5/5] fix(auto-zoom): judge a press without its own mouse-up by the grace window The Windows and macOS samplers fold a release into the press sample when both land in one tick, so a quick tap has no mouse-up even in a take that records them. It stayed "pressed" until the next click, and any later hidden sample (typing, leaving the display) dropped it from the suggestions. A press ended by its mouse-up is judged over the whole press; one without is judged by the 150 ms grace, as PipeWire takes already were. A mouse-up sample's own visibility no longer counts: it is sampled after the release. --- .../timeline/zoom-suggestions.test.ts | 33 +++++++++++++++++++ .../ai-edition/timeline/zoom-suggestions.ts | 26 +++++++-------- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts index 6569abb66..3cff335ef 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts @@ -135,6 +135,39 @@ describe("buildAutoZoomSuggestions", () => { expect(suggestions).toEqual([]); }); + it("keeps a tap whose release shares its press sample when the cursor hides later", () => { + // The take records mouse-ups, but the second tap was released within one sampler tick. + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(500, 0.8, 0.2), visible: true }, + { timeMs: 600, cx: 0.8, cy: 0.2, visible: true, interactionType: "mouseup" }, + { ...click(3000, 0.2, 0.8), visible: true }, + { timeMs: 3500, cx: 0.2, cy: 0.8, visible: false, interactionType: "move" }, + ], + totalMs: 6000, + existingRegions: [], + defaultDurationMs: 1000, + }); + expect(suggestions.map((suggestion) => suggestion.focus)).toEqual([ + { cx: 0.8, cy: 0.2 }, + { cx: 0.2, cy: 0.8 }, + ]); + }); + + it("keeps a click whose mouse-up sample is already hidden", () => { + // Released, then typing hid the pointer before the next sampler tick. + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(1000, 0.5, 0.5), visible: true }, + { timeMs: 1033, cx: 0.5, cy: 0.5, visible: false, interactionType: "mouseup" }, + ], + totalMs: 4000, + existingRegions: [], + defaultDurationMs: 1000, + }); + expect(suggestions.map((suggestion) => suggestion.focus)).toEqual([{ cx: 0.5, cy: 0.5 }]); + }); + it("does not join visible samples across a hidden interval into a dwell", () => { const suggestions = buildAutoZoomSuggestions({ cursorTelemetry: [ diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.ts b/src/lib/ai-edition/timeline/zoom-suggestions.ts index 26c7f0946..3e8418015 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.ts @@ -127,19 +127,19 @@ const CLICK_INTERACTION_TYPES: ReadonlySet sample.interactionType === "mouseup"); - // PipeWire and older sidecars have clicks but no mouse-up events. A hidden sample on the next - // sampler tick still identifies a slider press; a much later hidden interval must not erase an - // already completed click merely because that format cannot mark its release. - const LEGACY_PRESS_GRACE_MS = 150; + // A press without its own mouse-up sample has no known release: PipeWire and older sidecars + // record none, and the Windows and macOS samplers fold a release into the press sample when both + // land in one tick. A hidden sample on the next sampler tick still identifies a slider press; a + // much later hidden interval (typing, leaving the display) must not erase a completed click. + const UNRELEASED_PRESS_GRACE_MS = 150; let pendingClick: CursorTelemetryPoint | null = null; let hiddenAtMs: number | null = null; - const finishPress = () => { + const finishPress = (released: boolean) => { if (!pendingClick) return; const hiddenDuringPress = hiddenAtMs !== null && - (hasMouseupTelemetry || hiddenAtMs - pendingClick.timeMs <= LEGACY_PRESS_GRACE_MS); + (released || hiddenAtMs - pendingClick.timeMs <= UNRELEASED_PRESS_GRACE_MS); if (!hiddenDuringPress) { candidates.push({ centerTimeMs: pendingClick.timeMs, @@ -153,16 +153,16 @@ function detectZoomClickCandidates(samples: CursorTelemetryPoint[]): ZoomDwellCa for (const sample of samples) { if (sample.interactionType && CLICK_INTERACTION_TYPES.has(sample.interactionType)) { - finishPress(); + finishPress(false); pendingClick = sample.visible === false ? null : sample; - continue; - } - if (pendingClick && sample.visible === false && hiddenAtMs === null) { + } else if (sample.interactionType === "mouseup") { + // Sampled after the release: its visibility says nothing about the press. + finishPress(true); + } else if (pendingClick && sample.visible === false && hiddenAtMs === null) { hiddenAtMs = sample.timeMs; } - if (sample.interactionType === "mouseup") finishPress(); } - finishPress(); + finishPress(false); return candidates; }