diff --git a/crates/compositor/src/cursor.rs b/crates/compositor/src/cursor.rs index fb859ab34..d9c104a82 100644 --- a/crates/compositor/src/cursor.rs +++ b/crates/compositor/src/cursor.rs @@ -42,6 +42,9 @@ pub struct CursorTrack { /// réellement cliqué. `smoothed()` la garde telle quelle, alors que la piste lissée passe /// ailleurs à cet instant (cf. `pinned_at`). click_points: Vec<(f32, f32)>, + /// CHANGEMENTS de visibilité : `(instant, visible)`, triés. Comme `types`, c'est une + /// fonction en escalier. Une piste ancienne sans champ `visible` reste visible. + visibility: Vec<(f32, bool)>, /// CHANGEMENTS d'état du curseur : (instant, `"arrow"` / `"text"` / `"pointer"` / …), triés. /// Une fonction en escalier, pas une valeur par échantillon : l'état tient sur des secondes /// entières alors que la position est échantillonnée toutes les 33 ms (~30 Hz, cf. @@ -135,6 +138,15 @@ impl CursorTrack { samples: Vec<(f32, f32, f32)>, clicks: Vec, types: Vec<(f32, String)>, + ) -> CursorTrack { + CursorTrack::new_with_visibility(samples, clicks, types, Vec::new()) + } + + pub(crate) fn new_with_visibility( + samples: Vec<(f32, f32, f32)>, + clicks: Vec, + types: Vec<(f32, String)>, + visibility: Vec<(f32, bool)>, ) -> CursorTrack { let follow_samples = smooth_follow_samples(&samples); let click_points = clicks @@ -146,11 +158,23 @@ impl CursorTrack { follow_samples, clicks, click_points, + visibility, types, view_cache: Default::default(), } } + /// Visibilité enregistrée au temps `t`. Les anciens sidecars qui n'ont pas ce champ sont + /// visibles par défaut, comme les lecteurs TypeScript. + pub fn visible_at(&self, t: f32) -> bool { + let i = self.visibility.partition_point(|(tc, _)| *tc <= t); + if i == 0 { + true + } else { + self.visibility[i - 1].1 + } + } + /// État du curseur au temps `t` : la dernière transition à `t` ou avant. `None` avant la /// première (enregistrement sans état tagué → l'appelant retombe sur la flèche). pub fn type_at(&self, t: f32) -> Option<&str> { @@ -171,6 +195,21 @@ impl CursorTrack { let mut samples = Vec::new(); let mut clicks = Vec::new(); let mut types: Vec<(f32, String)> = Vec::new(); + let mut visibility_at_offset = true; + let mut visibility_sample_time = f64::NEG_INFINITY; + // A clipped track inherits the last recorded state at its source offset. Using the first + // later sample would reveal a cursor early when that sample is the transition to visible. + for s in arr { + let tm = s["timeMs"].as_f64().unwrap_or(-1.0); + if tm >= 0.0 && tm <= offset_ms && tm >= visibility_sample_time { + visibility_sample_time = tm; + visibility_at_offset = s + .get("visible") + .and_then(|value| value.as_bool()) + .unwrap_or(true); + } + } + let mut visibility = vec![(0.0, visibility_at_offset)]; let end = offset_ms + dur_s * 1000.0; for s in arr { let tm = s["timeMs"].as_f64().unwrap_or(-1.0); @@ -181,7 +220,14 @@ impl CursorTrack { let cx = s["cx"].as_f64().unwrap_or(0.0) as f32; let cy = s["cy"].as_f64().unwrap_or(0.0) as f32; samples.push((t, cx, cy)); - if s["interactionType"].as_str() == Some("click") { + let visible = s + .get("visible") + .and_then(|value| value.as_bool()) + .unwrap_or(true); + if visibility.last().map(|(_, previous)| *previous) != Some(visible) { + visibility.push((t, visible)); + } + if visible && s["interactionType"].as_str() == Some("click") { clicks.push(t); } // Seules les TRANSITIONS sont retenues — voir `types`. Le helper @@ -204,7 +250,10 @@ impl CursorTrack { samples.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); clicks.sort_by(|a, b| a.partial_cmp(b).unwrap()); types.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); - Ok(CursorTrack::new(samples, clicks, types)) + visibility.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + Ok(CursorTrack::new_with_visibility( + samples, clicks, types, visibility, + )) } /// Position lissée au temps `t`, pour le suivi auto du zoom. La télémétrie brute est @@ -411,10 +460,11 @@ impl CursorTrack { /// bruts (le bounce est temporel, pas positionnel — ne doit pas suivre le lissage). pub fn smoothed(&self, factor: f32) -> CursorTrack { if self.samples.len() < 2 || factor <= 0.0 { - return CursorTrack::new( + return CursorTrack::new_with_visibility( self.samples.clone(), self.clicks.clone(), self.types.clone(), + self.visibility.clone(), ); } const STEP_S: f32 = 1.0 / 240.0; @@ -437,8 +487,33 @@ impl CursorTrack { raw_y.push(cy); } let (stiffness, damping, mass) = cursor_spring_config(factor); - let xs = spring_smooth(&raw_x, stiffness, damping, mass, STEP_S); - let ys = spring_smooth(&raw_y, stiffness, damping, mass, STEP_S); + // Une phase masquée coupe le ressort. Au retour, le curseur repart du premier point + // réellement visible au lieu de traverser la trajectoire cachée avec du retard. + let mut xs = Vec::with_capacity(n); + let mut ys = Vec::with_capacity(n); + let mut run_start = 0; + while run_start < n { + let visible = self.visible_at(times[run_start]); + let mut run_end = run_start + 1; + while run_end < n && self.visible_at(times[run_end]) == visible { + run_end += 1; + } + xs.extend(spring_smooth( + &raw_x[run_start..run_end], + stiffness, + damping, + mass, + STEP_S, + )); + ys.extend(spring_smooth( + &raw_y[run_start..run_end], + stiffness, + damping, + mass, + STEP_S, + )); + run_start = run_end; + } let samples = times .into_iter() .zip(xs) @@ -450,7 +525,12 @@ impl CursorTrack { // cliqués restent ceux de la piste brute. CursorTrack { click_points: self.click_points.clone(), - ..CursorTrack::new(samples, self.clicks.clone(), self.types.clone()) + ..CursorTrack::new_with_visibility( + samples, + self.clicks.clone(), + self.types.clone(), + self.visibility.clone(), + ) } } @@ -755,6 +835,74 @@ mod tests { assert_eq!(smoothed.type_at(0.7), Some("text")); } + #[test] + fn load_preserves_hidden_intervals_and_ignores_hidden_clicks() { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let path = std::env::temp_dir().join(format!( + "openscreen-cursor-visibility-{}-{}.json", + std::process::id(), + unique + )); + std::fs::write( + &path, + r#"{"samples":[ + {"timeMs":0,"cx":0.1,"cy":0.1,"visible":true}, + {"timeMs":100,"cx":0.2,"cy":0.2,"visible":false,"interactionType":"click"}, + {"timeMs":600,"cx":0.9,"cy":0.9,"visible":true,"interactionType":"click"} + ]}"#, + ) + .expect("write temp sidecar"); + let path_str = path.to_str().expect("utf-8 temp path"); + let track = CursorTrack::load(path_str, 0.0, 1.0).expect("load sidecar"); + let clipped = CursorTrack::load(path_str, 200.0, 0.5).expect("load clipped sidecar"); + let _ = std::fs::remove_file(&path); + + assert!(track.visible_at(0.0)); + assert!(!track.visible_at(0.1)); + assert!(!track.visible_at(0.59)); + assert!(track.visible_at(0.6)); + assert_eq!( + track.clicks, + vec![0.6], + "le clic masqué ne doit pas animer le curseur" + ); + assert!( + !clipped.visible_at(0.0), + "la fenêtre commence dans la phase masquée" + ); + assert!( + !clipped.visible_at(0.39), + "l'état masqué tient jusqu'à la transition" + ); + assert!( + clipped.visible_at(0.4), + "la transition visible garde son temps relatif" + ); + + let smoothed = track.smoothed(0.5); + assert!( + !smoothed.visible_at(0.3), + "le lissage garde la phase masquée" + ); + assert!(smoothed.visible_at(0.6)); + let reappeared = smoothed.at(0.6).expect("position at reappearance"); + assert!( + (reappeared.0 - 0.9).abs() < 0.02 && (reappeared.1 - 0.9).abs() < 0.02, + "le ressort redémarre au point visible, sans traîne: {reappeared:?}" + ); + } + + #[test] + fn legacy_tracks_without_visibility_stay_visible() { + let track = CursorTrack::new(vec![(0.0, 0.1, 0.1), (1.0, 0.9, 0.9)], vec![], vec![]); + assert!(track.visible_at(0.0)); + assert!(track.visible_at(0.5)); + assert!(track.visible_at(10.0)); + } + /// JSON null et une clé `cursorType` absente resetent vers la flèche, /// au lieu de garder le dernier `pointer`/`text`. #[test] diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index d098b2268..cad782a6f 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -3479,6 +3479,9 @@ pub fn cursor_plane_point(cut: [f32; 4], uv_max: [f32; 2], p: (f32, f32)) -> Opt /// /// - curseur visible (`cursor_alpha`) : `cursor.show` explicite, la piste étant chargée même /// curseur masqué (le focus auto la suit) ; +/// - curseur caché par l'application (`visible: false`) : ses clics ne sont pas chargés +/// (`CursorTrack::load`). Un clic visible va au bout même si le pointeur disparaît juste après +/// (un glisser sur un champ numérique) : le couper là ferait sauter l'écran en plein creux ; /// - clics dans la fenêtre source du clip actif seulement (cf. `click_impact`) ; /// - vitesse : poids `clamp(2 − vitesse, 0, 1)`. À 100× une frame couvre 3,3 s de source, la /// courbe serait échantillonnée une fois, au hasard : une secousse d'une frame ; @@ -3520,6 +3523,11 @@ fn click_impact_at( pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option { let (rw, rh) = (input.render_px[0], input.render_px[1]); + // Caché par l'application (`visible: false`) : rien de ce plan n'est dessiné, ni le sprite, + // ni le modèle et son ombre, ni l'anneau d'un clic. Les trois backends passent tous par ici. + if !input.track.visible_at(input.t) { + return None; + } let alpha = cursor_alpha(input.scene, input.cfg, &input.live, input.track, input.t); if alpha <= 0.001 { return None; @@ -3610,12 +3618,18 @@ pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option Option = (0..=90).map(|i| (i as f32 / 30.0, 0.6, 0.3)).collect(); + let track = |visibility| -> &'static crate::cursor::CursorTrack { + Box::leak(Box::new(crate::cursor::CursorTrack::new_with_visibility( + samples.clone(), + vec![1.45], + vec![], + visibility, + ))) + }; + let json = zoomed_golden_scene_json() + .replace(r#""rotation":"none""#, r#""rotation":"iso""#) + .replace(r#""cursor":{"#, r#""cursor":{"clickImpact":true,"#); + let scene = Scene::from_json(&json).expect("scène"); + let tilt = |track| { + plan_frame(&FrameGeometryInput { cursor: Some(track), ..golden_input(&scene, &cfg) }) + .zoom_rotation_dyn + }; + let on = tilt(track(vec![])); + assert!(on[1] > 0.75 * crate::regions::CLICK_IMPACT_DEG, "garde : {on:?}"); + assert_eq!(tilt(track(vec![(0.0, true), (1.48, false)])), on); + } + /// L'orbite en focus auto passe par `plan_frame` : l'écran n'est pas incliné, la caméra vise et tourne /// avec le pointeur lu dans le RECADRAGE, la boîte zoome sur son centre sans glisser, la mise au /// point suit la visée, un clic fait reculer l'œil sans presser l'écran, et curseur masqué @@ -8346,6 +8387,51 @@ mod tests { assert_eq!(plan(false).for_backend(true).taps, plan(false).taps, "le sprite plat garde sa traînée"); } + /// Caché par l'application (`visible: false`), le curseur ne dessine rien, sprite plat comme + /// modèle (ombre et anneau compris), et sa traînée ne remonte pas dans la phase cachée à la + /// réapparition : il y filait à droite, il réapparaît là où il avait cliqué. + #[test] + fn a_hidden_phase_draws_no_cursor_and_no_trail_into_it() { + let scene = model_scene(); + let cfg = crate::config::all().pop().expect("cfg"); + let fg = full_frame_geometry(); + let track = crate::cursor::CursorTrack::new_with_visibility( + vec![(0.0, 0.3, 0.6), (1.0, 0.3, 0.6), (1.1, 0.35, 0.6), (1.467, 0.9, 0.6), (1.5, 0.3, 0.6), (3.0, 0.3, 0.6)], + vec![1.0], + vec![], + vec![(0.0, true), (1.1, false), (1.5, true)], + ); + let plan = |model3d: bool, t: f32| { + plan_cursor( + &fg, + &CursorPlanInput { + render_px: [1920.0, 1080.0], + u_max: 1.0, + v_max: 1.0, + cfg: &cfg, + live: LiveParams { + cursor_model3d: model3d, + cursor_bounce_scale: MODEL_CLICK_BOUNCE_REF, + cursor_motion_blur: 1.0, + ..LiveParams::default() + }, + scene: Some(&scene), + track: &track, + t, + }, + ) + }; + for model3d in [false, true] { + assert!(plan(model3d, 1.05).is_some(), "model3d {model3d} : visible avant"); + for t in [1.1, 1.2, 1.4, 1.49] { + assert!(plan(model3d, t).is_none(), "model3d {model3d} : dessiné caché à {t} s"); + } + let back = plan(model3d, 1.505).expect("réapparu"); + assert_eq!(back.taps, 1, "model3d {model3d} : traînée venue de la phase cachée"); + } + assert!(!plan(true, 1.08).expect("visible").impacts.is_empty(), "garde : l'anneau du clic"); + } + /// Sous un angle fixe, l'écran tourne autour du centre de sa boîte ZOOMÉE, réduit de son /// containment : le curseur y perdait ~18 % au point de focus même, pendant que le zoom /// grossissait l'image. Il y garde sa taille à plat, à la perspective de biais près ; ailleurs, diff --git a/electron/media/cursorSidecar.test.ts b/electron/media/cursorSidecar.test.ts index 1c0e46a4d..88c7ed0da 100644 --- a/electron/media/cursorSidecar.test.ts +++ b/electron/media/cursorSidecar.test.ts @@ -149,7 +149,9 @@ describe("readCursorTelemetryFile", () => { const result = await readCursorTelemetryFile(video, {}); expect(result.success).toBe(true); - expect(result.samples).toEqual([{ timeMs: 10, cx: 0.5, cy: 0.5, interactionType: "click" }]); + expect(result.samples).toEqual([ + { timeMs: 10, cx: 0.5, cy: 0.5, visible: true, interactionType: "click" }, + ]); // This projection feeds the ai-edition auto-zoom detector, whose click // candidates ARE the recorded interactions (issue #699). It used to strip // the field — locked by a test, even — and the detector never saw a click @@ -158,6 +160,18 @@ describe("readCursorTelemetryFile", () => { // the extra field. }); + it("keeps hidden cursor state for auto-zoom", async () => { + const video = path.join(dir, "hidden.mp4"); + await writeSidecar(video, { + samples: [{ timeMs: 10, cx: 0.5, cy: 0.5, visible: false, interactionType: "click" }], + }); + + const result = await readCursorTelemetryFile(video, {}); + + expect(result.success).toBe(true); + expect(result.samples[0].visible).toBe(false); + }); + it("keeps every click kind, not only the plain left click", async () => { const video = path.join(dir, "kinds.mp4"); await writeSidecar(video, { diff --git a/electron/media/cursorSidecar.ts b/electron/media/cursorSidecar.ts index a47f63f5b..0a588e0ae 100644 --- a/electron/media/cursorSidecar.ts +++ b/electron/media/cursorSidecar.ts @@ -242,6 +242,7 @@ export async function readCursorTelemetryFile( timeMs: sample.timeMs, cx: sample.cx, cy: sample.cy, + visible: sample.visible, interactionType: sample.interactionType, })), }; diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index 312c9a70d..dda4abbf7 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -263,6 +263,7 @@ export interface CursorTelemetryPoint { timeMs: number; cx: number; cy: number; + visible?: boolean; interactionType?: "move" | "click" | "double-click" | "right-click" | "middle-click" | "mouseup"; cursorType?: | "arrow" diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts index f36b1cc32..3cff335ef 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts @@ -52,6 +52,166 @@ describe("detectZoomDwellCandidates", () => { }); describe("buildAutoZoomSuggestions", () => { + it("ignores a click made while the cursor is hidden", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [{ ...click(2000, 0.5, 0.5), visible: false }], + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toEqual([]); + }); + + it("ignores a visible mouse-down when the cursor hides during that drag", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(1000, 0.5, 0.5), visible: true }, + { timeMs: 1016, cx: 0.5, cy: 0.5, visible: false, interactionType: "move" }, + { timeMs: 1600, cx: 0.7, cy: 0.5, visible: false, interactionType: "mouseup" }, + ], + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toEqual([]); + }); + + it("keeps a click released before the cursor later hides", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(1000, 0.5, 0.5), visible: true }, + { timeMs: 1050, cx: 0.5, cy: 0.5, visible: true, interactionType: "mouseup" }, + { timeMs: 1400, cx: 0.5, cy: 0.5, visible: false, interactionType: "move" }, + ], + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toHaveLength(1); + expect(suggestions[0].focus).toEqual({ cx: 0.5, cy: 0.5 }); + }); + + it("keeps a later visible click after a hidden drag", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(1000, 0.5, 0.5), visible: true }, + { timeMs: 1100, cx: 0.6, cy: 0.5, visible: false, interactionType: "move" }, + { timeMs: 1600, cx: 0.7, cy: 0.5, visible: true, interactionType: "mouseup" }, + { ...click(4000, 0.2, 0.8), visible: true }, + ], + totalMs: 6000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toHaveLength(1); + expect(suggestions[0].focus).toEqual({ cx: 0.2, cy: 0.8 }); + }); + + it("keeps a legacy click without mouseup when the cursor hides much later", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(1000, 0.2, 0.8), visible: true }, + { timeMs: 1050, cx: 0.3, cy: 0.7, visible: true, interactionType: "move" }, + { timeMs: 2000, cx: 0.7, cy: 0.3, visible: false, interactionType: "move" }, + ], + totalMs: 4000, + existingRegions: [], + defaultDurationMs: 1000, + }); + expect(suggestions).toHaveLength(1); + expect(suggestions[0].focus).toEqual({ cx: 0.2, cy: 0.8 }); + }); + + it("rejects a legacy click when the cursor hides on the next sampler tick", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(1000, 0.2, 0.8), visible: true }, + { timeMs: 1033, cx: 0.25, cy: 0.75, visible: false, interactionType: "move" }, + ], + totalMs: 4000, + existingRegions: [], + defaultDurationMs: 1000, + }); + expect(suggestions).toEqual([]); + }); + + it("keeps a tap whose release shares its press sample when the cursor hides later", () => { + // The take records mouse-ups, but the second tap was released within one sampler tick. + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(500, 0.8, 0.2), visible: true }, + { timeMs: 600, cx: 0.8, cy: 0.2, visible: true, interactionType: "mouseup" }, + { ...click(3000, 0.2, 0.8), visible: true }, + { timeMs: 3500, cx: 0.2, cy: 0.8, visible: false, interactionType: "move" }, + ], + totalMs: 6000, + existingRegions: [], + defaultDurationMs: 1000, + }); + expect(suggestions.map((suggestion) => suggestion.focus)).toEqual([ + { cx: 0.8, cy: 0.2 }, + { cx: 0.2, cy: 0.8 }, + ]); + }); + + it("keeps a click whose mouse-up sample is already hidden", () => { + // Released, then typing hid the pointer before the next sampler tick. + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { ...click(1000, 0.5, 0.5), visible: true }, + { timeMs: 1033, cx: 0.5, cy: 0.5, visible: false, interactionType: "mouseup" }, + ], + totalMs: 4000, + existingRegions: [], + defaultDurationMs: 1000, + }); + expect(suggestions.map((suggestion) => suggestion.focus)).toEqual([{ cx: 0.5, cy: 0.5 }]); + }); + + it("does not join visible samples across a hidden interval into a dwell", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { timeMs: 1000, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 1200, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 1400, cx: 0.5, cy: 0.5, visible: false }, + { timeMs: 1800, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 2000, cx: 0.5, cy: 0.5, visible: true }, + ], + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toEqual([]); + }); + + it("ignores a dwell recorded while the cursor is hidden", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: dwell(2000, 0.5, 0.5).map((sample) => ({ + ...sample, + visible: false, + })), + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toEqual([]); + }); + + it("keeps a visible dwell beside hidden cursor samples", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + ...dwell(1000, 0.2, 0.8), + { timeMs: 2000, cx: 0.2, cy: 0.8, visible: false }, + ], + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toHaveLength(1); + expect(suggestions[0].focus.cx).toBeCloseTo(0.2); + expect(suggestions[0].focus.cy).toBeCloseTo(0.8); + }); + it("returns a centered span around each accepted dwell", () => { const telemetry = dwell(2000, 0.5, 0.5); const suggestions = buildAutoZoomSuggestions({ diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.ts b/src/lib/ai-edition/timeline/zoom-suggestions.ts index b82872267..3e8418015 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.ts @@ -30,6 +30,7 @@ function normalizeTelemetrySample( timeMs: Math.max(0, Math.min(sample.timeMs, totalMs)), cx: Math.max(0, Math.min(sample.cx, 1)), cy: Math.max(0, Math.min(sample.cy, 1)), + visible: sample.visible, interactionType: sample.interactionType, }; } @@ -92,7 +93,13 @@ export function detectZoomDwellCandidates( }); }; - for (let index = 1; index < samples.length; index += 1) { + for (let index = 0; index < samples.length; index += 1) { + if (samples[index].visible === false) { + pushRunIfDwell(runStart, index); + runStart = index + 1; + continue; + } + if (index <= runStart) continue; const prev = samples[index - 1]; const curr = samples[index]; const distance = Math.hypot(curr.cx - prev.cx, curr.cy - prev.cy); @@ -119,16 +126,44 @@ const CLICK_INTERACTION_TYPES: ReadonlySet - sample.interactionType !== undefined && CLICK_INTERACTION_TYPES.has(sample.interactionType), - ) - .map((sample) => ({ - centerTimeMs: sample.timeMs, - focus: { cx: sample.cx, cy: sample.cy }, - strength: 0, - })); + const candidates: ZoomDwellCandidate[] = []; + // A press without its own mouse-up sample has no known release: PipeWire and older sidecars + // record none, and the Windows and macOS samplers fold a release into the press sample when both + // land in one tick. A hidden sample on the next sampler tick still identifies a slider press; a + // much later hidden interval (typing, leaving the display) must not erase a completed click. + const UNRELEASED_PRESS_GRACE_MS = 150; + + let pendingClick: CursorTelemetryPoint | null = null; + let hiddenAtMs: number | null = null; + const finishPress = (released: boolean) => { + if (!pendingClick) return; + const hiddenDuringPress = + hiddenAtMs !== null && + (released || hiddenAtMs - pendingClick.timeMs <= UNRELEASED_PRESS_GRACE_MS); + if (!hiddenDuringPress) { + candidates.push({ + centerTimeMs: pendingClick.timeMs, + focus: { cx: pendingClick.cx, cy: pendingClick.cy }, + strength: 0, + }); + } + pendingClick = null; + hiddenAtMs = null; + }; + + for (const sample of samples) { + if (sample.interactionType && CLICK_INTERACTION_TYPES.has(sample.interactionType)) { + finishPress(false); + pendingClick = sample.visible === false ? null : sample; + } else if (sample.interactionType === "mouseup") { + // Sampled after the release: its visibility says nothing about the press. + finishPress(true); + } else if (pendingClick && sample.visible === false && hiddenAtMs === null) { + hiddenAtMs = sample.timeMs; + } + } + finishPress(false); + return candidates; } export interface AutoZoomSuggestion { diff --git a/src/native/contracts.ts b/src/native/contracts.ts index a2115e7f6..10164ad18 100644 --- a/src/native/contracts.ts +++ b/src/native/contracts.ts @@ -27,6 +27,7 @@ export interface CursorTelemetryPoint { timeMs: number; cx: number; cy: number; + visible?: boolean; } export interface CursorRecordingSample extends CursorTelemetryPoint { diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index ea959a411..760959666 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -610,6 +610,7 @@ The mask comes from the native compositor (ONNX Runtime + the vendored selfie-se | Date | Build / tag | Platform | Pass/fail | Notes | |------|-------------|----------|-----------|-------| +| 2026-09-23 | dev `fix/725-hidden-cursor-auto-zoom`; rebuilt compositor addon SHA256 `a0bcd61e17084c010141049a9978121e45be2985449ff1618ab6a3145f8381f6` | Windows 11 Home 26200 | Partial — issue #725 slider slice passed | Human-driven real recording in the Electron dev build: moving a slider whose application cursor becomes hidden kept the reconstructed cursor hidden in the Studio recording and did not create a click-driven automatic zoom for that press. A visible control click still produced the normal automatic zoom. Sidecar inspection independently confirmed the hidden interval (`visible: false`) and that no zoom was centred on the press immediately preceding it. Recording, stop and editor open passed. Not covered: export, packaged build, audio/webcam, tray/HUD, other editor features, macOS or Linux. | | 2026-09-12 | dev `fix/643-gif-export-cancel`, based on `fbe461e9`; rebuilt compositor addon SHA256 `128c89fe65a459b3f17721d610ea9a78e0d3e980e9eb0b5fb2b0e5d018619e15` | Windows | Partial — export cancellation passed | Computer-use OS input, real Electron and native save dialog, imported synthetic 8 s source. GIF 15 FPS / Medium (720p) / loop on: observed 55/120 frames, clicked Cancel while rendering (pending snapshot 104/120), returned to the same options; no final or partial output remained. Retried in that dialog to the same path and received Saved to; resulting GIF is 1280×720, 120 frames, 8,200,202 bytes, fully decodes with ffmpeg (exit 0). Automated real-addon tests separately pass existing-destination preservation, GIF retry, ordinary GIF and MP4 export. Desktop adapter required screenshot-based coordinates and Ctrl+O to reach the editor; HUD reachability is not claimed. An unrelated process error window partly obscured the retry success screenshot; file verification independently passed. Skipped: recording, audio/mic, webcam, tray/HUD, effect/caption comparisons, second viewer, macOS/Linux and release promotion. Task-local profile and evidence retained; test app stopped. | | 2026-07-31 | dev build, `claude/e2e-tests-v1-8-0-474894` (e9578f09) | macOS 26.5, M1 | Partial — 1 defect | Ran launch/HUD, media, modifier anchoring, and export. **Defect: a dangling asset blanks the preview.** Modifier anchoring across a reorder verified in preview and in the exported frames. macOS export produced 1280×720 h264 + AAC at ~2× realtime. Chat sections skipped: no AI provider configured. HUD drag not runnable under computer-use (drop point is the desktop). | | 2026-08-13 | installed `v1.9.5-rc.1` | Windows 11 26200, 1920×1080 @ 100% | Partial — 1 defect | Ran launch/HUD, source selection, recording, stop, editor open. Fragmented MP4 confirmed on the shipped artifact: 48 `moof`+`mdat` pairs over 47.6s, `mvex` present, `mfra` on clean stop. **Defect: a recording that survives a helper kill is thrown away by the app** — killing `wgc-capture.exe` mid-recording leaves a fully decodable 41s file (2460 packets, `ffmpeg -f null -` exit 0) with no `.session.json` and no `.cursor.json`, and stop answers "The recording could not be saved". Fixed in #363, re-verified end to end. Truncation ablation at 60%: plain MP4 unreadable, fragmented plays 29s. **A dev build cannot test any of this** — the prebuilt worktree helper predated the change and silently ran the old path. Editor/export/chat sections not run. |