diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 387bed2b9..dd0f477a6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,10 +52,12 @@ jobs: uses: ./.github/actions/setup # STT is the bundled whisper-stt-server (whisper.cpp with native DTW token - # timestamps); no VAD model is fetched here. The binary is built by - # build-whisper-stt.yml and staged below — without that step the installer - # ships without speech-to-text. See - # technical-documentation/architecture/transcription-and-captions.md. + # timestamps); no model is fetched here. The binary is built by + # build-whisper-stt.yml and staged below, from the run built from this + # commit's helper sources — without that step the installer ships without + # speech-to-text. See + # technical-documentation/architecture/transcription-and-captions.md, and + # technical-documentation/engineering/release-and-secrets.md for the run. - name: Stage whisper-stt binaries shell: bash env: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e8a87f80f..8565f14e9 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -76,6 +76,17 @@ jobs: - name: Check media budget working-directory: website run: npm run check:media + # The demo loops are served from R2 (media.getopenscreen.com), not git, so + # nothing above would notice a loop a page names that was never uploaded, + # or a cut folder holding the wrong files. media/loops.json records what + # scripts/media/publish-loops.mjs uploaded; this holds the code to it and + # asks the CDN for every recorded file, paced under the zone's rate limit. + - name: Check demo loops + working-directory: website + run: npm run check:loops -- --live + - name: Check responsive backgrounds + working-directory: website + run: npm run check:backgrounds -- --live # The recreation's data is generated from the app's own source and a # vendored project document. It is committed, so it can go stale in either # direction — a locale key moves, a timeline constant changes — and the @@ -137,4 +148,4 @@ jobs: id-token: write steps: - id: deployment - uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 \ No newline at end of file + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 diff --git a/.gitignore b/.gitignore index 24ad425eb..8b2682b6e 100644 --- a/.gitignore +++ b/.gitignore @@ -95,6 +95,9 @@ result-* /tools/stt-eval/whispercpp-dtw-poc/fixtures/*.wav /tools/stt-eval/whispercpp-dtw-poc/results/ +# Word-timing harness (tools/stt-eval/word-timing): TTS corpus and results. +/tools/stt-eval/word-timing/data/ + opencode.json opencode.json diff --git a/README.md b/README.md index 872f99144..23cb6784d 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@

A free, open-source desktop app for recording your screen and turning the result into polished product demos and walkthroughs.

- Editing a recording in OpenScreen: wallpaper and video effects, an AI-assisted cut driven from the chat, then export + Editing a recording in OpenScreen: one click restyles the whole video, deleting a sentence in the transcript cuts it from the video, and the editor's agent adds a zoom on every click

diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index dcbc432cf..b2611d75b 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -1,12 +1,14 @@ # Third-party notices OpenScreen is MIT licensed (see [LICENSE](LICENSE)). The installers additionally -bundle the pre-built native components below. This file ships inside the +bundle the third-party components below. This file ships inside the application resources and satisfies the attribution and source-offer obligations that come with them. npm dependencies are not listed here: they are resolved from `package.json` and -distributed by their own registries, not redistributed inside our binaries. +distributed by their own registries, not redistributed inside our binaries. The +exception is code bundled into the renderer whose licence asks for its notice in +the distributed form (js-aruco2 below): the bundler strips the source headers. --- @@ -38,6 +40,23 @@ distributed by their own registries, not redistributed inside our binaries. replace them with your own build of the same FFmpeg version by overwriting the DLLs in `resources/electron/native/bin/win32-*/`. +## web-demuxer — WebAssembly demuxer + +- **Component**: `wasm/web-demuxer.wasm`, inside the application bundle. +- **Used by**: the editor's streaming decoder, audio waveforms and caption audio + extraction, which demux recordings in the renderer with it. +- **License**: MIT for web-demuxer itself — Copyright (c) 2024 ForeverSc, under the + same permission notice as OpenScreen's own [LICENSE](LICENSE). The module also + contains FFmpeg code compiled to WebAssembly, under the **GNU Lesser General + Public License v2.1 or later**, as web-demuxer's README states. +- **Upstream**: the npm package `web-demuxer` 4.0.0, + . The shipped file is byte-identical + to that release's `dist/wasm-files/web-demuxer.wasm`. +- **Corresponding source**: the web-demuxer repository at the 4.0.0 release, whose + `lib/` directory holds the FFmpeg-derived sources and their build scripts. +- **Replacing it**: the module is a separate file loaded at runtime, so it can be + replaced with your own build of the same version. + ## whisper.cpp and ggml - **Components**: `whisper-stt-server` and its ggml backend sidecars, under @@ -147,6 +166,132 @@ distributed by their own registries, not redistributed inside our binaries. - IBM Plex Mono 2.3 — Copyright © 2017 IBM Corp. `ofl/ibmplexmono/` of at `23e54b51dd`. +## Recordly — helper code + +- **Components**: parts of the zoom, motion-smoothing and custom-cursor code, and a + few lines elsewhere, the native capture helpers included. +- **License**: MIT — Copyright (c) 2026 webadderall, under the same permission + notice as OpenScreen's own [LICENSE](LICENSE). Published by Recordly under MIT, + before its relicensing in March 2026. + +## js-aruco2 — ArUco marker detection + +- **Components**: `src/aruco.js` and `src/cv.js` of the npm package `js-aruco2` + 2.0.0, compiled into the renderer bundle (`dist/assets/*.js`). +- **Used by**: the camera calibration dialog, which finds the four printed corner + markers in a camera still. +- **License**: MIT — Copyright (c) 2020 Damiano Falcioni, Copyright (c) 2011 Juan + Mellado, under the same permission notice as OpenScreen's own + [LICENSE](LICENSE). +- **Upstream**: , release 2.0.0. + +## OpenCV — ArUco 4×4 dictionary codes + +- **Component**: the marker codes in js-aruco2's + `src/dictionaries/aruco_4x4_1000.js`, compiled into the renderer bundle. They + are taken from OpenCV's `predefined_dictionaries.hpp`; the marker sheet the + editor prints draws markers 0–3 of this dictionary. +- **License**: BSD 3-Clause — Copyright (C) 2013, OpenCV Foundation, all rights + reserved. Third party copyrights are property of their respective owners. + + > Redistribution and use in source and binary forms, with or without + > modification, are permitted provided that the following conditions are met: + > + > - Redistributions of source code must retain the above copyright notice, + > this list of conditions and the following disclaimer. + > - Redistributions in binary form must reproduce the above copyright notice, + > this list of conditions and the following disclaimer in the documentation + > and/or other materials provided with the distribution. + > - Neither the names of the copyright holders nor the names of the + > contributors may be used to endorse or promote products derived from this + > software without specific prior written permission. + > + > This software is provided by the copyright holders and contributors "as is" + > and any express or implied warranties, including, but not limited to, the + > implied warranties of merchantability and fitness for a particular purpose + > are disclaimed. In no event shall copyright holders or contributors be liable + > for any direct, indirect, incidental, special, exemplary, or consequential + > damages (including, but not limited to, procurement of substitute goods or + > services; loss of use, data, or profits; or business interruption) however + > caused and on any theory of liability, whether in contract, strict liability, + > or tort (including negligence or otherwise) arising in any way out of the use + > of this software, even if advised of the possibility of such damage. + +## Rust crates — compiled into the compositor addon + +- **Component**: `compositor_view.node`, under + `resources/electron/native/bin/-/`. +- **What is listed**: every third-party crate in the addon's dependency graph in + `crates/Cargo.lock`, across all platforms. Build-time crates (procedural macros, + build scripts) are included even though none of their code ships. +- **Licenses**: as each crate publishes them on crates.io. Where a crate offers a + choice, OpenScreen uses it under MIT or Apache-2.0. Each crate's licence text is + in its source package at the version listed. +- **MIT OR Apache-2.0** (155): android_system_properties 0.1.5, anyhow 1.0.103, + arrayvec 0.7.8, ash 0.38.0+1.3.281, autocfg 1.5.1, bit-set 0.8.0, bit-vec 0.8.0, + bitflags 1.3.2, bitflags 2.13.1, bumpalo 3.20.3, cc 1.2.67, cexpr 0.6.0, cfg-if + 1.0.4, core-foundation 0.9.4, core-foundation-sys 0.8.7, core-graphics-types + 0.1.3, cosmic-text 0.19.0, crc32fast 1.5.0, ctor 0.2.9, document-features + 0.2.12, either 1.16.0, equivalent 1.0.2, fdeflate 0.3.7, find-msvc-tools 0.1.9, + flate2 1.1.9, font-types 0.11.3, font-types 0.12.2, foreign-types 0.5.0, + foreign-types-macros 0.2.4, foreign-types-shared 0.3.1, futures-core 0.3.33, + futures-task 0.3.33, futures-util 0.3.33, glob 0.3.3, gpu-alloc 0.6.2, + gpu-alloc-types 0.3.1, gpu-allocator 0.27.0, gpu-descriptor 0.3.2, + gpu-descriptor-types 0.2.0, hashbrown 0.15.5, hashbrown 0.17.1, heck 0.5.0, + image 0.25.10, indexmap 2.14.0, itertools 0.13.0, itoa 1.0.18, jni-sys 0.3.1, + jni-sys 0.4.1, jni-sys-macros 0.4.1, js-sys 0.3.103, khronos-egl 6.0.0, libc + 0.2.186, linebender_resource_handle 0.1.1, litrs 1.0.0, lock_api 0.4.14, log + 0.4.33, matrixmultiply 0.3.11, memmap2 0.9.11, metal 0.29.0, metal 0.31.0, + minimal-lexical 0.2.1, naga 24.0.0, ndarray 0.16.1, ndarray 0.17.2, ndk-sys + 0.5.0+25.2.9519653, num-complex 0.4.6, num-integer 0.1.47, num-traits 0.2.19, + once_cell 1.21.4, ort 2.0.0-rc.13, ort-sys 2.0.0-rc.13, parking_lot 0.12.5, + parking_lot_core 0.9.12, paste 1.0.15, pin-project-lite 0.2.17, pkg-config + 0.3.33, png 0.18.1, pollster 0.4.0, portable-atomic 1.15.0, portable-atomic-util + 0.2.7, presser 0.3.1, prettyplease 0.2.37, proc-macro2 1.0.106, profiling + 1.0.18, quote 1.0.46, range-alloc 0.1.5, rangemap 1.7.1, rawpointer 0.2.1, + read-fonts 0.37.0, read-fonts 0.41.0, regex 1.13.1, regex-automata 0.4.16, + regex-syntax 0.8.11, renderdoc-sys 1.1.0, roxmltree 0.20.0, rustc-hash 1.1.0, + rustc-hash 2.1.3, rustversion 1.0.23, scopeguard 1.2.0, semver 1.0.28, serde + 1.0.228, serde_core 1.0.228, serde_derive 1.0.228, serde_json 1.0.150, shlex + 1.3.0, shlex 2.0.1, skrifa 0.40.0, skrifa 0.44.0, smallvec 1.15.2, smol_str + 0.3.6, static_assertions 1.1.0, swash 0.2.10, syn 2.0.119, syn 3.0.3, sys-locale + 0.3.2, thiserror 1.0.69, thiserror 2.0.19, thiserror-impl 1.0.69, thiserror-impl + 2.0.19, ttf-parser 0.25.1, unicode-bidi 0.3.18, unicode-script 0.5.8, + unicode-segmentation 1.13.3, unicode-width 0.1.14, unicode-xid 0.2.6, + version_check 0.9.5, wasm-bindgen 0.2.126, wasm-bindgen-futures 0.4.76, + wasm-bindgen-macro 0.2.126, wasm-bindgen-macro-support 0.2.126, + wasm-bindgen-shared 0.2.126, web-sys 0.3.103, wgpu 24.0.5, wgpu-core 24.0.5, + wgpu-hal 24.0.4, wgpu-types 24.0.0, windows 0.58.0, windows_aarch64_gnullvm + 0.52.6, windows_aarch64_msvc 0.52.6, windows_i686_gnu 0.52.6, + windows_i686_gnullvm 0.52.6, windows_i686_msvc 0.52.6, windows_x86_64_gnu + 0.52.6, windows_x86_64_gnullvm 0.52.6, windows_x86_64_msvc 0.52.6, windows-core + 0.58.0, windows-implement 0.58.0, windows-interface 0.58.0, windows-link 0.2.1, + windows-result 0.2.0, windows-strings 0.1.0, windows-sys 0.61.2, windows-targets + 0.52.6, yazi 0.2.1, zeno 0.3.3. +- **MIT** (26): block 0.1.6, cfg_aliases 0.2.2, convert_case 0.6.0, core_maths + 0.1.1, fontconfig-parser 0.5.8, fontdb 0.23.0, harfrust 0.5.2, libm 0.2.16, + malloc_buf 0.0.6, napi 2.16.17, napi-build 2.3.2, napi-derive 2.16.13, + napi-derive-backend 1.0.75, napi-sys 2.4.0, nom 7.1.3, objc 0.2.7, ordered-float + 4.6.0, redox_syscall 0.5.18, simd-adler32 0.3.10, slab 0.4.12, strum 0.26.3, + strum_macros 0.26.4, tracing 0.1.44, tracing-core 0.1.36, xml-rs 0.8.28, zmij + 1.0.23. +- **MIT OR Apache-2.0 OR Zlib** (9): bytemuck 1.25.1, bytemuck_derive 1.11.0, + glow 0.16.0, miniz_oxide 0.8.9, raw-window-handle 0.6.2, tinyvec 1.12.0, + tinyvec_macros 0.1.1, zune-core 0.5.1, zune-jpeg 0.5.15. +- **Apache-2.0** (7): clang-sys 1.8.1, codespan-reporting 0.11.1, gl_generator + 0.14.0, glutin_wgl_sys 0.6.1, khronos_api 3.1.0, spirv 0.3.0+sdk-1.3.268.0, + unicode-linebreak 0.1.5. +- **Unlicense OR MIT** (5): aho-corasick 1.1.4, byteorder-lite 0.1.0, memchr + 2.8.3, termcolor 1.4.1, winapi-util 0.1.11. +- **Zlib** (2): foldhash 0.1.5, slotmap 1.1.1. +- **ISC** (2): libloading 0.8.9, libloading 0.9.0. +- **BSD-3-Clause OR Apache-2.0** (2): moxcms 0.8.1, pxfm 0.1.30. +- **0BSD OR MIT OR Apache-2.0** (1): adler2 2.0.1. +- **BSD-3-Clause** (1): bindgen 0.70.1. +- **CC0-1.0** (1): hexf-parse 0.2.1. +- **Apache-2.0 OR GPL-2.0-only** (1): self_cell 1.3.0. +- **(MIT OR Apache-2.0) AND Unicode-3.0** (1): unicode-ident 1.0.24. + ## OpenScreen native helpers `wgc-capture` (Windows Graphics Capture), the ScreenCaptureKit helper (macOS), diff --git a/crates/Cargo.toml b/crates/Cargo.toml index 3caa7edfd..a65f9aaef 100644 --- a/crates/Cargo.toml +++ b/crates/Cargo.toml @@ -68,6 +68,7 @@ features = [ "Win32_Graphics_Direct2D_Common", # D2D1_COLOR_F / PIXEL_FORMAT "Win32_Graphics_DirectWrite", # mise en page + rastérisation du texte "Win32_Graphics_Gdi", # brosses/police GUI de la preview (poc-d3d/src/app.rs) + "Win32_Security", # IDXGIResource1::CreateSharedHandle (shared_frames.rs) "Win32_System_Performance", # QueryPerformanceCounter / Frequency (§10) "Win32_System_LibraryLoader", # GetModuleHandleW (hInstance) "Win32_UI_WindowsAndMessaging", # fenêtres (harnais live + GUI du POC) diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs index 8701c8bfe..5a1ded0e1 100644 --- a/crates/compositor-view-napi/src/lib.rs +++ b/crates/compositor-view-napi/src/lib.rs @@ -11,10 +11,11 @@ use napi::{Env, JsFunction, Task}; use napi_derive::napi; use openscreen_compositor::compositor::{live_params_from_scene, Compositor}; use openscreen_compositor::d3d::{Backend, Gpu}; +use openscreen_compositor::frame_geometry::FootageQuad; use openscreen_compositor::gif_export::{GifExportParams, GifStats}; use openscreen_compositor::gif_export_control::{GifExportCancelled, GifExportControl}; use openscreen_compositor::live::{LiveView, PausedPreviews}; -use openscreen_compositor::scene::Scene; +use openscreen_compositor::scene::{Scene, SceneClipCamera}; use openscreen_compositor::{config, pipeline}; use std::collections::HashMap; use std::path::PathBuf; @@ -197,6 +198,16 @@ pub struct FramePacket { /// R,G,B,A tightly-packed, `width * height * 4` octets — ce que `putImageData` / /// `ImageData` attendent côté JS (canvas 2D, format natif RGBA8). pub data: Buffer, + /// Le métrage dans cette image : ses coins TL, TR, BR, BL (x, y en fractions de l'image, + /// huit nombres), là où l'éditeur pose le gimbal d'un flou. Absent avant toute composition. + pub footage: Option>, + /// Le métrage passe de ses coins à l'image par leur homographie (caméra réelle), pas par + /// leur interpolation bilinéaire. + pub footage_projective: bool, + /// Clip actif de la scène quand la frame a été composée (`FramePosition`). + pub clip_index: u32, + /// Temps source de la frame écran composée, en secondes. + pub source_time_sec: f64, } /// Renvoie la dernière frame readback du thread de rendu SI elle est plus récente que @@ -236,17 +247,95 @@ pub fn read_frame(id: i32, since_gen: f64) -> Result> { v.latest_frame_since(since_gen.max(0.0) as u64) } }; - Ok(slot.map(|(gen, w, h, pixels)| { + Ok(slot.map(|(gen, w, h, pixels, footage, position)| { debug_assert_eq!(pixels.len(), (w as usize) * (h as usize) * 4); FramePacket { gen: gen as f64, width: w, height: h, data: Buffer::from(pixels), + footage: footage_corners(footage), + footage_projective: footage.is_some_and(|q| q.projective), + clip_index: position.clip_index, + source_time_sec: position.source_time_sec, } })) } +/// Les coins du métrage (TL, TR, BR, BL) aplatis en huit nombres, tels que JS les lit. +fn footage_corners(footage: Option) -> Option> { + footage.map(|q| q.corners.iter().flatten().map(|&v| v as f64).collect()) +} + +/// Une frame de preview posée dans une texture partagée : de quoi l'importer côté GPU +/// (`sharedTexture.importSharedTexture`) au lieu d'en recevoir les pixels. +#[napi(object)] +pub struct SharedFramePacket { + pub gen: f64, + /// Case de l'anneau qui porte la frame, à rendre avec `gen` par `release_shared_frame` + /// quand Chromium l'a relâchée. + pub slot: u32, + /// Handle NT de la texture, 8 octets little-endian : la forme qu'attend Electron + /// (`SharedTextureHandle.ntHandle`). Valable dans ce processus seulement. + pub handle: Buffer, + pub width: u32, + pub height: u32, + pub footage: Option>, + pub footage_projective: bool, + pub clip_index: u32, + pub source_time_sec: f64, +} + +/// Livrer les frames de la vue par textures partagées plutôt que par `read_frame`. Rend +/// `false` quand la machine ne le peut pas (hors Windows, backend logiciel) : la vue reste +/// alors au readback. +#[napi] +pub fn set_shared_frames(id: i32, enabled: bool) -> bool { + registry() + .lock() + .unwrap() + .get(&id) + .is_some_and(|v| v.set_shared_frames(enabled)) +} + +/// Le pendant de `read_frame` pour une vue en textures partagées : la dernière frame posée +/// dans l'anneau, si elle est plus récente que `since_gen`. Sa case reste tenue jusqu'à +/// `release_shared_frame`. `Ok(None)` aussi quand la vue relit en RAM. +#[napi] +pub fn read_shared_frame(id: i32, since_gen: f64) -> Result> { + let frame = match registry().lock().unwrap().get(&id) { + None => return Ok(None), + Some(v) => { + // Même relais que `read_frame` : sans lui, un thread de rendu mort ne se verrait + // que par un canvas noir. + if let Some(fatal) = v.fatal_error() { + return Err(Error::from_reason(fatal)); + } + v.take_shared_frame(since_gen.max(0.0) as u64) + } + }; + Ok(frame.map(|f| SharedFramePacket { + gen: f.gen as f64, + slot: f.slot, + handle: f.handle.to_le_bytes().to_vec().into(), + width: f.width, + height: f.height, + footage: footage_corners(f.footage), + footage_projective: f.footage.is_some_and(|q| q.projective), + clip_index: f.position.clip_index, + source_time_sec: f.position.source_time_sec, + })) +} + +/// Chromium a relâché la frame `gen` de la case `slot` (`allReferencesReleased` côté +/// Electron) : le thread de rendu peut y réécrire. Sans effet sur une vue détruite. +#[napi] +pub fn release_shared_frame(id: i32, slot: u32, gen: f64) { + if let Some(v) = registry().lock().unwrap().get(&id) { + v.release_shared_frame(slot, gen.max(0.0) as u64); + } +} + /// Param live (inspector). Le type de valeur route vers le bon setter : /// bool = switch (webcamMirror…), number = slider (shadow/roundness/motionBlur/backgroundBlur), /// string = sélection (backgroundColor "#rrggbb"). @@ -279,6 +368,9 @@ pub fn present_time(id: i32, seconds: f64) { /// Remplace les sources du clip actif sans recréer la vue ni son thread de rendu. L'identité /// timeline et le playhead source sont atomiques avec le switch : deux clips partageant les /// mêmes fichiers restent distincts, et les deux décodeurs ouvrent directement la bonne frame. +/// +/// `additional_cameras`: cameras 2-4 of the clip (index k-1 = camera k, an empty `path` = no +/// camera in that slot). Absent = none; the view decodes only those its layout regions show. #[napi] pub fn set_active_clip( id: i32, @@ -287,18 +379,33 @@ pub fn set_active_clip( webcam_offset_sec: f64, clip_index: u32, source_time_sec: f64, + additional_cameras: Option>, ) { if let Some(v) = registry().lock().unwrap().get(&id) { + let cameras = additional_cameras + .unwrap_or_default() + .into_iter() + .map(|c| SceneClipCamera { path: c.path, offset_sec: c.offset_sec }) + .collect(); v.set_active_clip( &screen_path, &webcam_path, webcam_offset_sec, + cameras, clip_index as usize, source_time_sec, ); } } +/// An additional camera of a clip (= TS `CompositorClipCamera`). +#[napi(object)] +pub struct ClipCameraInput { + pub path: String, + /// Camera source time = screen source time - this. + pub offset_sec: f64, +} + /// Installe la scène de l'app (JSON `SceneDescription`) sur la vue : layout preset piloté par /// l'app au lieu de la fixture. JSON invalide → ignoré côté natif. #[napi] @@ -415,6 +522,26 @@ pub struct ClipInput { pub webcam_offset_sec: f64, /// `false` évite une ouverture ffmpeg vouée à échouer et réserve du silence à ce clip. pub has_audio: bool, + /// Cameras 2-4 of the clip (index k-1 = camera k, an empty `path` = none). Absent = none. + pub additional_cameras: Option>, +} + +/// `ClipInput` → `pipeline::ClipSource`, shared by the MP4 and the GIF export. +fn clip_source(c: ClipInput) -> pipeline::ClipSource { + pipeline::ClipSource { + screen: c.screen_path, + webcam: c.webcam_path, + source_start_sec: c.source_start_sec, + source_end_sec: c.source_end_sec, + webcam_offset_sec: c.webcam_offset_sec, + has_audio: c.has_audio, + additional_cameras: c + .additional_cameras + .unwrap_or_default() + .into_iter() + .map(|cam| pipeline::ClipCamera { path: cam.path, offset_sec: cam.offset_sec }) + .collect(), + } } /// The export's pixel size. The SHAPE belongs to the scene: `scene.output` is what the live @@ -612,17 +739,7 @@ pub fn export_multi( params: Option, on_progress: Option, ) -> Result> { - let clips = clips - .into_iter() - .map(|c| pipeline::ClipSource { - screen: c.screen_path, - webcam: c.webcam_path, - source_start_sec: c.source_start_sec, - source_end_sec: c.source_end_sec, - webcam_offset_sec: c.webcam_offset_sec, - has_audio: c.has_audio, - }) - .collect(); + let clips = clips.into_iter().map(clip_source).collect(); Ok(AsyncTask::new(ExportMultiTask { out_path, clips, @@ -793,17 +910,7 @@ pub fn export_gif( // Deliberately the same argument shape as `export_multi`: the caller builds // one clip list and one scene, and picks the container. Cursor comes from // the scene like every other effect — there is no GIF-specific input left. - let clips = clips - .into_iter() - .map(|c| pipeline::ClipSource { - screen: c.screen_path, - webcam: c.webcam_path, - source_start_sec: c.source_start_sec, - source_end_sec: c.source_end_sec, - webcam_offset_sec: c.webcam_offset_sec, - has_audio: c.has_audio, - }) - .collect(); + let clips = clips.into_iter().map(clip_source).collect(); let gif_params = params .map(|p| GifExportParams { width: p.width, diff --git a/crates/compositor/build.rs b/crates/compositor/build.rs index 66f465153..c55d8ee38 100644 --- a/crates/compositor/build.rs +++ b/crates/compositor/build.rs @@ -191,7 +191,7 @@ fn main() { fn compile_hlsl(out: &Path) { use windows::core::PCSTR; use windows::Win32::Graphics::Direct3D::Fxc::{D3DCompile, D3DCOMPILE_OPTIMIZATION_LEVEL3}; - use windows::Win32::Graphics::Direct3D::ID3DBlob; + use windows::Win32::Graphics::Direct3D::{ID3DBlob, D3D_SHADER_MACRO}; fn blob_bytes(blob: &ID3DBlob) -> Vec { unsafe { @@ -203,17 +203,25 @@ fn compile_hlsl(out: &Path) { const SRC: &str = "src/shaders.hlsl"; println!("cargo:rerun-if-changed={SRC}"); let hlsl = std::fs::read(SRC).expect(SRC); - // Tous les points d'entrée que `Compositor::new_inner` instancie, avec leur profil. - for (entry, profile) in [ - ("vs_main", "vs_5_0"), - ("ps_main", "ps_5_0"), - ("vs_fs", "vs_5_0"), - ("ps_y", "ps_5_0"), - ("ps_uv", "ps_5_0"), - ("ps_blur", "ps_5_0"), - ("ps_tex", "ps_5_0"), - ("ps_kawase_down", "ps_5_0"), - ("ps_kawase_up", "ps_5_0"), + // `ps_main` avec les modèles 3D : la variante que seuls ces calques utilisent (le haut de + // `shaders.hlsl`). La liste se termine par une entrée nulle. + let models = [ + D3D_SHADER_MACRO { Name: PCSTR(b"LAYER_MODELS\0".as_ptr()), Definition: PCSTR(b"1\0".as_ptr()) }, + D3D_SHADER_MACRO { Name: PCSTR::null(), Definition: PCSTR::null() }, + ]; + // Tous les points d'entrée que `Compositor::new_inner` instancie, avec leur profil, le nom du + // `.cso` qu'en lit `shader!` et leurs macros. + for (entry, profile, cso, defines) in [ + ("vs_main", "vs_5_0", "vs_main", None), + ("ps_main", "ps_5_0", "ps_main", None), + ("ps_main", "ps_5_0", "ps_main_models", Some(models.as_ptr())), + ("vs_fs", "vs_5_0", "vs_fs", None), + ("ps_y", "ps_5_0", "ps_y", None), + ("ps_uv", "ps_5_0", "ps_uv", None), + ("ps_blur", "ps_5_0", "ps_blur", None), + ("ps_tex", "ps_5_0", "ps_tex", None), + ("ps_kawase_down", "ps_5_0", "ps_kawase_down", None), + ("ps_kawase_up", "ps_5_0", "ps_kawase_up", None), ] { let (entry_z, profile_z) = (format!("{entry}\0"), format!("{profile}\0")); let (mut code, mut errors): (Option, Option) = (None, None); @@ -222,7 +230,7 @@ fn compile_hlsl(out: &Path) { hlsl.as_ptr().cast(), hlsl.len(), PCSTR(b"shaders.hlsl\0".as_ptr()), - None, + defines, None, PCSTR(entry_z.as_ptr()), PCSTR(profile_z.as_ptr()), @@ -234,9 +242,9 @@ fn compile_hlsl(out: &Path) { }; if let Err(e) = compiled { let log = errors.as_ref().map(blob_bytes).unwrap_or_default(); - panic!("{SRC} : {entry} ne compile pas ({e})\n{}", String::from_utf8_lossy(&log)); + panic!("{SRC} : {cso} ne compile pas ({e})\n{}", String::from_utf8_lossy(&log)); } - std::fs::write(out.join(format!("{entry}.cso")), blob_bytes(&code.unwrap())).expect(entry); + std::fs::write(out.join(format!("{cso}.cso")), blob_bytes(&code.unwrap())).expect(cso); } } diff --git a/crates/compositor/examples/live_free_run_bench_macos.rs b/crates/compositor/examples/live_free_run_bench_macos.rs new file mode 100644 index 000000000..8b3ec5c3b --- /dev/null +++ b/crates/compositor/examples/live_free_run_bench_macos.rs @@ -0,0 +1,125 @@ +//! Bench de la lecture libre de la preview (macOS) : rejoue la boucle de `render_thread` sur +//! un seul clip, à une vitesse donnée, et mesure le retard que la vue prend sur l'horloge. +//! +//! Même arithmétique que `render_thread` : l'accumulateur avance de `dt × vitesse`, `step()` +//! compose chaque frame due jusqu'à `max_steps` par tick (au-delà le reliquat est abandonné), +//! puis relit le RT UNE fois par tick, comme la publication macOS (pas de textures partagées). +//! `dt` est borné à 0,1 s comme dans `render_thread` : un tick plus long perd de l'horloge. +//! +//! Utilisation : +//! `cargo run --release --example live_free_run_bench_macos -- [vitesse=4] [secondes=8] [largeur=1920] [hauteur=1080]` + +#[cfg(target_os = "macos")] +fn main() -> anyhow::Result<()> { + use openscreen_compositor::compositor::Compositor; + use openscreen_compositor::config; + use openscreen_compositor::d3d::Gpu; + use openscreen_compositor::frame_geometry::live_params_from_scene; + use openscreen_compositor::live::Player; + use openscreen_compositor::scene::Scene; + use std::time::{Duration, Instant}; + + let mut args = std::env::args().skip(1); + let path = args.next().unwrap_or_else(|| { + eprintln!("usage: live_free_run_bench_macos [vitesse=4] [secondes=8] [largeur=1920] [hauteur=1080]"); + std::process::exit(2); + }); + let speed: f64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(4.0); + let seconds: f64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(8.0); + let width: u32 = args.next().and_then(|s| s.parse().ok()).unwrap_or(1920); + let height: u32 = args.next().and_then(|s| s.parse().ok()).unwrap_or(1080); + + let source = path.replace('\\', "/"); + let scene_json = format!( + r##"{{ + "clips": [{{"screenPath":"{source}","webcamPath":"","sourceStartSec":0,"sourceEndSec":100000,"webcamOffsetSec":0,"hasAudio":false}}], + "layout": {{"preset":"no-webcam","webcamSize":1.0,"webcamShape":"rounded","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, + "effects": {{"padding":0.1,"blur":false,"shadow":0.5,"roundnessFrac":0.02,"motionBlur":0.0}}, + "background": {{"kind":"color","color":"#303030"}}, + "zoomRegions": [], + "speedRegions": [], + "cursor": {{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}}, + "cropByClip": [null], + "output": {{"width":{width},"height":{height},"fps":30}} + }}"## + ); + + let gpu = Gpu::create(false)?; + let mut cfg = config::all().pop().expect("au moins une config"); + cfg.zoom = false; + cfg.layout_anim = false; + let comp = Compositor::new_sized(&gpu, width, height)?; + let scene = Scene::from_json(&scene_json)?; + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_scene(Some(scene.clone())); + comp.clear_cursor(); + + unsafe { + let mut player = Player::open(&path, "", &gpu)?; + player.set_programme_clock(Some(&scene), 0); + anyhow::ensure!(player.present_frame(&comp, &cfg, 0.0)?, "aucune frame à 0 s"); + // Temps source réellement joué, cumulé frame par frame : un clip plus court que la mesure + // reboucle à 0 (EOF), et la seule position finale compterait alors une seule passe. + let mut played = 0.0f64; + + let mut acc = 0.0f64; + let mut last = Instant::now(); + let t0 = last; + let (mut ticks, mut adopted, mut capped, mut published) = (0u64, 0u64, 0u64, 0u64); + // Ce que l'horloge de l'app a parcouru, en temps source : la cible que la vue devrait tenir. + let mut clock_source = 0.0f64; + while t0.elapsed().as_secs_f64() < seconds { + let now = Instant::now(); + let dt = (now - last).as_secs_f64().min(0.1); + last = now; + ticks += 1; + acc += dt * speed; + clock_source += (now - t0).as_secs_f64() * speed - clock_source; + let max_steps = ((3.0 * speed.max(1.0)).ceil() as i32).min(64); + let mut n = 0; + let mut stepped = false; + loop { + let before = player.screen_time_sec(); + if !player.step(&comp, &cfg, before + acc)? { + break; + } + stepped = true; + adopted += 1; + let after = player.screen_time_sec(); + played += if after >= before { after - before } else { after }; + acc = if after >= before { (acc - (after - before)).max(0.0) } else { 0.0 }; + n += 1; + if n >= max_steps { + capped += 1; + acc = 0.0; + break; + } + } + if stepped { + comp.readback_direct()?; + published += 1; + } else { + std::thread::sleep(Duration::from_millis(4)); + } + } + let wall = t0.elapsed().as_secs_f64(); + println!( + "vitesse {speed}x, {width}x{height}, {wall:.2} s : horloge {:.2} s source, vue {played:.2} s source, retard {:.2} s source ({:.2} s de lecture)", + clock_source, + clock_source - played, + (clock_source - played) / speed + ); + println!( + "ticks {ticks} ({:.1}/s), frames adoptées {adopted} ({:.1}/s), plafond atteint {capped} fois, publiées {published} ({:.1}/s)", + ticks as f64 / wall, + adopted as f64 / wall, + published as f64 / wall + ); + } + Ok(()) +} + +#[cfg(not(target_os = "macos"))] +fn main() { + eprintln!("live_free_run_bench_macos : macOS uniquement"); +} diff --git a/crates/compositor/src/camera_layers.rs b/crates/compositor/src/camera_layers.rs new file mode 100644 index 000000000..ed5447b97 --- /dev/null +++ b/crates/compositor/src/camera_layers.rs @@ -0,0 +1,476 @@ +//! Which camera layers a frame draws, and where, inside camera layout regions. +//! +//! Pure planning: `plan_frame` calls `camera_layers_at` and the backends draw the result. A +//! region's layers glide in from whatever was on screen before it (the default camera-1 PiP, +//! or the layers of a region that ends exactly where it starts) and glide back out to the +//! default, on the Full Camera envelope (`regions::camera_fullscreen_region_phase`): the same +//! window lengths, the same `ease_out_screen_studio` curve, measured on the screen clock. A +//! camera-1 Full Camera region that meets a layout region at a seam is a neighbour like any +//! other: camera 0 frame-filling, gliding straight into or out of the layout region's layers. + +use crate::frame_geometry::webcam_shape_code; +use crate::regions::{ + ease_out_screen_studio, ScreenClock, FULLSCREEN_LEAD_OUT_WINDOW_S, REGION_SEAM_S, + TRANSITION_WINDOW_S, +}; +use crate::scene::{SceneCameraFullscreenRegion, SceneCameraLayer, SceneCameraLayoutRegion}; + +/// Two regions closer than this (source seconds) hand over directly, without the default. +const ADJACENT_S: f64 = REGION_SEAM_S; +/// Layers fainter than this are not drawn at all. +const MIN_OPACITY: f32 = 1e-3; +/// Cameras beyond camera 0 a compositor takes frames for (`set_extra_camera_frames`). +pub const MAX_EXTRA_CAMERAS: usize = 3; + +/// One camera to draw this frame. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct CameraLayerPlan { + pub camera: usize, + /// x, y, w, h in output fractions. + pub dst: [f32; 4], + /// Corner radius as a fraction of min(dst w, h) in pixels. + pub radius_frac: f32, + /// `webcam_shape_code` vocabulary. + pub shape: u32, + /// 0..1. + pub opacity: f32, + pub fills_frame: bool, +} + +fn plan_of(layer: &SceneCameraLayer) -> CameraLayerPlan { + let r = layer.rect; + CameraLayerPlan { + camera: layer.camera, + dst: [r.x, r.y, r.width, r.height], + radius_frac: layer.radius_frac, + shape: webcam_shape_code(&layer.shape), + opacity: 1.0, + fills_frame: layer.fills_frame, + } +} + +fn lerp(a: f32, b: f32, k: f32) -> f32 { + a + (b - a) * k +} + +/// `a` turning into `b` at `k` (0 = all `a`, 1 = all `b`). A camera in both moves; a camera in +/// only one of them fades. Draw order: frame-filling layers first, then the rest, each group in +/// `b`'s order followed by what only `a` still shows. +fn blend(a: &[CameraLayerPlan], b: &[CameraLayerPlan], k: f32) -> Vec { + let k = k.clamp(0.0, 1.0); + let mut out = Vec::with_capacity(a.len() + b.len()); + for to in b { + out.push(match a.iter().find(|from| from.camera == to.camera) { + Some(from) => { + // Discrete properties switch half-way through the move. + let flags = if k >= 0.5 { to } else { from }; + CameraLayerPlan { + camera: to.camera, + dst: std::array::from_fn(|i| lerp(from.dst[i], to.dst[i], k)), + radius_frac: lerp(from.radius_frac, to.radius_frac, k), + shape: flags.shape, + opacity: 1.0, + fills_frame: flags.fills_frame, + } + } + None => CameraLayerPlan { opacity: k, ..*to }, + }); + } + for from in a { + if !b.iter().any(|to| to.camera == from.camera) { + out.push(CameraLayerPlan { opacity: 1.0 - k, ..*from }); + } + } + out.retain(|layer| layer.opacity > MIN_OPACITY); + // Stable: each group keeps the order built above. + out.sort_by_key(|layer| !layer.fills_frame); + out +} + +/// Camera 0 as a Full Camera region shows it: the whole frame, square corners. Same discrete +/// shape as the default layer, so the hand-over to the default inside the region is seamless. +fn full_camera_layer(default: &CameraLayerPlan) -> CameraLayerPlan { + CameraLayerPlan { + camera: 0, + dst: [0.0, 0.0, 1.0, 1.0], + radius_frac: 0.0, + shape: default.shape, + opacity: 1.0, + fills_frame: true, + } +} + +/// The layers to draw at source time `t`, in draw order. +/// +/// `default_cam0`: camera 0's layer as today's plan places it (None = camera 0 not drawn). It +/// is what shows outside every region, and what a region glides from and back to. +/// +/// `fullscreen`: the camera-1 Full Camera regions. One that meets a layout region at a seam +/// is that region's neighbour, with camera 0 frame-filling: the layout region glides in from +/// it, or holds to its end and the Full Camera region's lead-in glides on from its layers +/// (the Full Camera phase holds 1 on that side, `regions::camera_fullscreen_region_phase`). +/// +/// Time base: copied from `camera_fullscreen_region_phase` — the region is found on SOURCE +/// time (its bounds compared to `t` as `f32`), then the windows are measured on the SCREEN +/// clock (`clock.at`), so a speed region does not stretch or squash the glide. +pub fn camera_layers_at( + regions: &[SceneCameraLayoutRegion], + fullscreen: &[SceneCameraFullscreenRegion], + t: f32, + clock: &ScreenClock, + default_cam0: Option, +) -> Vec { + let default: Vec = default_cam0.into_iter().collect(); + let full_cam0: Vec = default_cam0.iter().map(full_camera_layer).collect(); + // An empty (or reversed) region would match `t` at a single instant and draw its layers + // at full strength for that sample; it is neither drawn nor anyone's neighbour. + let non_empty = || regions.iter().enumerate().filter(|(_, r)| r.end_sec > r.start_sec); + let full_cameras = || fullscreen.iter().filter(|r| r.end_sec > r.start_sec); + let layers_of = |r: &SceneCameraLayoutRegion| r.layers.iter().map(plan_of).collect::>(); + let Some(index) = non_empty() + .find(|(_, r)| r.start_sec as f32 <= t && t <= r.end_sec as f32) + .map(|(i, _)| i) + else { + return full_camera_lead_in(regions, fullscreen, t, clock, &full_cam0).unwrap_or(default); + }; + let region = ®ions[index]; + let others = || non_empty().filter(move |(i, _)| *i != index).map(|(_, r)| r); + let prev = others() + .find(|r| (r.end_sec - region.start_sec).abs() <= ADJACENT_S) + .map(layers_of) + .or_else(|| { + full_cameras() + .any(|f| (f.end_sec - region.start_sec).abs() <= ADJACENT_S) + .then(|| full_cam0.clone()) + }); + let has_next = others().any(|r| (r.start_sec - region.end_sec).abs() <= ADJACENT_S) + || full_cameras().any(|f| (f.start_sec - region.end_sec).abs() <= ADJACENT_S); + let current = layers_of(region); + + let (start, end, t) = ( + clock.at(region.start_sec as f32), + clock.at(region.end_sec as f32), + clock.at(t), + ); + let half = (end - start) * 0.5; + let win_in = TRANSITION_WINDOW_S.min(half); + let win_out = FULLSCREEN_LEAD_OUT_WINDOW_S.min(half); + if t - start < win_in { + let from = prev.as_deref().unwrap_or(&default); + blend(from, ¤t, ease_out_screen_studio((t - start) / win_in)) + } else if !has_next && end - t < win_out { + // With a region right after this one, the hand-over is that region's lead-in instead. + blend(¤t, &default, 1.0 - ease_out_screen_studio((end - t) / win_out)) + } else { + blend(&[], ¤t, 1.0) + } +} + +/// Inside a Full Camera region that starts where a layout region ends, during its lead-in: +/// the layout region's layers gliding into camera 0 frame-filling (`full_cam0`), on the same +/// window and curve as a layout region's lead-in. None anywhere else. +fn full_camera_lead_in( + regions: &[SceneCameraLayoutRegion], + fullscreen: &[SceneCameraFullscreenRegion], + t: f32, + clock: &ScreenClock, + full_cam0: &[CameraLayerPlan], +) -> Option> { + let full = fullscreen.iter().find(|f| { + f.end_sec > f.start_sec && f.start_sec as f32 <= t && t <= f.end_sec as f32 + })?; + let before = regions.iter().find(|r| { + r.end_sec > r.start_sec && (r.end_sec - full.start_sec).abs() <= ADJACENT_S + })?; + let (start, end) = (clock.at(full.start_sec as f32), clock.at(full.end_sec as f32)); + let t = clock.at(t); + let win_in = TRANSITION_WINDOW_S.min((end - start) * 0.5); + if t - start >= win_in { + return None; + } + let from: Vec = before.layers.iter().map(plan_of).collect(); + Some(blend(&from, full_cam0, ease_out_screen_studio((t - start) / win_in))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::scene::SceneRect; + + const PIP: [f32; 4] = [0.75, 0.7, 0.22, 0.22]; + const CORNER: [f32; 4] = [0.05, 0.05, 0.2, 0.2]; + const FULL: [f32; 4] = [0.0, 0.0, 1.0, 1.0]; + + fn default_pip() -> Option { + Some(CameraLayerPlan { + camera: 0, + dst: PIP, + radius_frac: 0.5, + shape: 1, + opacity: 1.0, + fills_frame: false, + }) + } + + fn region(start: f64, end: f64, layers: &[(usize, [f32; 4], bool)]) -> SceneCameraLayoutRegion { + SceneCameraLayoutRegion { + clip_index: Some(0), + start_sec: start, + end_sec: end, + layers: layers + .iter() + .map(|&(camera, r, fills_frame)| SceneCameraLayer { + camera, + rect: SceneRect { x: r[0], y: r[1], width: r[2], height: r[3] }, + radius_frac: if fills_frame { 0.0 } else { 0.3 }, + shape: if fills_frame { "rectangle" } else { "rounded" }.to_string(), + fills_frame, + }) + .collect(), + } + } + + fn at(regions: &[SceneCameraLayoutRegion], t: f32) -> Vec { + camera_layers_at(regions, &[], t, &ScreenClock::default(), default_pip()) + } + + fn layer(layers: &[CameraLayerPlan], camera: usize) -> Option { + layers.iter().copied().find(|l| l.camera == camera) + } + + fn strictly_between(v: f32, a: f32, b: f32) -> bool { + v > a.min(b) && v < a.max(b) + } + + #[test] + fn outside_every_region_only_the_default_camera_is_drawn() { + let regions = [region(2.0, 6.0, &[(1, FULL, true)])]; + for t in [0.0, 1.9, 6.1, 9.0] { + assert_eq!(at(®ions, t), vec![default_pip().unwrap()], "t = {t}"); + } + assert!(camera_layers_at(®ions, &[], 1.0, &ScreenClock::default(), None).is_empty()); + } + + #[test] + fn inside_a_region_after_the_lead_in_its_layers_are_drawn_fully() { + let regions = [region(2.0, 6.0, &[(1, FULL, true), (0, CORNER, false)])]; + let layers = at(®ions, 4.0); + assert_eq!(layers.len(), 2); + assert_eq!(layers[0].camera, 1); + assert_eq!(layers[0].dst, FULL); + assert_eq!(layers[0].opacity, 1.0); + assert!(layers[0].fills_frame); + assert_eq!(layers[1].camera, 0); + assert_eq!(layers[1].dst, CORNER); + assert_eq!(layers[1].opacity, 1.0); + assert_eq!(layers[1].shape, webcam_shape_code("rounded")); + assert_eq!(layers[1].radius_frac, 0.3); + } + + #[test] + fn entering_a_region_glides_camera_0_and_fades_in_a_new_camera() { + let regions = [region(2.0, 6.0, &[(1, FULL, true), (0, CORNER, false)])]; + let layers = at(®ions, 2.0 + TRANSITION_WINDOW_S / 2.0); + let cam0 = layer(&layers, 0).expect("camera 0"); + for i in 0..4 { + assert!(strictly_between(cam0.dst[i], PIP[i], CORNER[i]), "dst[{i}] = {}", cam0.dst[i]); + } + assert_eq!(cam0.opacity, 1.0); + let cam1 = layer(&layers, 1).expect("camera 1"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0), "opacity {}", cam1.opacity); + assert_eq!(cam1.dst, FULL); + // On the region's start the glide has not begun. + assert_eq!(at(®ions, 2.0), vec![default_pip().unwrap()]); + } + + #[test] + fn leaving_a_region_returns_to_the_default() { + let regions = [region(2.0, 6.0, &[(1, FULL, true), (0, CORNER, false)])]; + let win_out = FULLSCREEN_LEAD_OUT_WINDOW_S.min(2.0); + let layers = at(®ions, 6.0 - win_out / 2.0); + let cam0 = layer(&layers, 0).expect("camera 0"); + for i in 0..4 { + assert!(strictly_between(cam0.dst[i], PIP[i], CORNER[i]), "dst[{i}] = {}", cam0.dst[i]); + } + let cam1 = layer(&layers, 1).expect("camera 1"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0), "opacity {}", cam1.opacity); + // On the region's end it is back to the default, exactly. + let end = at(®ions, 6.0); + assert_eq!(end.len(), 1); + assert_eq!(end[0].camera, 0); + for i in 0..4 { + assert!((end[0].dst[i] - PIP[i]).abs() < 1e-6); + } + } + + #[test] + fn adjacent_regions_glide_directly() { + let regions = [region(2.0, 5.0, &[(1, FULL, true)]), region(5.0, 8.0, &[(2, FULL, true)])]; + // The end of A keeps A: no lead-out to the default PiP. + let before = at(®ions, 5.0 - 0.01); + assert_eq!(before.len(), 1); + assert_eq!(before[0].camera, 1); + assert_eq!(before[0].opacity, 1.0); + // B's lead-in hands camera 1 over to camera 2. + let during = at(®ions, 5.0 + TRANSITION_WINDOW_S / 2.0); + let cam1 = layer(&during, 1).expect("camera 1 fading out"); + let cam2 = layer(&during, 2).expect("camera 2 fading in"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0)); + assert!(strictly_between(cam2.opacity, 0.0, 1.0)); + // Neither region has camera 0, so the default PiP never shows around the seam. + let mut t = 4.8f32; + while t <= 5.2 { + assert!(layer(&at(®ions, t), 0).is_none(), "camera 0 at t = {t}"); + t += 0.01; + } + } + + fn full_camera(start: f64, end: f64) -> SceneCameraFullscreenRegion { + SceneCameraFullscreenRegion { + clip_index: Some(0), + start_sec: start, + end_sec: end, + rotation: 0, + mirror: None, + full_frame: false, + } + } + + /// The plan with Full Camera regions, camera 0's default placed as `plan_frame` places it: + /// grown from the PiP to the frame by the Full Camera progress. + fn at_with_full( + regions: &[SceneCameraLayoutRegion], + fulls: &[SceneCameraFullscreenRegion], + t: f32, + ) -> Vec { + let clock = ScreenClock::default(); + let p = crate::regions::camera_fullscreen_progress_at(fulls, t, &clock, regions); + let default = CameraLayerPlan { + dst: std::array::from_fn(|i| lerp(PIP[i], FULL[i], p)), + fills_frame: p >= 1.0, + ..default_pip().unwrap() + }; + camera_layers_at(regions, fulls, t, &clock, Some(default)) + } + + /// Camera 0 on the straight path between two rects: never the default PiP's geometry. + fn on_the_path(dst: [f32; 4], a: [f32; 4], b: [f32; 4]) -> bool { + (0..4).all(|i| dst[i] >= a[i].min(b[i]) - 1e-5 && dst[i] <= a[i].max(b[i]) + 1e-5) + } + + #[test] + fn full_camera_then_layout_region_glides_directly() { + let regions = [region(5.0, 8.0, &[(1, FULL, true), (0, CORNER, false)])]; + let fulls = [full_camera(2.0, 5.0)]; + // The end of the Full Camera region keeps camera 0 on the frame: no shrink. + let before = at_with_full(®ions, &fulls, 5.0 - 0.01); + assert_eq!(before.len(), 1); + assert_eq!(before[0].camera, 0); + assert_eq!(before[0].dst, FULL); + assert!(before[0].fills_frame); + // The layout region's lead-in moves camera 0 from the frame to its corner. + let during = at_with_full(®ions, &fulls, 5.0 + TRANSITION_WINDOW_S / 2.0); + let cam0 = layer(&during, 0).expect("camera 0"); + for i in 2..4 { + assert!(strictly_between(cam0.dst[i], FULL[i], CORNER[i]), "dst[{i}] = {}", cam0.dst[i]); + } + let cam1 = layer(&during, 1).expect("camera 1 fading in"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0)); + // The default PiP never shows around the seam. + let mut t = 3.5f32; + while t <= 5.0 + TRANSITION_WINDOW_S + 0.1 { + let cam0 = layer(&at_with_full(®ions, &fulls, t), 0).expect("camera 0"); + assert!(on_the_path(cam0.dst, FULL, CORNER), "t = {t}: {:?}", cam0.dst); + assert_eq!(cam0.opacity, 1.0, "t = {t}"); + t += 0.005; + } + } + + #[test] + fn layout_region_then_full_camera_glides_directly() { + let regions = [region(2.0, 5.0, &[(1, FULL, true), (0, CORNER, false)])]; + let fulls = [full_camera(5.0, 8.0)]; + // The end of the layout region keeps its layers: no lead-out to the default PiP. + let before = at_with_full(®ions, &fulls, 5.0 - 0.01); + assert_eq!(layer(&before, 0).expect("camera 0").dst, CORNER); + assert_eq!(layer(&before, 1).expect("camera 1").opacity, 1.0); + // The Full Camera lead-in takes camera 0 from its corner to the frame; camera 1 fades. + let during = at_with_full(®ions, &fulls, 5.0 + TRANSITION_WINDOW_S / 2.0); + let cam0 = layer(&during, 0).expect("camera 0"); + for i in 2..4 { + assert!(strictly_between(cam0.dst[i], CORNER[i], FULL[i]), "dst[{i}] = {}", cam0.dst[i]); + } + let cam1 = layer(&during, 1).expect("camera 1 fading out"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0)); + // After the lead-in camera 0 is the default again, which the held phase keeps full. + let after = at_with_full(®ions, &fulls, 5.0 + TRANSITION_WINDOW_S + 0.01); + assert_eq!(after.len(), 1); + assert_eq!(after[0].dst, FULL); + let mut t = 3.5f32; + while t <= 7.0 { + let cam0 = layer(&at_with_full(®ions, &fulls, t), 0).expect("camera 0"); + assert!(on_the_path(cam0.dst, CORNER, FULL), "t = {t}: {:?}", cam0.dst); + assert_eq!(cam0.opacity, 1.0, "t = {t}"); + t += 0.005; + } + } + + #[test] + fn a_full_camera_region_off_the_seam_is_no_neighbour() { + // 10 ms apart: the layout region glides from the default as before. + let regions = [region(5.01, 8.0, &[(1, FULL, true), (0, CORNER, false)])]; + let fulls = [full_camera(2.0, 5.0)]; + assert_eq!( + at_with_full(®ions, &fulls, 5.01 + TRANSITION_WINDOW_S / 2.0), + at(®ions, 5.01 + TRANSITION_WINDOW_S / 2.0) + ); + let shrinking = layer(&at_with_full(®ions, &fulls, 4.9), 0).expect("camera 0"); + assert!(shrinking.dst[2] < 1.0, "the Full Camera region leads out: {:?}", shrinking.dst); + } + + #[test] + fn a_short_region_halves_its_windows() { + let regions = [region(2.0, 3.0, &[(1, FULL, true)])]; + // Both windows are 0.5 s: the middle is the only frame at full strength. + let mid = at(®ions, 2.5); + assert_eq!(mid.len(), 1); + assert_eq!(mid[0].camera, 1); + assert_eq!(mid[0].opacity, 1.0); + for t in [2.25, 2.75] { + let cam1 = layer(&at(®ions, t), 1).expect("camera 1"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0), "t = {t}"); + } + } + + #[test] + fn an_empty_region_draws_nothing_of_its_own() { + // start == end (and a reversed region) would otherwise match `t` at its one instant + // and draw its layers at full strength for that sample. + for regions in [ + [region(3.0, 3.0, &[(1, FULL, true)])], + [region(3.0, 2.0, &[(1, FULL, true)])], + ] { + for t in [2.0, 2.5, 3.0] { + let layers = at(®ions, t); + assert!(layer(&layers, 1).is_none(), "t = {t}"); + assert_eq!(layers.len(), 1, "t = {t}"); + assert_eq!(layers[0].camera, 0, "t = {t}"); + } + } + // Nor does it count as a neighbour: the region before it still leads out. + let regions = [region(2.0, 6.0, &[(1, FULL, true)]), region(6.0, 6.0, &[(1, FULL, true)])]; + let cam1 = layer(&at(®ions, 5.9), 1).expect("camera 1"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0)); + } + + #[test] + fn fills_frame_layers_are_drawn_first() { + let regions = [region(2.0, 6.0, &[(0, CORNER, false), (1, FULL, true)])]; + let order: Vec = at(®ions, 4.0).iter().map(|l| l.camera).collect(); + assert_eq!(order, vec![1, 0]); + // During the lead-in too. + let order: Vec = + at(®ions, 2.0 + TRANSITION_WINDOW_S / 2.0).iter().map(|l| l.camera).collect(); + assert_eq!(order, vec![1, 0]); + } +} diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index 198e6e637..e9505224c 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -52,6 +52,12 @@ use crate::scene::{Scene, SceneBackground, WallpaperMotion}; const LAYER_WGSL: &str = include_str!("vk_shaders/layer.wgsl"); const BLUR_WGSL: &str = include_str!("vk_shaders/blur.wgsl"); +/// `layer.wgsl` avec ou sans ses modeles 3D (modes 15 a 17) : la constante que le shader teste +/// pour les compiler, prefixee au source (pourquoi deux pipelines : le haut de `layer.wgsl`). +fn layer_source(models: bool) -> String { + format!("const LAYER_MODELS: bool = {models};\n{LAYER_WGSL}") +} + /// Budget du cache de textures image (`img_cache`), en octets. /// /// Doit tenir le JEU ACTIF d'une frame -- au pire un wallpaper d'ecran ET un @@ -61,16 +67,24 @@ const BLUR_WGSL: &str = include_str!("vk_shaders/blur.wgsl"); /// en parcourant les 18 wallpapers livres) en laissant le jeu actif resident. const IMG_CACHE_BUDGET_BYTES: u64 = 512 * 1024 * 1024; -/// Taille du buffer uniforme d'un calque : `LayerCB` entier (176 octets), le `struct Layer` de +/// Taille du buffer uniforme d'un calque : `LayerCB` entier (256 octets), le `struct Layer` de /// `layer.wgsl`. `blur.wgsl` n'en lit que les 128 premiers. const LAYER_BYTES: u64 = std::mem::size_of::() as u64; /// `&LayerCB` -> ses octets. `LayerCB` est `#[repr(C, align(16))]`, son layout EST le buffer -/// uniforme WGSL (dix vec4 et un vec2 + 2 f32 = 176 octets). +/// uniforme WGSL (seize vec4 = 256 octets). fn layer_bytes(cb: &LayerCB) -> &[u8] { unsafe { std::slice::from_raw_parts(cb as *const LayerCB as *const u8, LAYER_BYTES as usize) } } +/// Le maillage de Prism Glow tel que le lit le WGSL (`PrismMesh`) : les triangles, les +/// polygones, un point par vec4 (le pas d'un tableau uniform), puis les boîtes. +fn prism_mesh_bytes() -> Vec { + use crate::prism_mesh::{BOXES, POLYS, TRIS}; + let points = POLYS.iter().map(|&[x, y]| [x, y, 0.0, 0.0]); + TRIS.iter().copied().chain(points).chain(BOXES).flatten().flat_map(f32::to_ne_bytes).collect() +} + /// Un calque de fond deja lie, en attente de son `draw`. `_buf`/`_tex`/`_view` /// ne sont jamais relus : ils gardent en vie ce que le bind group reference /// jusqu'au submit. Ce backend encode toute la frame avant de la soumettre, la @@ -81,10 +95,25 @@ fn layer_bytes(cb: &LayerCB) -> &[u8] { /// ET le fond de la bulle webcam sont desormais construits par les memes /// methodes. struct BgDraw { - _buf: wgpu::Buffer, + layer: LayerBind, _tex: Option, _view: Option, +} + +/// Un calque pret a dessiner : son uniforme (qui doit vivre jusqu'au draw), son bind group, et +/// s'il est l'un des modeles 3D, que seul `pipeline_models` compile (`LayerCB::needs_models`). +/// `draw_layer` en tire le pipeline : c'est le mode du calque qui decide, pas le site d'appel. +struct LayerBind { + _buf: wgpu::Buffer, bind: wgpu::BindGroup, + models: bool, +} + +/// Le fond tel que la passe 1 (et son flou) l'a laisse dans le RT, et tout ce qui l'a decide +/// (`key`, cf. `compose_frame`). +struct BgCache { + key: crate::frame_geometry::BackgroundKey, + tex: wgpu::Texture, } /// Une copie RT -> staging DEJA SOUMISE, dont le mapping est arme mais pas @@ -262,16 +291,23 @@ pub struct Compositor { // Pipeline de calque (VS + FS `layer.wgsl`), sampler lineaire, bind group // layout (uniform + 2 textures + sampler). Immuables apres `new_sized`. + // `pipeline` n'a pas les modeles 3D (modes 15 a 17) : ils passent par `pipeline_models`. pipeline: wgpu::RenderPipeline, /// Meme shader et meme layout que `pipeline`, blend ADDITIF pondere par la /// constante de blend. Sert a sommer les copies de la trainee du curseur /// dans `accum` ; cf. `blend_add` cote Windows. pipeline_add: wgpu::RenderPipeline, + /// `pipeline` et `pipeline_add` avec les modeles 3D : le curseur modelise, l'impact de son + /// clic, l'appareil et son ombre, et rien d'autre (cf. `layer_source`, `draw_layer`). + pipeline_models: wgpu::RenderPipeline, + pipeline_add_models: wgpu::RenderPipeline, /// Copie plein ecran d'`accum` vers le RT en « over » premultiplie /// (`blur.wgsl` : `vs_fullscreen` + `fs_copy`). Utilise le layout du blur. pipeline_copy: wgpu::RenderPipeline, bind_group_layout: wgpu::BindGroupLayout, sampler: wgpu::Sampler, + /// Le maillage de Prism Glow (`prism_mesh.rs`) tel que le lit le WGSL (`PrismMesh`, binding 7). + prism_mesh: wgpu::Buffer, // Chaine de blur Kawase du fond (`blur.wgsl`) : layout dedie (uniform + 1 // tex + sampler), 2 pipelines (down/up), 3 textures de pyramide (1/2, 1/4, @@ -313,10 +349,15 @@ pub struct Compositor { readback_yuv: RefCell, // Etat pilote par live.rs (interior mutability : les methodes sont `&self`). + /// The frames of cameras 1..=3 (`set_extra_camera_frames`), as addresses (0 = none) so + /// the compositor's auto traits stay what they were without the raw pointers. + extra_camera_frames: std::cell::Cell<[usize; crate::camera_layers::MAX_EXTRA_CAMERAS]>, live_params: RefCell, scene: RefCell>, cursor: RefCell>, cursor_time: RefCell>, + /// Le métrage dans la dernière image composée, que l'éditeur lit avec elle (`live.rs`). + footage: std::cell::Cell>, timeline_time: RefCell>, /// Temps programme (secondes de sortie) -- cf. `FrameGeometryInput::programme_time`. programme_time: RefCell>, @@ -341,6 +382,9 @@ pub struct Compositor { /// touche depuis appartient au jeu actif et ne peut pas etre evince -- voir /// `cached_image`. img_frame_start: std::cell::Cell, + /// Le dernier fond fixe compose. Un fond qui ne bouge pas est le meme a chaque frame : on le + /// recopie au lieu de le redessiner (passe 1 de `compose_frame`). + bg_cache: RefCell>, /// Champs de distance des sprites de curseur (mode 15), R16F, par chemin, avec leur forme. /// Pas d'eviction : seuls les sprites du theme en cours y passent (~2,6 Mo pour les seize). sdf_cache: RefCell>, @@ -410,10 +454,14 @@ impl Compositor { feature_level: gpu.feature_level, }; - let module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("layer.wgsl"), - source: wgpu::ShaderSource::Wgsl(LAYER_WGSL.into()), - }); + let mk_module = |label: &str, models: bool| { + gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some(label), + source: wgpu::ShaderSource::Wgsl(layer_source(models).into()), + }) + }; + let module = mk_module("layer.wgsl", false); + let module_models = mk_module("layer.wgsl (modeles)", true); let sampler = gpu.device.create_sampler(&wgpu::SamplerDescriptor { label: Some("layer"), address_mode_u: wgpu::AddressMode::ClampToEdge, @@ -474,26 +522,44 @@ impl Compositor { // Pyramide de profondeur de champ (modes 8 et 18), `dummy` ailleurs : le // mode 18 lit a la fois le metrage, son rendu isole (binding 4) et elle. tex_entry(6), + // Le maillage de Prism Glow (mode 15). Un uniform et non des tables dans le + // shader : lavapipe les recopiait a chaque pixel de chaque calque, et une + // frame sans curseur se rendait 3,4 fois plus lentement. + wgpu::BindGroupLayoutEntry { + binding: 7, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, ], }); + let prism_mesh = gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("prism-mesh"), + contents: &prism_mesh_bytes(), + usage: wgpu::BufferUsages::UNIFORM, + }); let pipeline_layout = gpu.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("layer"), bind_group_layouts: &[&bind_group_layout], push_constant_ranges: &[], }); - // Deux pipelines pour le MEME shader de calque : seul le blend change. - let mk_layer = |label: &str, blend: wgpu::BlendState| { + // Deux pipelines par source de calque : seul le blend change. + let mk_layer = |label: &str, module: &wgpu::ShaderModule, blend: wgpu::BlendState| { gpu.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some(label), layout: Some(&pipeline_layout), vertex: wgpu::VertexState { - module: &module, + module, entry_point: Some("vs_main"), compilation_options: wgpu::PipelineCompilationOptions::default(), buffers: &[], }, fragment: Some(wgpu::FragmentState { - module: &module, + module, entry_point: Some("fs_main"), compilation_options: wgpu::PipelineCompilationOptions::default(), targets: &[Some(wgpu::ColorTargetState { @@ -512,22 +578,23 @@ impl Compositor { cache: None, }) }; - let pipeline = mk_layer("layer", wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING); + let over = wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING; + let pipeline = mk_layer("layer", &module, over); + let pipeline_models = mk_layer("layer-models", &module_models, over); // SOMME pondere : `src * constante + dst`. La constante (posee par pass // via `set_blend_constant`) vaut 1/taps, donc N copies d'un curseur // parfaitement immobile redonnent exactement ce curseur. Transcription // du `blend_add` D3D11 (BLEND_FACTOR / ONE / OP_ADD sur couleur ET // alpha) ; l'alpha doit suivre la couleur, sinon la somme n'est plus // premultipliee et la composition finale delave la trainee. - let add = wgpu::BlendComponent { + let sum = wgpu::BlendComponent { src_factor: wgpu::BlendFactor::Constant, dst_factor: wgpu::BlendFactor::One, operation: wgpu::BlendOperation::Add, }; - let pipeline_add = mk_layer( - "layer-add", - wgpu::BlendState { color: add, alpha: add }, - ); + let add = wgpu::BlendState { color: sum, alpha: sum }; + let pipeline_add = mk_layer("layer-add", &module, add); + let pipeline_add_models = mk_layer("layer-add-models", &module_models, add); // --- Chaine de blur Kawase du fond (`blur.wgsl`) --- let blur_module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { @@ -660,9 +727,12 @@ impl Compositor { render_h: h, pipeline, pipeline_add, + pipeline_models, + pipeline_add_models, pipeline_copy, bind_group_layout, sampler, + prism_mesh, blur_bgl, blur_down, blur_up, @@ -678,10 +748,12 @@ impl Compositor { readback, yuv: RefCell::new(None), readback_yuv, + extra_camera_frames: std::cell::Cell::new([0; crate::camera_layers::MAX_EXTRA_CAMERAS]), live_params: RefCell::new(LiveParams::default()), scene: RefCell::new(None), cursor: RefCell::new(None), cursor_time: RefCell::new(None), + footage: std::cell::Cell::new(None), timeline_time: RefCell::new(None), programme_time: RefCell::new(None), text_raster: crate::text::TextRasterizer::new().ok(), @@ -689,6 +761,7 @@ impl Compositor { sdf_cache: RefCell::new(std::collections::HashMap::new()), img_tick: std::cell::Cell::new(0), img_frame_start: std::cell::Cell::new(0), + bg_cache: RefCell::new(None), ann_copy, ann_copy_view, ann_copy_mips, @@ -749,6 +822,7 @@ impl Compositor { ann_copy, ann_copy_view, ann_copy_mips, + bg_cache: RefCell::new(None), ..self }) } @@ -817,7 +891,8 @@ impl Compositor { view_formats: &[], }) }; - let rt = mk("rt", wgpu::TextureUsages::COPY_SRC); + // COPY_DST : le fond fixe y est recopie (`bg_cache`). + let rt = mk("rt", wgpu::TextureUsages::COPY_SRC | wgpu::TextureUsages::COPY_DST); let accum = mk("accum", wgpu::TextureUsages::empty()); let rt_view = rt.create_view(&wgpu::TextureViewDescriptor::default()); let accum_view = accum.create_view(&wgpu::TextureViewDescriptor::default()); @@ -1095,6 +1170,9 @@ impl Compositor { pub fn set_scene(&self, s: Option) { *self.scene.borrow_mut() = s; + // A new scene can drop the regions that drew an extra camera: forget its frames so a + // stale pointer is never drawn before the next `set_extra_camera_frames`. + self.extra_camera_frames.set([0; crate::camera_layers::MAX_EXTRA_CAMERAS]); } pub fn set_cursor(&self, track: crate::cursor::CursorTrack) { @@ -1129,8 +1207,29 @@ impl Compositor { } /// Pas de cache de SRV cote wgpu (les `TextureView`s sont recreees a chaque - /// draw depuis le carrier) -- no-op conserve pour la symetrie d'API. - pub fn clear_srv_cache(&self) {} + /// draw depuis le carrier). Only the extra cameras' frame slots are reset. + pub fn clear_srv_cache(&self) { + // The extra cameras' frames belong to the decoders just closed: forget them too, so + // a stale pointer is never read before the next `set_extra_camera_frames`. + self.extra_camera_frames.set([0; crate::camera_layers::MAX_EXTRA_CAMERAS]); + } + + /// Frames for cameras 1..=3 (index 0 = scene camera 1). A null or missing entry means that + /// camera is not drawn this frame. The compositor keeps the pointers and reads them in every + /// `compose_frame` until the next call, so they must stay valid until then. + pub unsafe fn set_extra_camera_frames(&self, frames: &[*const AVFrame]) { + let mut slots = [0usize; crate::camera_layers::MAX_EXTRA_CAMERAS]; + for (slot, frame) in slots.iter_mut().zip(frames) { + *slot = *frame as usize; + } + self.extra_camera_frames.set(slots); + } + + /// The frame set for scene camera `camera` (1..=3), null when there is none. + fn extra_camera_frame(&self, camera: usize) -> *const AVFrame { + let slots = self.extra_camera_frames.get(); + camera.checked_sub(1).and_then(|k| slots.get(k).copied()).unwrap_or(0) as *const AVFrame + } // -- seam frame (lit le carrier `data[0]`) -- @@ -1251,24 +1350,16 @@ impl Compositor { fn draw_screen_group( &self, rpass: &mut wgpu::RenderPass<'_>, - shadow: &Option<(wgpu::Buffer, wgpu::BindGroup)>, - window_frame: &Option<(wgpu::Buffer, wgpu::BindGroup)>, - screen: &wgpu::BindGroup, - device_frame: &Option<(wgpu::Buffer, wgpu::BindGroup)>, + shadow: &Option, + window_frame: &Option, + screen: &LayerBind, + device_frame: &Option, ) { - if let Some((_buf, bind)) = shadow { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } - if let Some((_buf, bind)) = window_frame { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } - rpass.set_bind_group(0, screen, &[]); - rpass.draw(0..4, 0..1); - if let Some((_buf, bind)) = device_frame { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); + for layer in [shadow.as_ref(), window_frame.as_ref(), Some(screen), device_frame.as_ref()] + .into_iter() + .flatten() + { + self.draw_layer(rpass, layer, false); } } @@ -1277,12 +1368,13 @@ impl Compositor { cb: &LayerCB, planes: Option<(&wgpu::TextureView, &wgpu::TextureView, &wgpu::TextureView)>, dummy: &wgpu::TextureView, - ) -> (wgpu::Buffer, wgpu::BindGroup) { + ) -> LayerBind { self.make_bind_b4(cb, planes, dummy, None, None) } /// `make_bind`, binding 4 impose (le rendu isole du mode 18, a la place du masque webcam /// qu'il ne lit pas), et la pyramide de profondeur de champ en binding 6 (modes 8 et 18). + /// Le mode 15 y met le champ de son sprite et le sprite. fn make_bind_b4( &self, cb: &LayerCB, @@ -1290,7 +1382,7 @@ impl Compositor { dummy: &wgpu::TextureView, b4: Option<&wgpu::TextureView>, dof: Option<&wgpu::TextureView>, - ) -> (wgpu::Buffer, wgpu::BindGroup) { + ) -> LayerBind { let uniform = self.gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("layer-uniform"), contents: layer_bytes(cb), @@ -1338,9 +1430,29 @@ impl Compositor { binding: 6, resource: wgpu::BindingResource::TextureView(dof.unwrap_or(dummy)), }, + wgpu::BindGroupEntry { binding: 7, resource: self.prism_mesh.as_entire_binding() }, ], }); - (uniform, bind) + LayerBind { _buf: uniform, bind, models: cb.needs_models() } + } + + /// Dessine un calque par le pipeline plat que l'appelant a lie (`pipeline`, ou `pipeline_add` + /// si `add`) -- sauf un modele 3D, qui passe le temps de son draw par la variante + /// « modeles » de ce pipeline, la seule a le compiler (pourquoi : le haut de `layer.wgsl`). + fn draw_layer(&self, rpass: &mut wgpu::RenderPass<'_>, layer: &LayerBind, add: bool) { + rpass.set_bind_group(0, &layer.bind, &[]); + if layer.models { + let (models, flat) = if add { + (&self.pipeline_add_models, &self.pipeline_add) + } else { + (&self.pipeline_models, &self.pipeline) + }; + rpass.set_pipeline(models); + rpass.draw(0..4, 0..1); + rpass.set_pipeline(flat); + } else { + rpass.draw(0..4, 0..1); + } } /// Charge un PNG/JPEG (chemin fichier ou data URI) en texture RGBA8. Port @@ -1484,7 +1596,8 @@ impl Compositor { /// son propre repli, et un echec silencieux redonnerait le noir qu'on corrige. /// /// `motion` anime l'image au temps programme `programme_t` ; la bulle webcam - /// passe `WallpaperMotion::None`. + /// passe `WallpaperMotion::None`. `transparency` = `layer_fx.x` (0 = opaque): a camera + /// layer's background box fades with the layer. #[allow(clippy::too_many_arguments)] fn image_bg_draw( &self, @@ -1495,6 +1608,7 @@ impl Compositor { aspect: f32, motion: WallpaperMotion, programme_t: f32, + transparency: f32, dummy: &wgpu::TextureView, ) -> Result { let (tex, iw, ih) = self.cached_image(path)?; @@ -1516,11 +1630,12 @@ impl Compositor { mode: 6.0, fx: [0.0, 0.0, anim[0], anim[1]], mb, + layer_fx: [transparency, 0.0, 0.0, 0.0], ..Default::default() }; let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); - let (buf, bind) = self.make_bind(&cb, Some((&view, &view, &view)), dummy); - Ok(BgDraw { _buf: buf, _tex: Some(tex), _view: Some(view), bind }) + let layer = self.make_bind(&cb, Some((&view, &view, &view)), dummy); + Ok(BgDraw { layer, _tex: Some(tex), _view: Some(view) }) } /// Prepare le fond du mode « personnalise », peint DANS la bulle webcam juste @@ -1536,25 +1651,27 @@ impl Compositor { /// /// `quad_px` / `radius_px` sont ceux de la bulle : le fond doit epouser ses /// coins arrondis, sinon un rectangle deborde derriere la camera. + /// + /// `transparency` = the camera layer's `layer_fx.x`: the box fades with the camera. fn webcam_bg_draw( &self, bg: Option<&SceneBackground>, dst: [f32; 4], quad_px: [f32; 2], radius_px: f32, + transparency: f32, dummy: &wgpu::TextureView, ) -> BgDraw { const BLACK: [f32; 4] = [0.0, 0.0, 0.0, 1.0]; - let flat = |cb: LayerCB| { - let (buf, bind) = self.make_bind(&cb, None, dummy); - BgDraw { _buf: buf, _tex: None, _view: None, bind } - }; + let layer_fx = [transparency, 0.0, 0.0, 0.0]; + let flat = |cb: LayerCB| BgDraw { layer: self.make_bind(&cb, None, dummy), _tex: None, _view: None }; let solid = |color: [f32; 4]| LayerCB { dst, quad_px, radius_px, mode: 1.0, color, + layer_fx, ..Default::default() }; match bg { @@ -1572,6 +1689,7 @@ impl Compositor { quad_px, radius_px, fx: [a.sin(), -a.cos(), 0.0, 0.0], + layer_fx, ..crate::frame_geometry::gradient_layer(stops, offsets, BLACK) }) } @@ -1580,7 +1698,10 @@ impl Compositor { // elle que l'image doit remplir sans etirement. let aspect = if quad_px[1] > 0.0 { quad_px[0] / quad_px[1] } else { 1.0 }; let still = WallpaperMotion::None; - match self.image_bg_draw(path, dst, quad_px, radius_px, aspect, still, 0.0, dummy) { + let drawn = self.image_bg_draw( + path, dst, quad_px, radius_px, aspect, still, 0.0, transparency, dummy, + ); + match drawn { Ok(d) => d, Err(e) => { // Meme contrat que le fond d'ecran : un chemin casse est @@ -1685,7 +1806,7 @@ impl Compositor { // `color.a = 1` n'est pas decoratif : `fs_main` calcule son alpha en // `layer.color.a * alpha_mask`, donc le defaut (0) rendrait un quad // entierement transparent. - let (_uniform, bind) = self.make_bind( + let layer = self.make_bind( &LayerCB { dst: [0.0, 0.0, 1.0, 1.0], src, @@ -1718,8 +1839,7 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline); - rpass.set_bind_group(0, &bind, &[]); - rpass.draw(0..4, 0..1); + self.draw_layer(&mut rpass, &layer, false); } encoder.copy_texture_to_buffer( wgpu::TexelCopyTextureInfo { @@ -2024,6 +2144,11 @@ impl Compositor { *self.webcam_mask.borrow_mut() = None; } + /// Le métrage dans la dernière image composée : ses coins et son warp (`FootageQuad`). + pub fn footage_quad(&self) -> Option { + self.footage.get() + } + /// Rend une frame dans le RT interne. Le screen `screen`/`webcam` sont des /// carriers `linux_frames` ; la geometrie vient de `plan_frame`. Coeur : /// fond uni + ecran cover-fit. `readback_direct` lit ensuite le RT. @@ -2099,6 +2224,7 @@ impl Compositor { timeline_t_override: *self.timeline_time.borrow(), programme_time: *self.programme_time.borrow(), }); + self.footage.set(Some(g.footage_quad([rw, rh]))); // (`wtw`/`wth` sont les dims de la TEXTURE webcam, consommees par le // cover-crop du calque PiP plus bas.) @@ -2210,12 +2336,12 @@ impl Compositor { g.tilt_pixel_trail([rw, rh]), ), }; - // Bind group construit AVANT le pass (doit vivre pendant tout le pass) ; - // `_screen_uniform` garde le buffer uniforme en vie (reference par le bind). + // Bind group construit AVANT le pass (doit vivre pendant tout le pass) ; `LayerBind` + // garde le buffer uniforme en vie (reference par le bind). let dummy = self.dummy_view(); // Binding 6 du mode 8 : la pyramide si l'effet tourne, sinon `dummy` (`k = 0`, le // shader n'y lit rien). - let (_screen_uniform, screen_bind) = self.make_bind_b4( + let screen_bind = self.make_bind_b4( &screen_layer, Some((&sy, &su, &sv)), &dummy, @@ -2291,18 +2417,30 @@ impl Compositor { ) }); + // Le fond de la frame d'avant, s'il est celui qu'on demande (`BackgroundKey`) : la passe 1 + // le recopie au lieu de le redessiner. + let bg_desc = scene_ref.as_ref().map(|s| &s.background); + let bg_cached = self + .bg_cache + .borrow() + .as_ref() + .is_some_and(|c| c.key.matches(bg_desc, cfg.bg_blur, lp.bg_color, [rw, rh])); + let bg_wanted = bg_layer.is_some(); + // Fond (gradient mode 5 OU image mode 6), dessine dans la passe de fond. - let bg_draw = bg_layer.and_then(|bl| match bl { + let bg_draw = bg_layer.filter(|_| !bg_cached).and_then(|bl| match bl { BgLayer::Gradient(cb) => { - let (buf, bind) = self.make_bind(&cb, None, &dummy); - Some(BgDraw { _buf: buf, _tex: None, _view: None, bind }) + Some(BgDraw { layer: self.make_bind(&cb, None, &dummy), _tex: None, _view: None }) } // Le wallpaper couvre tout le cadre, donc dst plein et pas de coins : // `image_bg_draw` sert aussi la bulle webcam, qui elle en a. BgLayer::Image(path, motion) => { let full = [0.0, 0.0, 1.0, 1.0]; let t = g.programme_t; - match self.image_bg_draw(&path, full, [0.0, 0.0], 0.0, rw / rh, motion, t, &dummy) { + let drawn = self.image_bg_draw( + &path, full, [0.0, 0.0], 0.0, rw / rh, motion, t, 0.0, &dummy, + ); + match drawn { Ok(d) => Some(d), Err(e) => { eprintln!("[fond image] \"{path}\" : {e:#}"); @@ -2334,7 +2472,10 @@ impl Compositor { // lancement gracieux, le temps que l'inference rende son premier masque. // // Calcule ICI, avant le draw comme avant l'ombre : les deux en dependent. - let (effect_code, blur_intensity, webcam_bg) = { + // Without layout regions camera 0 is drawn from `w_dst` (`webcam_bg`, `webcam_draw`, + // `webcam_shadow` below); with them every planned layer is (`camera_layer_draws`). + let unplanned = g.camera_layers.is_empty(); + let (effect_code, blur_intensity, custom_bg) = { let has_mask = self.webcam_mask.borrow().is_some(); let effect = scene_ref .as_ref() @@ -2350,24 +2491,16 @@ impl Compositor { // fond. Le shader ne sait peindre qu'une couleur plate sous le // masque ; degrades et images y tombaient sur du noir, et le defaut // EST une image. - Some((code, e)) if code > 2.5 => { - // Sans piste webcam le fond peindrait un rectangle seul dans le - // cadre : il ne se prepare que si la camera se dessine. - let bg = webcam_planes.is_some().then(|| { - self.webcam_bg_draw( - e.background.as_ref(), - g.w_dst, - g.w_px, - g.w_radius, - &dummy, - ) - }); - (1.0, 0.0, bg) - } + Some((code, e)) if code > 2.5 => (1.0, 0.0, Some(e.background.as_ref())), Some((code, e)) => (code, e.blur_intensity.clamp(0.0, 1.0), None), None => (0.0, 0.0, None), } }; + // Sans piste webcam le fond peindrait un rectangle seul dans le cadre : il ne se + // prepare que si la camera se dessine. + let webcam_bg = custom_bg + .filter(|_| unplanned && webcam_planes.is_some()) + .map(|bg| self.webcam_bg_draw(bg, g.w_dst, g.w_px, g.w_radius, 0.0, &dummy)); // L'ombre se juge sur le mode DE LA SCENE, pas sur `effect_code` : le fond // personnalise se compose desormais en detourage (code 1) tout en gardant // sa bulle, et tester le code compose la lui retirerait. Meme lecture que @@ -2376,7 +2509,10 @@ impl Compositor { scene_ref.as_ref().and_then(|s| s.webcam_effect.as_ref()), Some(e) if e.shader_code() == 1.0 ) && self.webcam_mask.borrow().is_some(); - let webcam_draw = webcam_planes.as_ref().map(|(wy, wu, wv)| { + // Camera 0's crop, mirror and desk turn as camera settings, plus its homography + // (`Scene::camera(0)`): with one, `src` is not read and the crop is dropped. + let cam0 = g.camera0_settings(scene_ref.as_ref()); + let webcam_draw = webcam_planes.as_ref().filter(|_| unplanned).map(|(wy, wu, wv)| { // COVER-CROP. `src` etait cable a [0,0,1,1], donc la texture entiere // etait etiree sur la boite quelle que soit sa forme : le facteur de // deformation valait exactement `box_ar / cam_ar`. Invisible en PiP @@ -2386,45 +2522,27 @@ impl Compositor { // // `cover_crop_uv` est la primitive partagee que macOS et Windows // utilisent ; elle rend le rect inchange quand il a deja le bon - // ratio, donc aucun placement correct ne bouge. - let [cu0, cv0, cu1, cv1] = crate::frame_geometry::webcam_source_rect( + // ratio, donc aucun placement correct ne bouge. Mirror and the desk-shot + // turn are both bound swaps: u for horizontal, v for vertical. + let src = crate::frame_geometry::camera_source_rect( + Some(&cam0), [wcw, wch], [wtw as f32, wth as f32], - scene_ref - .as_ref() - .and_then(|scene| scene.layout.webcam_crop), g.w_px[0] / g.w_px[1].max(0.0001), ); - // MIROIR : on inverse l'intervalle u. Le VS interpole `src` - // lineairement et `fs_main` ne re-clampe pas `i.uv`, donc un - // intervalle a l'envers suffit -- aucune retouche du WGSL. Apres le - // cover-crop les deux bornes sont strictement a l'interieur de la - // texture, donc le sampler ClampToEdge ne bave pas sur les bords. - let (u0, u1) = if lp.webcam_mirror { (cu1, cu0) } else { (cu0, cu1) }; // `src_prev` doit valoir EXACTEMENT le `src` de ce draw, miroir // compris : le shader s'en sert pour reconstruire l'UV de la frame // precedente, et un rect source qui ne correspond pas au calque // dessine ferait diverger la trainee vers une zone de la texture qui // n'a jamais ete affichee. Seul `dst_prev` porte le mouvement. - let cb = LayerCB { - dst: g.w_dst, - src: [u0, cv0, u1, cv1], - quad_px: g.w_px, - radius_px: g.w_radius, - mode: 0.0, - // `color.a` porte l'alpha du decoupage (`color.a * personne`) ; le - // RGB n'est plus lu, le fond ayant deja ete peint sous la camera. - color: [0.0, 0.0, 0.0, 1.0], - // `fx.xy` = etendue valide de la texture webcam, par quoi le - // shader divise `uv` pour retomber dans l'espace du masque ; - // `fx.z` = mode, `fx.w` = intensite du flou. Contrat commun aux - // trois back-ends, cf. `layer.wgsl` et `webcam-segmentation.md`. - fx: [w_valid[0], w_valid[1], effect_code, blur_intensity], - src_prev: [u0, cv0, u1, cv1], - dst_prev: g.w_dst_prev, - mb: [g.mb_taps, g.mb_amount, 1.0, 0.0], - ..Default::default() - }; + // `fx.xy` = etendue valide de la texture webcam, par quoi le + // shader divise `uv` pour retomber dans l'espace du masque ; + // `fx.z` = mode, `fx.w` = intensite du flou. Contrat commun aux + // trois back-ends, cf. `layer.wgsl` et `webcam-segmentation.md`. + let cb = crate::frame_geometry::with_camera_homography( + g.webcam_video_cb(src, w_valid, effect_code, blur_intensity), + Some(&cam0), + ); // Le masque est lie par `make_bind` sur tous les draws, pas seulement // celui-ci : le layout l'exige (cf. `tex_entry(4)`). self.make_bind(&cb, Some((wy, wu, wv)), &dummy) @@ -2458,6 +2576,71 @@ impl Compositor { self.make_bind(&cb, None, &dummy) }); + // Every planned camera layer, in order, each as shadow, then (camera 0) the custom + // background, then the video: one uniform buffer and bind group per draw, drawn in + // the "webcam-pass" below. Camera 0 keeps its frame, effects and mask; the extra + // cameras draw from `set_extra_camera_frames` without effects, and a camera without a + // frame is skipped. `extra_planes` keeps their views alive until the pass is encoded. + let mut camera_layer_draws: Vec = Vec::new(); + let mut extra_planes = Vec::new(); + let render = [rw, rh]; + let cam0_base = g.webcam_video_cb([0.0; 4], w_valid, effect_code, blur_intensity); + let plain = |layer: LayerBind| BgDraw { layer, _tex: None, _view: None }; + for plan in &g.camera_layers { + if plan.camera == 0 { + // `webcam_planes` is already gated on `lp.has_webcam`. + let Some((wy, wu, wv)) = webcam_planes.as_ref() else { continue }; + let video = crate::frame_geometry::camera_layer_cb( + plan, + Some(&cam0), + [wcw, wch], + [wtw as f32, wth as f32], + render, + &cam0_base, + ); + if let Some(shadow) = + g.camera_layer_shadow(plan, &video, render, cfg.shadow, is_cutout) + { + camera_layer_draws.push(plain(self.make_bind(&shadow, None, &dummy))); + } + if let Some(bg) = custom_bg { + let (dst, quad_px, radius_px) = (video.dst, video.quad_px, video.radius_px); + let t = video.layer_fx[0]; + let bg = self.webcam_bg_draw(bg, dst, quad_px, radius_px, t, &dummy); + camera_layer_draws.push(bg); + } + let bind = self.make_bind(&video, Some((wy, wu, wv)), &dummy); + camera_layer_draws.push(plain(bind)); + } else { + let frame = self.extra_camera_frame(plan.camera); + if Self::pixel_buffer_of(frame).is_none() { + continue; + } + let Ok(planes) = self.nv12_srvs(frame) else { continue }; + let (tw, th) = self.tex_dims(frame); + let visible = [(*frame).width as f32, (*frame).height as f32]; + let tex = [tw as f32, th as f32]; + let valid = [visible[0] / tex[0].max(1.0), visible[1] / tex[1].max(1.0)]; + let video = crate::frame_geometry::camera_layer_cb( + plan, + scene_ref.as_ref().and_then(|s| s.camera(plan.camera)), + visible, + tex, + render, + &crate::frame_geometry::extra_camera_base_cb(valid), + ); + if let Some(shadow) = + g.camera_layer_shadow(plan, &video, render, cfg.shadow, false) + { + camera_layer_draws.push(plain(self.make_bind(&shadow, None, &dummy))); + } + let (ey, eu, ev) = &planes; + let bind = self.make_bind(&video, Some((ey, eu, ev)), &dummy); + camera_layer_draws.push(plain(bind)); + extra_planes.push(planes); + } + } + // ANNOTATIONS -- calque le plus haut, place relativement au rect ecran // (les coords x/y/w/h de l'annotation sont des fractions de ce rect, cf. // `scene.rs`). Le rect est `g.s_ann`, l'ecran SANS ZOOM, et surtout pas @@ -2475,16 +2658,15 @@ impl Compositor { // replis, meme ordre. Seul le texte diverge, tinte cote shader (atlas R8) // au lieu d'une couleur bakee dans la texture. struct AnnDraw { - _buf: wgpu::Buffer, + layer: LayerBind, /// Gardent l'atlas / la texture image en vie jusqu'au submit. `None` /// pour les quads qui n'echantillonnent rien (plaque de fond, fleche). _glyphs: Option, _tex: Option, - bind: wgpu::BindGroup, } impl AnnDraw { - fn plain(buf: wgpu::Buffer, bind: wgpu::BindGroup) -> AnnDraw { - AnnDraw { _buf: buf, _glyphs: None, _tex: None, bind } + fn plain(layer: LayerBind) -> AnnDraw { + AnnDraw { layer, _glyphs: None, _tex: None } } } // FENETRE TEMPORELLE. Sans ce test, TOUTES les annotations du projet sont @@ -2502,6 +2684,8 @@ impl Compositor { .as_ref() .is_some_and(|s| s.annotations.iter().any(|a| a.kind == "blur" && visible(a))); let mut ann_draws: Vec = Vec::new(); + // Les flous de confidentialite, dessines sur le metrage avant le curseur. + let mut privacy_draws: Vec = Vec::new(); if let Some(scene) = scene_ref.as_ref() { // La liste arrive deja triee par zIndex cote app : l'ordre d'iteration // EST l'ordre de peinture. @@ -2543,8 +2727,7 @@ impl Compositor { mb: [1.0, half_stroke, 0.0, 0.0], ..Default::default() }; - let (buf, bind) = self.make_bind(&cb, None, &dummy); - ann_draws.push(AnnDraw::plain(buf, bind)); + ann_draws.push(AnnDraw::plain(self.make_bind(&cb, None, &dummy))); } "blur" => { let Some(blur) = a.blur.as_ref() else { continue }; @@ -2587,12 +2770,12 @@ impl Compositor { }; // La copie mipmappee au binding 1 (texY), la ou le mode 10 // la lit. - let (buf, bind) = self.make_bind( + let layer = self.make_bind( &cb, Some((&self.ann_copy_view, &self.ann_copy_view, &self.ann_copy_view)), &dummy, ); - ann_draws.push(AnnDraw::plain(buf, bind)); + privacy_draws.push(AnnDraw::plain(layer)); } "image" => { let Some(src) = a.image_path.as_ref().filter(|s| !s.is_empty()) else { @@ -2649,13 +2832,8 @@ impl Compositor { fx: [0.0, 0.0, 1.0, 1.0], ..Default::default() }; - let (buf, bind) = self.make_bind(&cb, Some((&view, &view, &view)), &dummy); - ann_draws.push(AnnDraw { - _buf: buf, - _glyphs: None, - _tex: Some(tex), - bind, - }); + let layer = self.make_bind(&cb, Some((&view, &view, &view)), &dummy); + ann_draws.push(AnnDraw { layer, _glyphs: None, _tex: Some(tex) }); } "text" => { let Some(raster) = self.text_raster.as_ref() else { continue }; @@ -2663,6 +2841,13 @@ impl Compositor { if text.content.trim().is_empty() { continue; } + // The desk label shows only while the camera is covered: skip the rest of + // its section before rasterizing anything. + if crate::text_anim::is_desk_cover(text.animation.as_deref()) + && g.webcam_cover <= 0.0 + { + continue; + } let color = parse_hex(&text.color).unwrap_or([1.0, 1.0, 1.0, 1.0]); let background = parse_hex(&text.background_color).unwrap_or([0.0, 0.0, 0.0, 0.0]); @@ -2697,10 +2882,11 @@ impl Compositor { // et remis a l'echelle de la sortie, comme la taille de // police : en px absolus la meme animation sauterait deux // fois plus haut dans un rendu 4K que dans l'apercu. - let anim = crate::text_anim::text_animation_state( + let anim = crate::text_anim::annotation_text_state( text.animation.as_deref(), (g.source_t - a.start_sec as f32) * 1000.0, ((a.end_sec - a.start_sec) * 1000.0) as f32, + g.webcam_cover, ); let anim_px = rh / crate::text_anim::ANIMATION_REFERENCE_HEIGHT; let (mut ax, mut ay, mut aw, mut ah) = ( @@ -2784,8 +2970,7 @@ impl Compositor { ), ..Default::default() }; - let (pbuf, pbind) = self.make_bind(&plate, None, &dummy); - ann_draws.push(AnnDraw::plain(pbuf, pbind)); + ann_draws.push(AnnDraw::plain(self.make_bind(&plate, None, &dummy))); } } @@ -2798,14 +2983,9 @@ impl Compositor { ..Default::default() }; // Atlas R8 au binding 1 (texY) que le mode 11 echantillonne. - let (buf, bind) = + let layer = self.make_bind(&cb, Some((&glyphs.view, &glyphs.view, &glyphs.view)), &dummy); - ann_draws.push(AnnDraw { - _buf: buf, - _glyphs: Some(glyphs), - _tex: None, - bind, - }); + ann_draws.push(AnnDraw { layer, _glyphs: Some(glyphs), _tex: None }); } _ => {} } @@ -2822,12 +3002,18 @@ impl Compositor { // sont PAS dessinees sur le RT mais dans `accum`, puis compositees en une // fois -- cf. le commentaire au point de dessin. struct CursorDraw { - _bufs: Vec, /// Le sprite, et pour le curseur modelise son champ de distance. _tex: Vec<(wgpu::Texture, wgpu::TextureView)>, - binds: Vec, + /// Une copie par echantillon de la trainee, le sprite (modes 7 et 13) ou le curseur + /// modelise (mode 15). + binds: Vec, + /// L'union des quads de `binds` (x0, y0, x1, y1 en fractions de la sortie) : tout ce + /// que la trainee peint dans `accum`, donc tout ce que sa recopie doit relire. + bounds: [f32; 4], /// L'impact des clics (mode 16), dessine SOUS le curseur, sur le RT. - impacts: Vec, + impacts: Vec, + /// Le cristal de Prism Glow : il lit la copie de l'image composee (`CursorPlan::glass`). + glass: bool, } let cursor_draw: Option = (|| { let track = cursor_ref.as_ref()?; @@ -2870,13 +3056,18 @@ impl Compositor { }) .collect() }; - let (mut bufs, mut binds) = (Vec::new(), Vec::new()); - let mut impacts = Vec::new(); - for cb in &plan.impacts { - let (buf, bind) = self.make_bind(cb, None, &dummy); - bufs.push(buf); - impacts.push(bind); - } + let mut binds = Vec::new(); + let mut bounds = [f32::MAX, f32::MAX, f32::MIN, f32::MIN]; + let mut grow = |dst: [f32; 4]| { + bounds = [ + bounds[0].min(dst[0]), + bounds[1].min(dst[1]), + bounds[2].max(dst[0] + dst[2]), + bounds[3].max(dst[1] + dst[3]), + ]; + }; + let impacts: Vec = + plan.impacts.iter().map(|cb| self.make_bind(cb, None, &dummy)).collect(); let sprites = scene_ref .as_ref() @@ -2906,7 +3097,9 @@ impl Compositor { let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); // Curseur modelise (mode 15) : ce meme sprite extrude, `plan_cursor` en a tire la - // pose. Sprite au binding 1 (texY), champ au binding 2 (texU). Parite Windows/macOS. + // pose. La copie de l'image composee au binding 1 (texY), que refracte le cristal de + // Prism Glow, le champ au binding 4 (texMask), le sprite au binding 6 (texDof). + // Parite Windows/macOS. if let Some(pose) = plan.model { match self.cursor_sdf(&sprite.path) { Ok((sdf, shape)) => { @@ -2923,16 +3116,21 @@ impl Compositor { ) else { continue; }; - let (buf, bind) = - self.make_bind(&cb, Some((&view, &sdf_view, &view)), &dummy); - bufs.push(buf); - binds.push(bind); + binds.push(self.make_bind_b4( + &cb, + Some((&self.ann_copy_view, &dummy, &dummy)), + &dummy, + Some(&sdf_view), + Some(&view), + )); + grow(cb.dst); } return (!binds.is_empty()).then_some(CursorDraw { - _bufs: bufs, _tex: vec![(tex, view), (sdf, sdf_view)], binds, + bounds, impacts, + glass: plan.glass, }); } Err(e) => eprintln!("[curseur] champ de \"{}\" : {e:#}", sprite.path), @@ -2952,11 +3150,16 @@ impl Compositor { [rw, rh], ); // Sprite RGBA au binding 1 (texY) que le mode 7 echantillonne. - let (buf, bind) = self.make_bind(&cb, Some((&view, &view, &view)), &dummy); - bufs.push(buf); - binds.push(bind); + binds.push(self.make_bind(&cb, Some((&view, &view, &view)), &dummy)); + grow(cb.dst); } - Some(CursorDraw { _bufs: bufs, _tex: vec![(tex, view)], binds, impacts }) + Some(CursorDraw { + _tex: vec![(tex, view)], + binds, + bounds, + impacts, + glass: false, + }) })(); // Bind group de la passe de composition d'`accum` (layout du blur : // uniform + texture + sampler). Construit hors de la pass, comme les @@ -2998,7 +3201,7 @@ impl Compositor { }); // Passe 0 : pyramide de profondeur de champ, niveau 0 vide d'abord (le « over » // rend alors la source telle quelle), puis ses mips. - if let (Some(p), Some((_buf, bind))) = (dof_pyramid, &dof_fill) { + if let (Some(p), Some(fill)) = (dof_pyramid, &dof_fill) { { let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("dof-pyramid-pass"), @@ -3015,8 +3218,7 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline); - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); + self.draw_layer(&mut rpass, fill, false); } self.generate_mips(&mut encoder, &p.mips); } @@ -3039,38 +3241,74 @@ impl Compositor { rpass.set_pipeline(&self.pipeline); self.draw_screen_group(&mut rpass, &screen_shadow, &window_frame, &screen_bind, &device_frame); } - // Passe 1 : fond (clear a `bg_clear` + gradient mode 5 eventuel). - { - let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("bg-pass"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: &self.rt_view, - resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color { - r: bg_clear[0] as f64, - g: bg_clear[1] as f64, - b: bg_clear[2] as f64, - a: bg_clear[3] as f64, - }), - store: wgpu::StoreOp::Store, - }, - })], - depth_stencil_attachment: None, - timestamp_writes: None, - occlusion_query_set: None, - }); - if let Some(bg) = &bg_draw { - rpass.set_pipeline(&self.pipeline); - rpass.set_bind_group(0, &bg.bind, &[]); - rpass.draw(0..4, 0..1); + // Passe 1 : fond (clear a `bg_clear` + gradient mode 5 eventuel) puis son flou -- ou, s'il + // n'a pas change (`BackgroundKey`), la copie qu'en a gardee la frame d'avant. + let extent = wgpu::Extent3d { + width: self.render_w, + height: self.render_h, + depth_or_array_layers: 1, + }; + if bg_cached { + let cache = self.bg_cache.borrow(); + let tex = &cache.as_ref().expect("bg_cached").tex; + encoder.copy_texture_to_texture(tex.as_image_copy(), self.rt.as_image_copy(), extent); + } else { + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("bg-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color { + r: bg_clear[0] as f64, + g: bg_clear[1] as f64, + b: bg_clear[2] as f64, + a: bg_clear[3] as f64, + }), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + if let Some(bg) = &bg_draw { + rpass.set_pipeline(&self.pipeline); + self.draw_layer(&mut rpass, &bg.layer, false); + } + } + // Blur du fond (avant l'ecran), si active par la scene/l'inspector. + self.blur_bg(&mut encoder, cfg.bg_blur); + // Pas de cache pour un fond anime, ni pour un fond qui n'a pas pu se charger : la frame + // suivante reessaie. + let key = (bg_draw.is_some() == bg_wanted) + .then(|| { + crate::frame_geometry::BackgroundKey::of(bg_desc, cfg.bg_blur, lp.bg_color, [rw, rh]) + }) + .flatten(); + if let Some(key) = key { + let mut cache = self.bg_cache.borrow_mut(); + let tex = match cache.take() { + Some(c) if c.tex.size() == extent => c.tex, + _ => self.gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("bg-cache"), + size: extent, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::COPY_SRC | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }), + }; + encoder.copy_texture_to_texture(self.rt.as_image_copy(), tex.as_image_copy(), extent); + *cache = Some(BgCache { key, tex }); } } - // Blur du fond (avant l'ecran), si active par la scene/l'inspector. - self.blur_bg(&mut encoder, cfg.bg_blur); - // Passe 2 : avant-plan (ecran + webcam), compose par-dessus le fond - // (eventuellement floute) avec `LoadOp::Load`. Les annotations sont dans - // une passe a part, cf. plus bas. + // Passe 2 : l'ecran, compose par-dessus le fond (eventuellement floute) avec + // `LoadOp::Load`. Puis les flous de confidentialite, le curseur, la camera et les autres + // annotations, dans cet ordre : celui de Windows et macOS. { let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("fg-pass"), @@ -3091,10 +3329,7 @@ impl Compositor { // elle doit passer sous lui mais au-dessus du fond (et, pour la // camera, au-dessus de l'ecran). match &trail { - Some((_buf, bind)) => { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } + Some(layer) => self.draw_layer(&mut rpass, layer, false), None => self.draw_screen_group( &mut rpass, &screen_shadow, @@ -3103,40 +3338,22 @@ impl Compositor { &device_frame, ), } - if let Some((_buf, bind)) = &webcam_shadow { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } - // Fond personnalise : ENTRE l'ombre et la camera. C'est ce sandwich qui - // remplace la branche « mode 3 » du shader — la camera, decoupee, se - // fond dessus par alpha ; l'ombre reste dessous, elle appartient a la - // bulle et non a son contenu. - if let Some(bg) = &webcam_bg { - rpass.set_bind_group(0, &bg.bind, &[]); - rpass.draw(0..4, 0..1); - } - if let Some((_buf, bind)) = &webcam_draw { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } } // Fige la frame composee pour les annotations « flou ». ICI et nulle part - // ailleurs : apres l'ecran et la camera (sinon un flou masquerait du vide) - // et avant la premiere annotation (sinon deux flous qui se recouvrent + // ailleurs : apres l'ecran (sinon un flou masquerait du vide) + // et avant le premier flou (sinon deux flous qui se recouvrent // s'echantillonnent l'un l'autre). Une passe de rendu ne peut pas lire sa // propre cible, d'ou la copie -- et d'ou le fait que les annotations // doivent avoir leur propre passe. if needs_ann_copy { self.generate_ann_mips(&mut encoder); } - // Passe 3 : annotations puis curseur net, par-dessus tout le reste. Elle - // existe meme sans flou : deux passes consecutives sur la MEME cible avec - // `LoadOp::Load` ne coutent rien de plus qu'une seule sur un GPU - // desktop, et un seul chemin de code vaut mieux qu'un branchement qui ne - // serait exerce que dans un projet sur dix. - { + // Passe 2 bis : les flous de confidentialite, sur le metrage, AVANT le curseur (parite + // Windows et macOS). Le curseur reste net par-dessus, et le cristal de Prism Glow, qui + // refracte l'image composee, n'y voit que des pixels deja floutes. + if !privacy_draws.is_empty() { let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("ann-pass"), + label: Some("privacy-pass"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { view: &self.rt_view, resolve_target: None, @@ -3150,21 +3367,44 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline); - for a in &ann_draws { - rpass.set_bind_group(0, &a.bind, &[]); - rpass.draw(0..4, 0..1); + for a in &privacy_draws { + self.draw_layer(&mut rpass, &a.layer, false); } + } + // Le cristal de Prism Glow refracte l'image telle qu'elle est composee a cet instant, + // flous compris : sa copie dans `ann_copy` (libre, les flous l'ont deja lue ; le mip 0 + // suffit), une fois pour toutes les copies de la trainee. + if cursor_draw.as_ref().is_some_and(|c| c.glass) { + encoder.copy_texture_to_texture( + self.rt.as_image_copy(), + self.ann_copy.as_image_copy(), + wgpu::Extent3d { width: self.render_w, height: self.render_h, depth_or_array_layers: 1 }, + ); + } + // Le curseur net, et l'impact des clics sous lui. La trainee, elle, a besoin de sa + // propre cible : elle suit. + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("cursor-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline); // L'impact des clics, sous le curseur et sa trainee (dessinee plus bas). - for bind in cursor_draw.iter().flat_map(|c| &c.impacts) { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); + for layer in cursor_draw.iter().flat_map(|c| &c.impacts) { + self.draw_layer(&mut rpass, layer, false); } - // Curseur en dernier : au-dessus de l'ecran et des annotations. - // Une seule copie = curseur net, il tient dans cette pass. La - // trainee, elle, a besoin de sa propre cible (voir plus bas). if let Some(c) = cursor_draw.as_ref().filter(|c| c.binds.len() == 1) { - rpass.set_bind_group(0, &c.binds[0], &[]); - rpass.draw(0..4, 0..1); + self.draw_layer(&mut rpass, &c.binds[0], false); } } // TRAINEE DU CURSEUR : flou REEL, pas des copies discretes. @@ -3200,11 +3440,10 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline_add); - for (k, bind) in c.binds.iter().enumerate() { + for (k, layer) in c.binds.iter().enumerate() { let w = crate::frame_geometry::cursor_tap_weight(k as u32, c.binds.len() as u32) as f64; rpass.set_blend_constant(wgpu::Color { r: w, g: w, b: w, a: w }); - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); + self.draw_layer(&mut rpass, layer, true); } } let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { @@ -3221,9 +3460,79 @@ impl Compositor { timestamp_writes: None, occlusion_query_set: None, }); - rpass.set_pipeline(&self.pipeline_copy); - rpass.set_bind_group(0, abind, &[]); - rpass.draw(0..3, 0..1); + // Rien n'a ete peint dans `accum` hors des quads de la trainee : la recopie s'y borne + // au lieu de repasser sur toute la sortie, ce qui coutait 0,5 ms par frame 1080p + // (Radeon 610M) pour un curseur de quelques dizaines de pixels. + let (w, h) = (self.render_w as f32, self.render_h as f32); + let x0 = (c.bounds[0] * w).floor().clamp(0.0, w) as u32; + let y0 = (c.bounds[1] * h).floor().clamp(0.0, h) as u32; + let x1 = (c.bounds[2] * w).ceil().clamp(0.0, w) as u32; + let y1 = (c.bounds[3] * h).ceil().clamp(0.0, h) as u32; + if x1 > x0 && y1 > y0 { + rpass.set_scissor_rect(x0, y0, x1 - x0, y1 - y0); + rpass.set_pipeline(&self.pipeline_copy); + rpass.set_bind_group(0, abind, &[]); + rpass.draw(0..3, 0..1); + } + } + // La camera, au-dessus de l'ecran et du curseur. + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("webcam-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline); + if let Some(layer) = &webcam_shadow { + self.draw_layer(&mut rpass, layer, false); + } + // Fond personnalise : ENTRE l'ombre et la camera. C'est ce sandwich qui + // remplace la branche « mode 3 » du shader — la camera, decoupee, se + // fond dessus par alpha ; l'ombre reste dessous, elle appartient a la + // bulle et non a son contenu. + if let Some(bg) = &webcam_bg { + self.draw_layer(&mut rpass, &bg.layer, false); + } + if let Some(layer) = &webcam_draw { + self.draw_layer(&mut rpass, layer, false); + } + for draw in &camera_layer_draws { + self.draw_layer(&mut rpass, &draw.layer, false); + } + } + // Passe 3 : les autres annotations, par-dessus tout le reste (les flous sont passes avant + // le curseur). Elle existe meme sans annotation : deux passes consecutives sur la MEME + // cible avec `LoadOp::Load` ne coutent rien de plus qu'une seule sur un GPU desktop, et un + // seul chemin de code vaut mieux qu'un branchement qui ne serait exerce que dans un projet + // sur dix. + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("ann-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline); + for a in &ann_draws { + self.draw_layer(&mut rpass, &a.layer, false); + } } self.gpu.context.submit(std::iter::once(encoder.finish())); Ok(()) @@ -4041,7 +4350,7 @@ mod tests { planes: (&wgpu::TextureView, &wgpu::TextureView, &wgpu::TextureView), ) -> (u32, u32, Vec) { let dummy = comp.dummy_view(); - let (_buf, bind) = comp.make_bind(cb, Some(planes), &dummy); + let layer = comp.make_bind(cb, Some(planes), &dummy); let mut encoder = comp.gpu.device.create_command_encoder( &wgpu::CommandEncoderDescriptor { label: Some("test-layer") }, ); @@ -4061,8 +4370,7 @@ mod tests { occlusion_query_set: None, }); rpass.set_pipeline(&comp.pipeline); - rpass.set_bind_group(0, &bind, &[]); - rpass.draw(0..4, 0..1); + comp.draw_layer(&mut rpass, &layer, false); } comp.gpu.context.submit(std::iter::once(encoder.finish())); unsafe { comp.readback_direct().expect("readback_direct") } @@ -4559,12 +4867,37 @@ mod tests { /// `compose_pip` sur une scene deja construite, pour les tests qui la retouchent. fn compose_pip_scene(comp: &Compositor, gpu: &Gpu, scene: Scene, shadow: bool) -> Vec { + let screen = FakeFrame::new(gpu, 128, 128, |_, _| 126); + let webcam = FakeFrame::new(gpu, 64, 64, |_, _| Y_WHITE); + compose_pip_frames(comp, scene, shadow, &screen, &webcam) + } + + /// `compose_pip_scene` with the caller's own screen and camera frames. + fn compose_pip_frames( + comp: &Compositor, + scene: Scene, + shadow: bool, + screen: &FakeFrame, + webcam: &FakeFrame, + ) -> Vec { + compose_pip_frames_with_extras(comp, scene, shadow, screen, webcam, &[]) + } + + /// `compose_pip_frames` with extra camera frames. They are handed over AFTER `set_scene`, + /// which forgets the previous ones, and before the compose, as the product code does. + fn compose_pip_frames_with_extras( + comp: &Compositor, + scene: Scene, + shadow: bool, + screen: &FakeFrame, + webcam: &FakeFrame, + extra: &[*const AVFrame], + ) -> Vec { comp.set_live_params(live_params_from_scene(&scene)); comp.set_has_webcam(true); comp.set_scene(Some(scene)); + unsafe { comp.set_extra_camera_frames(extra) }; - let screen = FakeFrame::new(gpu, 128, 128, |_, _| 126); - let webcam = FakeFrame::new(gpu, 64, 64, |_, _| Y_WHITE); let mut cfg = Cfg::c8(); cfg.bg_blur = 0.0; cfg.zoom = false; @@ -4679,6 +5012,9 @@ mod tests { clip_index: None, start_sec: 2.0, end_sec: 8.0, + rotation: 0, + mirror: None, + full_frame: false, }); // 5 s : la montee (~1 s) est finie, le retour (~1,5 s) n'a pas commence. comp.set_timeline_time(Some(5.0)); @@ -4690,6 +5026,241 @@ mod tests { ); } + /// `pip_scene_json` on a black background: with a black screen too, the camera is drawn over + /// black wherever it lands, so its colour can be read without knowing where the PiP is. + fn black_pip_scene() -> Scene { + let json = pip_scene_json(NO_EFFECT).replace("#0080ff", "#000000"); + Scene::from_json(&json).expect("scene json") + } + + /// The x of every pixel of a `width`-wide RGBA image that `pred` accepts. + fn xs_where(rgba: &[u8], width: usize, pred: impl Fn(&[u8]) -> bool) -> Vec { + rgba.chunks_exact(4) + .enumerate() + .filter(|(_, px)| pred(px)) + .map(|(i, _)| i % width) + .collect() + } + + /// The RGBA of pixel (x, y) of a 320-wide readback. + fn pixel_at(rgba: &[u8], x: usize, y: usize) -> [u8; 4] { + let i = (y * 320 + x) * 4; + [rgba[i], rgba[i + 1], rgba[i + 2], rgba[i + 3]] + } + + /// A 64x64 camera frame of one RGB colour (BT.709 limited, `rgb_to_nv12`). + fn solid_camera(gpu: &Gpu, rgb: [u8; 3]) -> FakeFrame { + let (w, h) = (64u32, 64u32); + let pixels: Vec = (0..w * h).flat_map(|_| rgb).collect(); + let (y, uv) = rgb_to_nv12(&pixels, w, h); + FakeFrame::from_planes(gpu, w, h, &y, &uv) + } + + /// Camera 1 over the whole frame, and camera 0 as a PiP in the bottom-right corner + /// (x 224..304, y 126..171 of a 320x180 frame). + const CAMERA_1_FULL: &str = r#"{"camera":1,"rect":{"x":0,"y":0,"width":1,"height":1},"shape":"rectangle","fillsFrame":true}"#; + const CAMERA_0_PIP: &str = r#"{"camera":0,"rect":{"x":0.7,"y":0.7,"width":0.25,"height":0.25},"shape":"rectangle"}"#; + + /// A camera layout region from 2 s to 8 s with these layers (JSON objects, comma-separated). + fn layout_region(layers: &str) -> crate::scene::SceneCameraLayoutRegion { + serde_json::from_str(&format!(r#"{{"startSec":2.0,"endSec":8.0,"layers":[{layers}]}}"#)) + .expect("region json") + } + + /// The time in the lead-in of `layout_region` where a layer only the region shows is drawn + /// at opacity 0.5 (and one only the default shows, at 0.5 too). Checked against the plan. + fn half_way_in(region: &crate::scene::SceneCameraLayoutRegion) -> f32 { + let (mut lo, mut hi) = (0.0f32, 1.0f32); + for _ in 0..40 { + let mid = 0.5 * (lo + hi); + if crate::regions::ease_out_screen_studio(mid) < 0.5 { + lo = mid; + } else { + hi = mid; + } + } + let t = 2.0 + lo * crate::regions::TRANSITION_WINDOW_S; + let default_cam0 = crate::camera_layers::CameraLayerPlan { + camera: 0, + dst: [0.7, 0.7, 0.25, 0.25], + radius_frac: 0.0, + shape: 0, + opacity: 1.0, + fills_frame: false, + }; + let clock = crate::regions::ScreenClock::new(&[], 0); + let layers = crate::camera_layers::camera_layers_at( + std::slice::from_ref(region), + &[], + t, + &clock, + Some(default_cam0), + ); + assert!( + layers.iter().any(|l| (l.opacity - 0.5).abs() < 1e-3), + "no layer at half opacity at {t}: {layers:?}" + ); + t + } + + /// `compose_pip_frames` at source time `t`, with these extra camera frames set (and cleared + /// again afterwards). + fn compose_layers( + comp: &Compositor, + scene: Scene, + t: f32, + screen: &FakeFrame, + camera_0: &FakeFrame, + extra: &[*const AVFrame], + ) -> Vec { + comp.set_timeline_time(Some(t)); + let rgba = compose_pip_frames_with_extras(comp, scene, false, screen, camera_0, extra); + unsafe { comp.set_extra_camera_frames(&[]) }; + rgba + } + + fn is_green(px: [u8; 4]) -> bool { + px[1] > 200 && px[0] < 60 && px[2] < 60 + } + + /// A `camera-full-pip` region: camera 1 (green) fills the frame, camera 0 (white) is the PiP + /// over it. The centre shows camera 1, the PiP rect camera 0. + #[test] + fn two_cameras_draw_in_their_planned_rects() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let screen = FakeFrame::new(&gpu, 128, 128, |_, _| Y_BLACK); + let white = FakeFrame::new(&gpu, 64, 64, |_, _| Y_WHITE); + let green = solid_camera(&gpu, [0, 255, 0]); + let mut scene = black_pip_scene(); + scene.camera_layout_regions.push(layout_region(&format!("{CAMERA_1_FULL},{CAMERA_0_PIP}"))); + // 5 s: past the lead-in, before the lead-out. + let rgba = compose_layers(&comp, scene, 5.0, &screen, &white, &[green.as_ptr()]); + let centre = pixel_at(&rgba, 160, 90); + assert!(is_green(centre), "the centre is not camera 1: {centre:?}"); + let pip = pixel_at(&rgba, 264, 148); + assert!(pip[..3].iter().all(|&c| c > 240), "the PiP rect is not camera 0: {pip:?}"); + let corner = pixel_at(&rgba, 10, 10); + assert!(is_green(corner), "camera 1 does not fill the frame: {corner:?}"); + } + + /// Without a frame for camera 1 its layer is skipped, empty slice or null pointer alike: + /// the centre shows the screen, as it does without any region. + #[test] + fn a_missing_extra_camera_skips_its_layer() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let screen = FakeFrame::new(&gpu, 128, 128, |_, _| 126); + let white = FakeFrame::new(&gpu, 64, 64, |_, _| Y_WHITE); + let control = compose_layers(&comp, black_pip_scene(), 5.0, &screen, &white, &[]); + let screen_px = pixel_at(&control, 160, 90); + assert!(screen_px[1] > 100 && screen_px[1] < 160, "control: the centre is not the screen"); + + let mut scene = black_pip_scene(); + scene.camera_layout_regions.push(layout_region(&format!("{CAMERA_1_FULL},{CAMERA_0_PIP}"))); + for extra in [&[][..], &[std::ptr::null()][..]] { + let rgba = compose_layers(&comp, scene.clone(), 5.0, &screen, &white, extra); + assert_eq!(pixel_at(&rgba, 160, 90), screen_px, "{} frames set", extra.len()); + let pip = pixel_at(&rgba, 264, 148); + assert!(pip[..3].iter().all(|&c| c > 240), "camera 0 is no longer drawn: {pip:?}"); + } + } + + /// Half-way into the lead-in camera 1 fades in at opacity 0.5: over the black screen the + /// centre is half as green as in the hold. + #[test] + fn a_fading_extra_camera_is_half_transparent() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let screen = FakeFrame::new(&gpu, 128, 128, |_, _| Y_BLACK); + let white = FakeFrame::new(&gpu, 64, 64, |_, _| Y_WHITE); + let green = solid_camera(&gpu, [0, 255, 0]); + let region = layout_region(&format!("{CAMERA_1_FULL},{CAMERA_0_PIP}")); + let t = half_way_in(®ion); + let mut scene = black_pip_scene(); + scene.camera_layout_regions.push(region); + + let hold = compose_layers(&comp, scene.clone(), 5.0, &screen, &white, &[green.as_ptr()]); + let full = pixel_at(&hold, 160, 90); + assert!(is_green(full), "control: the centre is not camera 1 in the hold: {full:?}"); + let half = compose_layers(&comp, scene, t, &screen, &white, &[green.as_ptr()]); + let px = pixel_at(&half, 160, 90); + let expected = full[1] as i32 / 2; + assert!( + (px[1] as i32 - expected).abs() <= 8 && px[0] < 30 && px[2] < 30, + "half-way in: {px:?}, expected green near {expected}" + ); + } + + /// Camera 0 fading out at opacity 0.5 (a region that shows only camera 1, which has no + /// frame) draws the (white) camera at half strength over the (black) screen: every camera + /// pixel is mid grey, none of them is white any more. + #[test] + fn transparency_half_halves_the_camera_over_the_screen() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let screen = FakeFrame::new(&gpu, 128, 128, |_, _| Y_BLACK); + let webcam = FakeFrame::new(&gpu, 64, 64, |_, _| Y_WHITE); + let opaque = compose_pip_frames(&comp, black_pip_scene(), false, &screen, &webcam); + let whole = camera_pixels(&opaque); + assert!(whole > 200, "the camera is not on screen, the test proves nothing"); + + let region = layout_region(CAMERA_1_FULL); + let t = half_way_in(®ion); + let mut scene = black_pip_scene(); + scene.camera_layout_regions.push(region); + let half = compose_layers(&comp, scene, t, &screen, &webcam, &[]); + assert_eq!(camera_pixels(&half), 0, "a half transparent camera still has white pixels"); + let grey = half + .chunks_exact(4) + .filter(|px| px[..3].iter().all(|&c| (126..=130).contains(&c))) + .count(); + assert!( + grey as f32 >= whole as f32 * 0.9, + "{grey} mid-grey pixels for {whole} camera pixels drawn opaque" + ); + } + + /// Camera 0's perspective applies without any layout region. A homography that flips the + /// camera horizontally draws its right half on the left of the layer: a camera red on the + /// left and blue on the right shows blue left of red. The control (no perspective) shows it + /// the other way round, so the flip is the homography's doing. + #[test] + fn camera_0_perspective_applies_without_layout_regions() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let screen = FakeFrame::new(&gpu, 128, 128, |_, _| Y_BLACK); + // BT.709 limited: red = (Y 63, Cb 102, Cr 240), blue = (Y 32, Cb 240, Cr 118). + let (w, h) = (64u32, 64u32); + let y: Vec = (0..w * h).map(|i| if i % w < w / 2 { 63 } else { 32 }).collect(); + let uv: Vec = (0..h / 2) + .flat_map(|_| (0..w / 2).flat_map(|j| if j < w / 4 { [102, 240] } else { [240, 118] })) + .collect(); + let webcam = FakeFrame::from_planes(&gpu, w, h, &y, &uv); + let red = |px: &[u8]| px[0] > 200 && px[1] < 60 && px[2] < 60; + let blue = |px: &[u8]| px[2] > 200 && px[0] < 60 && px[1] < 60; + + let plain = compose_pip_frames(&comp, black_pip_scene(), false, &screen, &webcam); + let (r, b) = (xs_where(&plain, 320, red), xs_where(&plain, 320, blue)); + assert!(r.len() > 50 && b.len() > 50, "control: {} red, {} blue pixels", r.len(), b.len()); + assert!(r.iter().max() < b.iter().min(), "control: the camera's red half is on the left"); + + let mut scene = black_pip_scene(); + assert!(scene.camera_layout_regions.is_empty()); + scene.cameras.push(crate::scene::SceneCamera { + index: 0, + rotation: 0, + mirror: None, + crop: None, + homography: Some([-1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]), + aspect: None, + }); + let flipped = compose_pip_frames(&comp, scene, false, &screen, &webcam); + let (r, b) = (xs_where(&flipped, 320, red), xs_where(&flipped, 320, blue)); + assert!(r.len() > 50 && b.len() > 50, "flipped: {} red, {} blue pixels", r.len(), b.len()); + assert!(b.iter().max() < r.iter().min(), "flipped: the left half of the layer is blue"); + } + /// Le tour complet, celui qui a besoin d'ONNX Runtime : capture -> inference /// -> masque -> composite, entraine par `compose_frame` seul. Se saute /// proprement sans la bibliotheque, ce que fait la CI — cf. @@ -4915,12 +5486,23 @@ mod tests { /// Ecran gris sur fond gris moyen, avec ou sans cadre de fenetre (`frame` est insere tel /// quel dans `effects`), droit ou incline (`rotation`, JSON). fn compose_framed(comp: &Compositor, gpu: &Gpu, frame: &str, rotation: &str) -> Vec { + compose_framed_shadow(comp, gpu, frame, rotation, 0.6) + } + + /// `compose_framed`, avec l'ombre de l'ecran `shadow` (0 l'eteint). + fn compose_framed_shadow( + comp: &Compositor, + gpu: &Gpu, + frame: &str, + rotation: &str, + shadow: f32, + ) -> Vec { let json = format!( r##"{{"clips":[{{"screenPath":"/s.mp4","webcamPath":"","sourceStartSec":0,"sourceEndSec":10,"webcamOffsetSec":0,"hasAudio":false}}], "layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rounded", "webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false, "screenRect":{{"x":0.1,"y":0.1,"width":0.8,"height":0.8}}}}, - "effects":{{"padding":0.2,"blur":false,"shadow":0.6,"roundnessFrac":0.03,"motionBlur":0{frame}}}, + "effects":{{"padding":0.2,"blur":false,"shadow":{shadow},"roundnessFrac":0.03,"motionBlur":0{frame}}}, "background":{{"kind":"color","color":"#6070a0"}}, "zoomRegions":[{{"clipIndex":0,"startSec":0,"endSec":10,"scale":1,"focusX":0.5,"focusY":0.5,"rotation":{rotation}}}], "annotations":[], @@ -5203,6 +5785,66 @@ mod tests { assert!(right[2] > 200 && right[0] < 60 && right[1] < 60, "droite {right:?} : bleu attendu"); } + /// Le fond fixe est recopie d'une frame a l'autre (`bg_cache`) au lieu d'etre redessine, et + /// un fond qui change ne laisse rien du precedent : un compositeur qui a rendu A, A, B puis A + /// rend chacun a l'octet comme un compositeur neuf. Un fond anime n'est jamais repris. + #[test] + fn a_static_background_is_reused_and_a_changed_one_redrawn() { + let Some(gpu) = gpu() else { return }; + let screen = FakeFrame::new(&gpu, 640, 360, |_, _| 60); + let webcam = FakeFrame::new(&gpu, 64, 64, |_, _| 60); + let render = |comp: &Compositor, background: &str| { + let json = format!( + r##"{{"clips":[{{"screenPath":"/s.mp4","webcamPath":"","sourceStartSec":0,"sourceEndSec":10,"webcamOffsetSec":0,"hasAudio":false}}], + "layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rounded", + "webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false, + "screenRect":{{"x":0.3,"y":0.3,"width":0.4,"height":0.4}}}}, + "effects":{{"padding":0.4,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}}, + "background":{background}, + "zoomRegions":[],"annotations":[], + "cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0, + "clipToBounds":false,"theme":"default"}}, + "cropByClip":[null], + "output":{{"width":640,"height":360,"fps":30}}}}"## + ); + let scene = Scene::from_json(&json).expect("scene json"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_has_webcam(false); + comp.set_scene(Some(scene)); + let mut cfg = Cfg::c8(); + cfg.bg_blur = 0.0; + cfg.zoom = false; + cfg.layout_anim = false; + cfg.cursor = false; + cfg.shadow = false; + cfg.mblur_n = 1; + unsafe { + comp.compose_frame(screen.as_ptr(), webcam.as_ptr(), 0.0, &cfg) + .expect("compose_frame"); + comp.readback_direct().expect("readback_direct").2 + } + }; + let fresh = || Compositor::new_sized(&gpu, 640, 360).expect("Compositor::new_sized"); + let a = r##"{"kind":"gradient","angleDeg":90,"stops":["rgb(255, 0, 0)","rgb(0, 0, 255)"],"offsets":[0,1]}"##; + let b = r##"{"kind":"color","color":"#00ff00"}"##; + + let comp = fresh(); + let a1 = render(&comp, a); + assert!(comp.bg_cache.borrow().is_some(), "un fond fixe doit etre garde"); + let a2 = render(&comp, a); + let b1 = render(&comp, b); + let a3 = render(&comp, a); + assert!(a1 == a2, "le fond recopie doit etre celui qui a ete dessine"); + assert!(b1 == render(&fresh(), b), "un autre fond ne doit rien garder du precedent"); + assert!(a3 == a1, "revenir au premier fond doit le redessiner a l'identique"); + assert!(a1 != b1, "garde : les deux fonds different"); + + let aurora = r##"{"kind":"gradient","angleDeg":90,"stops":["rgb(255, 0, 0)","rgb(0, 0, 255)"],"offsets":[0,1],"motion":"aurora"}"##; + let comp = fresh(); + render(&comp, aurora); + assert!(comp.bg_cache.borrow().is_none(), "un fond anime ne doit pas etre garde"); + } + /// Le cadre de fenetre (mode 14) se dessine sur Linux comme ailleurs : `"none"` rend /// l'image d'avant a l'octet, le theme clair ajoute une barre claire, droite comme /// inclinee, et les deux themes different. `OPENSCREEN_FRAME_OUT` recoit les PNG. @@ -5313,6 +5955,22 @@ mod tests { } } + /// L'ombre d'un appareil est un modele elle aussi (mode 17, `device_shadow_cb`) : comme + /// l'appareil, elle n'existe que dans `pipeline_models`, et le pipeline plat n'en peindrait + /// rien. Elle doit donc changer l'image, a plat comme incline. + #[test] + fn a_device_frame_casts_its_shadow() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("Compositor::new_sized"); + let laptop = r#","frame":"laptop""#; + for (name, rotation) in [("flat", "null"), ("iso", r#""iso""#)] { + let lit = compose_framed_shadow(&comp, &gpu, laptop, rotation, 0.6); + let bare = compose_framed_shadow(&comp, &gpu, laptop, rotation, 0.0); + let shaded = lit.chunks_exact(4).zip(bare.chunks_exact(4)).filter(|(a, b)| a != b).count(); + assert!(shaded > 2_000, "{name} : l'ombre de l'appareil ne change que {shaded} px"); + } + } + // ----------------------------------------------------------------------- // Curseur modélisé (mode 15) : pendant de `tests/cursor_model_render.rs` (Windows). // ----------------------------------------------------------------------- @@ -5523,6 +6181,22 @@ mod tests { .collect() } + /// Les pixels que couvre un curseur de VERRE (le cristal de Prism Glow, qui laisse voir le + /// contenu et le réfracte) : opaques, ou changés sur les deux teintes sans y être plus sombres + /// (une ombre ne fait qu'assombrir ; le verre réfracte, reflète et luit). + fn model_covered(on_blue: &[u8], on_orange: &[u8], bare: &[u8], bare_orange: &[u8]) -> Vec { + let opaque = model_opaque(on_blue, on_orange, bare); + (0..1280 * 720) + .map(|i| { + let lit = |on: &[u8], under: &[u8]| { + let (a, b) = (&on[i * 4..i * 4 + 3], &under[i * 4..i * 4 + 3]); + a != b && model_luma(a) >= model_luma(b) - 2.0 + }; + opaque[i] || (lit(on_blue, bare) && lit(on_orange, bare_orange)) + }) + .collect() + } + fn model_iou(a: &[bool], b: &[bool]) -> f32 { let inter = a.iter().zip(b).filter(|(x, y)| **x && **y).count(); let union = a.iter().zip(b).filter(|(x, y)| **x || **y).count(); @@ -5592,6 +6266,34 @@ mod tests { } } + /// La trainee du curseur modelise s'accumule par `pipeline_add_models` : le pipeline plat ne + /// compile pas le mode 15 et n'en peindrait rien. En mouvement, flou du curseur au maximum, + /// le modele doit donc bien apparaitre. + #[test] + fn the_modelled_cursor_draws_its_trail() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("Compositor::new_sized"); + let (y, uv) = model_screen_planes(false); + let screen = FakeFrame::from_planes(&gpu, 640, 360, &y, &uv); + // Traverse 60 % de la largeur en une seconde autour de t = 2 s : ~13 px par frame, donc + // plusieurs copies dans `accum`. + let moving = crate::cursor::CursorTrack::new( + vec![(1.5, 0.2, 0.45), (2.5, 0.8, 0.45)], + vec![], + vec![(0.0, "arrow".to_string())], + ); + let json = |show: bool| { + model_scene_json("null", Some(true), "default", show, 3.0) + .replace(r#""motionBlur":0,"clickBounce""#, r#""motionBlur":1,"clickBounce""#) + }; + assert!(json(true).contains(r#""motionBlur":1,"clickBounce""#), "garde : le flou du curseur"); + let bare = compose_model(&comp, &screen, &json(false), &moving); + let trail = compose_model(&comp, &screen, &json(true), &moving); + let drawn = bare.chunks_exact(4).zip(trail.chunks_exact(4)).filter(|(a, b)| a != b).count(); + // Taille 3 : ~1000 px de silhouette a l'arret, etales ici par la trainee. + assert!(drawn > 1000, "trainee du curseur modelise : {drawn} px dessines"); + } + /// Chaque état garde son art et sa silhouette : posé au centre de l'écran (vu de face), le /// modèle couvre ce que couvre le sprite plat, avec ses couleurs ; il porte une ombre en /// l'air et suit l'inclinaison du plan. Le réglage éteint garde le sprite plat. @@ -5681,7 +6383,8 @@ mod tests { /// Pendant de `the_sculpted_cursors_stand_at_the_hotspot` (Windows) : la flèche et la main /// sculptées des thèmes d'origine (`sculpt.rs`) passent par le WGSL, tiennent au hotspot, en - /// bas à droite de lui, et portent leur ombre en l'air. + /// bas à droite de lui, et portent leur ombre en l'air. Le cristal de Prism Glow est de verre : + /// son corps compte ce qu'il couvre, et une bonne part de lui change avec la teinte de l'écran. #[test] fn the_sculpted_cursors_stand_at_the_hotspot() { let Some(gpu) = gpu() else { return }; @@ -5693,6 +6396,7 @@ mod tests { let extruded = model_scene_json("null", Some(true), "default", true, 5.0); let hidden = model_scene_json("null", Some(true), "default", false, 5.0); let bare = compose_model(&comp, &blue, &hidden, &model_track("arrow", false, 0.5)); + let bare_orange = compose_model(&comp, &orange, &hidden, &model_track("arrow", false, 0.5)); let mut failures = Vec::new(); for state in ["arrow", "pointer"] { let still = model_track(state, false, 0.5); @@ -5707,7 +6411,12 @@ mod tests { failures.push(format!("{theme}/{state}: le sprite extrude au lieu du modele")); } let (tip, u) = model_tip(&json, &still); - let mask = model_opaque(&hover, &hover_b, &bare); + let glass = theme == "prism-glow"; + let mask = if glass { + model_covered(&hover, &hover_b, &bare, &bare_orange) + } else { + model_opaque(&hover, &hover_b, &bare) + }; let (mut body, mut c, mut near) = (0usize, [0.0f32; 2], f32::MAX); for (i, _) in mask.iter().enumerate().filter(|(_, m)| **m) { let (x, y) = ((i % 1280) as f32, (i / 1280) as f32); @@ -5735,6 +6444,124 @@ mod tests { if shadow * 10 < body * 3 { failures.push(format!("{theme}/{state}: pas d'ombre en l'air ({shadow} px)")); } + if glass { + let through = (0..1280 * 720).filter(|&i| mask[i] && hover[i * 4..i * 4 + 3] != hover_b[i * 4..i * 4 + 3]).count(); + println!("{theme}/{state} : {through} px du corps laissent voir l'ecran"); + if through * 10 < body * 3 { + failures.push(format!("{theme}/{state}: le cristal ne laisse voir l'ecran que sur {through} px")); + } + } + } + } + assert!(failures.is_empty(), "{failures:#?}"); + } + + /// Le métrage, puis ses flous de confidentialité, puis le curseur, net par-dessus (parité + /// Windows et macOS) : le cristal de Prism Glow réfracte l'image déjà floutée. Sous le flou, + /// des rayures fines décalées d'une colonne donnent la même image floutée ; si le verre lisait + /// la vidéo brute, il montrerait les rayures, et les deux rendus différeraient à travers lui. + #[test] + fn the_crystal_refracts_the_blurred_picture() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("Compositor::new_sized"); + // Moitié gauche grise, moitié droite (la zone floutée) rayée une colonne sur deux. + let (w, h) = (640u32, 360u32); + let stripes = |phase: u32| { + let y: Vec = (0..w * h) + .map(|i| if i % w < w / 2 { 150 } else if (i % w + phase) % 2 == 0 { 40 } else { 220 }) + .collect(); + FakeFrame::from_planes(&gpu, w, h, &y, &vec![128; (w * (h / 2)) as usize]) + }; + let (a, b) = (stripes(0), stripes(1)); + let blur = r#""annotations":[{"id":"b","startSec":0,"endSec":10,"kind":"blur","x":0.5,"y":0.0,"w":0.5,"h":1.0,"blur":{"style":"blur","shape":"rectangle","color":"white","intensity":24,"blockSize":16}}]"#; + let json = |show: bool| { + model_scene_json("null", Some(true), "default", show, 5.0) + .replace(r#"/arrow.png","#, r#"/arrow.png","sculpt":"prism-glow/arrow","#) + .replace(r#""annotations":[]"#, blur) + }; + let mut failures = Vec::new(); + // Tout entier dans la zone, puis à cheval sur son bord. + for x in [0.7f32, 0.48] { + let track = crate::cursor::CursorTrack::new(vec![(0.0, x, 0.45), (9.0, x, 0.45)], vec![], vec![(0.0, "arrow".to_string())]); + let (ra, rb) = (compose_model(&comp, &a, &json(true), &track), compose_model(&comp, &b, &json(true), &track)); + let bare = compose_model(&comp, &a, &json(false), &track); + model_save(&format!("glass-over-blur-{x}"), &ra); + let px = |img: &[u8], i: usize, c: usize| img[i * 4 + c] as i32; + let hidden = (0..1280 * 720).filter(|&i| (0..3).any(|c| (px(&ra, i, c) - px(&rb, i, c)).abs() > 24)).count(); + // Le serti net du curseur : des sauts francs, entre voisins, de ce qu'il change. + let d = |i: usize, c: usize| px(&ra, i, c) - px(&bare, i, c); + let sharp = (0..1280 * 720 - 1) + .filter(|&i| i % 1280 < 1279 && (0..3).any(|c| (d(i, c) - d(i + 1, c)).abs() > 60)) + .count(); + println!("cristal en x = {x} : {sharp} bords francs, {hidden} px qui suivent les rayures cachées"); + if sharp < 50 { + failures.push(format!("x = {x} : {sharp} bords francs, le curseur n'est pas net par-dessus le flou")); + } + if hidden > 0 { + failures.push(format!("x = {x} : {hidden} px montrent les rayures cachées sous le flou")); + } + } + assert!(failures.is_empty(), "{failures:#?}"); + } + + /// Les thèmes cerclés gardent le trait de leur dessin (`design/cursors/`) : sur la + /// fleche comme sur la main, le modele est pour une bonne part de la couleur du trait (le + /// plateau, le jonc, les rainures entre les doigts), autour de la couleur du corps, et montre + /// ce qu'il porte devant : l'etoile jaune de Star Sprout, le calque et les tirets jaunes de la + /// fleche de Pop Coral, les tirets corail de sa main. Les parts comptent les pixels francs : + /// ceux que l'antialiasing mêle aux bords n'entrent dans aucune. + #[test] + fn the_rimmed_models_keep_the_outline_of_their_art() { + type Rgb = [i32; 3]; + let navy = |[r, g, b]: Rgb| b > r + 20 && r < 110 && g < 130; + let black = |[r, g, b]: Rgb| r.max(g).max(b) < 70; + let mint = |[r, g, b]: Rgb| g > 200 && g > r + 15 && b > 150; + let ivory = |[r, g, b]: Rgb| r > 200 && g > 190 && b > 160 && r - b < 70; + let coral = |[r, g, b]: Rgb| r > 200 && g < 150 && b < 140; + let yellow = |[r, g, b]: Rgb| r > 200 && g > 150 && b < 120; + // (thème, état, trait, corps, ornement, parts minimales du trait, du corps, de l'ornement) + let cases: [(&str, &str, &dyn Fn(Rgb) -> bool, &dyn Fn(Rgb) -> bool, &dyn Fn(Rgb) -> bool, [f32; 3]); 6] = [ + ("studio-ink", "arrow", &black, &ivory, &|_| false, [0.3, 0.15, 0.0]), + ("studio-ink", "pointer", &black, &ivory, &|_| false, [0.2, 0.3, 0.0]), + ("pop-coral", "arrow", &navy, &coral, &yellow, [0.2, 0.2, 0.05]), + ("pop-coral", "pointer", &navy, &yellow, &coral, [0.18, 0.3, 0.02]), + ("star-sprout", "arrow", &navy, &mint, &yellow, [0.25, 0.2, 0.03]), + ("star-sprout", "pointer", &navy, &ivory, &yellow, [0.25, 0.2, 0.03]), + ]; + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("Compositor::new_sized"); + let (y, uv) = model_screen_planes(false); + let blue = FakeFrame::from_planes(&gpu, 640, 360, &y, &uv); + let (y, uv) = model_screen_planes(true); + let orange = FakeFrame::from_planes(&gpu, 640, 360, &y, &uv); + let extruded = model_scene_json("null", Some(true), "default", true, 5.0); + let hidden = model_scene_json("null", Some(true), "default", false, 5.0); + let bare = compose_model(&comp, &blue, &hidden, &model_track("arrow", false, 0.5)); + let mut failures = Vec::new(); + for (theme, state, rim, body, extra, [min_rim, min_body, min_extra]) in cases { + let still = model_track(state, false, 0.5); + let json = extruded + .replace(&format!(r#"/{state}.png","#), &format!(r#"/{state}.png","sculpt":"{theme}/{state}","#)); + let (hover, hover_b) = (compose_model(&comp, &blue, &json, &still), compose_model(&comp, &orange, &json, &still)); + let mask = model_opaque(&hover, &hover_b, &bare); + let (mut total, mut counts) = (0usize, [0usize; 3]); + for (i, _) in mask.iter().enumerate().filter(|(_, m)| **m) { + let p = [hover[i * 4] as i32, hover[i * 4 + 1] as i32, hover[i * 4 + 2] as i32]; + total += 1; + for (k, class) in [rim, body, extra].iter().enumerate() { + counts[k] += usize::from(class(p)); + } + } + let [rim_share, body_share, extra_share] = counts.map(|k| k as f32 / total.max(1) as f32); + println!("{theme}/{state} : {total} px, trait {rim_share:.3}, corps {body_share:.3}, ornement {extra_share:.3}"); + if !(min_rim..0.7).contains(&rim_share) { + failures.push(format!("{theme}/{state}: {rim_share:.3} de trait, il a disparu ou tout mange")); + } + if body_share < min_body { + failures.push(format!("{theme}/{state}: {body_share:.3} de couleur du corps")); + } + if extra_share < min_extra { + failures.push(format!("{theme}/{state}: {extra_share:.3} d'ornement")); } } assert!(failures.is_empty(), "{failures:#?}"); diff --git a/crates/compositor/src/compositor_macos.rs b/crates/compositor/src/compositor_macos.rs index b00a288ee..a9a571e98 100644 --- a/crates/compositor/src/compositor_macos.rs +++ b/crates/compositor/src/compositor_macos.rs @@ -261,9 +261,14 @@ pub struct Compositor { scene: RefCell>, cursor: RefCell>, cursor_time: RefCell>, + /// Le métrage dans la dernière image composée, que l'éditeur lit avec elle (`live.rs`). + footage: std::cell::Cell>, timeline_time: RefCell>, /// Temps programme (secondes de sortie) — cf. `FrameGeometryInput::programme_time`. programme_time: RefCell>, + /// The frames of cameras 1..=3 (`set_extra_camera_frames`), as addresses (0 = none) so + /// the compositor's auto traits stay what they were without the raw pointers. + extra_camera_frames: std::cell::Cell<[usize; crate::camera_layers::MAX_EXTRA_CAMERAS]>, live_params: RefCell, metal_texture_cache: CVMetalTextureCache, /// Dernier command buffer soumis, gardé pour pouvoir l'attendre AU MOMENT où le CPU lit @@ -316,6 +321,17 @@ pub struct Compositor { pipeline_fs_tex: metal::RenderPipelineState, /// `vs_main` + `ps_main` en additif : les échantillons de traînée du curseur. pipeline_add: metal::RenderPipelineState, + /// `pipeline_main` et `pipeline_add` compilés AVEC les modèles 3D (`LAYER_MODELS`, cf. + /// l'en-tête de `shaders.metal`). Les deux autres ne les ont pas ; `draw_layer` bascule sur + /// ceux-ci le temps du draw d'un modèle (`LayerCB::needs_models`). + pipeline_main_models: metal::RenderPipelineState, + pipeline_add_models: metal::RenderPipelineState, + /// La passe en cours a été ouverte sur `pipeline_add` (cf. `begin_pass`) : dit à + /// `draw_layer` laquelle des deux variantes « modèles » prendre. + layer_add: std::cell::Cell, + /// Le dernier fond fixe composé et sa clé (`BackgroundKey`) : recopié dans le RT tant + /// que la clé tient, au lieu d'être redessiné. + bg_cache: RefCell>, /// Buffer d'accumulation ISOLÉ (transparent) pour la traînée. Accumuler directement sur /// le RT reviendrait à AJOUTER du blanc à ce qui est déjà dessous : sur un fond clair, /// le curseur disparaît. Même raisonnement que côté D3D11. @@ -537,6 +553,18 @@ enum Blend { Add, } +/// `shaders.metal` avec les modèles 3D : la source dont seuls les pipelines de ces calques sont +/// tirés (cf. son en-tête). +fn msl_source_with_models() -> String { + format!("#define LAYER_MODELS 1\n{}", include_str!("shaders.metal")) +} + +/// Le fond tel que la passe de fond (et son flou) l'a laissé dans le RT, et la clé qui le décrit. +struct BgCache { + key: crate::frame_geometry::BackgroundKey, + tex: metal::Texture, +} + /// Un pipeline state à une seule pièce jointe couleur. fn make_pipeline( device: &metal::Device, @@ -661,6 +689,25 @@ impl Compositor { metal::MTLPixelFormat::RGBA8Unorm, Blend::Add, )?; + let library_models = device + .new_library_with_source(&msl_source_with_models(), &metal::CompileOptions::new()) + .map_err(|e| anyhow!("MTLDevice::new_library_with_source (modèles) a échoué : {e}"))?; + let pipeline_main_models = make_pipeline( + device, + &library_models, + "vs_main", + "ps_main", + metal::MTLPixelFormat::RGBA8Unorm, + Blend::Over, + )?; + let pipeline_add_models = make_pipeline( + device, + &library_models, + "vs_main", + "ps_main", + metal::MTLPixelFormat::RGBA8Unorm, + Blend::Add, + )?; let pipeline_kdown = make_pipeline( device, &library, "vs_fs", "ps_kawase_down", metal::MTLPixelFormat::RGBA8Unorm, Blend::Replace, @@ -682,8 +729,10 @@ impl Compositor { scene: RefCell::new(None), cursor: RefCell::new(None), cursor_time: RefCell::new(None), + footage: std::cell::Cell::new(None), timeline_time: RefCell::new(None), programme_time: RefCell::new(None), + extra_camera_frames: std::cell::Cell::new([0; crate::camera_layers::MAX_EXTRA_CAMERAS]), live_params: RefCell::new(LiveParams::default()), metal_texture_cache: cache, last_cmd: RefCell::new(None), @@ -703,6 +752,10 @@ impl Compositor { pipeline_fs_uv, pipeline_fs_tex, pipeline_add, + pipeline_main_models, + pipeline_add_models, + layer_add: std::cell::Cell::new(false), + bg_cache: RefCell::new(None), accum, trail, blur_half, @@ -761,6 +814,7 @@ impl Compositor { blur_quarter, blur_eighth, ann_copy, + bg_cache: RefCell::new(None), ..self }) } @@ -786,6 +840,9 @@ impl Compositor { pub fn set_scene(&self, s: Option) { *self.scene.borrow_mut() = s; + // A new scene can drop the regions that drew an extra camera: forget its frames so a + // stale pointer is never drawn before the next `set_extra_camera_frames`. + self.extra_camera_frames.set([0; crate::camera_layers::MAX_EXTRA_CAMERAS]); } pub fn set_cursor(&self, track: crate::cursor::CursorTrack) { @@ -887,6 +944,26 @@ impl Compositor { /// IOSurface, pas par pointeur Rust. `flush()` est donc la vidange elle-même. pub fn clear_srv_cache(&self) { self.metal_texture_cache.flush(); + // The extra cameras' frames belong to the decoders just closed: forget them too, so + // a stale pointer is never read before the next `set_extra_camera_frames`. + self.extra_camera_frames.set([0; crate::camera_layers::MAX_EXTRA_CAMERAS]); + } + + /// Frames for cameras 1..=3 (index 0 = scene camera 1). A null or missing entry means that + /// camera is not drawn this frame. The compositor keeps the pointers and reads them in every + /// `compose_frame` until the next call, so they must stay valid until then. + pub unsafe fn set_extra_camera_frames(&self, frames: &[*const AVFrame]) { + let mut slots = [0usize; crate::camera_layers::MAX_EXTRA_CAMERAS]; + for (slot, frame) in slots.iter_mut().zip(frames) { + *slot = *frame as usize; + } + self.extra_camera_frames.set(slots); + } + + /// The frame set for scene camera `camera` (1..=3), null when there is none. + fn extra_camera_frame(&self, camera: usize) -> *const AVFrame { + let slots = self.extra_camera_frames.get(); + camera.checked_sub(1).and_then(|k| slots.get(k).copied()).unwrap_or(0) as *const AVFrame } /// Les verbes de dessin, côté Metal. Mêmes noms et mêmes paramètres que leurs @@ -909,7 +986,20 @@ impl Compositor { enc.set_fragment_texture(0, Some(y)); enc.set_fragment_texture(1, Some(uv)); } - enc.draw_primitives(metal::MTLPrimitiveType::TriangleStrip, 0, 4); + // Un modèle 3D passe, le temps de son draw, par la variante « modèles » du pipeline de la + // passe, la seule à le compiler ; le pipeline plat reprend ensuite la main. + if cb.needs_models() { + let (models, flat) = if self.layer_add.get() { + (&self.pipeline_add_models, &self.pipeline_add) + } else { + (&self.pipeline_main_models, &self.pipeline_main) + }; + enc.set_render_pipeline_state(models); + enc.draw_primitives(metal::MTLPrimitiveType::TriangleStrip, 0, 4); + enc.set_render_pipeline_state(flat); + } else { + enc.draw_primitives(metal::MTLPrimitiveType::TriangleStrip, 0, 4); + } } /// Quad de couleur pleine / gradient / ombre — tout ce qui n'échantillonne pas la vidéo. @@ -1087,13 +1177,15 @@ impl Compositor { programme_t: f32, ) -> Result<()> { let full = [0.0, 0.0, 1.0, 1.0]; - self.draw_image_in(enc, path, full, [0.0, 0.0], 0.0, output_aspect, motion, programme_t) + let (aspect, t) = (output_aspect, programme_t); + self.draw_image_in(enc, path, full, [0.0, 0.0], 0.0, aspect, motion, t, 0.0) } /// `draw_image_bg` pour un rect quelconque — la bulle webcam s'en sert avec ses coins /// arrondis. `output_aspect` est le ratio du RECT visé, pas celui de la sortie : le crop /// « cover » se calcule contre la zone qu'on remplit. `motion` anime l'image au temps - /// programme `programme_t` ; la bulle passe `WallpaperMotion::None`. + /// programme `programme_t` ; la bulle passe `WallpaperMotion::None`. `transparency` = + /// `layer_fx.x` (0 = opaque): a camera layer's background box fades with the layer. #[allow(clippy::too_many_arguments)] unsafe fn draw_image_in( &self, @@ -1105,6 +1197,7 @@ impl Compositor { output_aspect: f32, motion: WallpaperMotion, programme_t: f32, + transparency: f32, ) -> Result<()> { let (tex, iw, ih) = self.cached_image(path)?; let ai = iw as f32 / ih.max(1) as f32; @@ -1128,6 +1221,7 @@ impl Compositor { mode: 6.0, fx: [0.0, 0.0, anim[0], anim[1]], mb, + layer_fx: [transparency, 0.0, 0.0, 0.0], ..Default::default() }, ); @@ -1146,6 +1240,8 @@ impl Compositor { /// /// `quad_px` / `radius_px` sont ceux de la bulle : le fond doit épouser ses coins arrondis, /// sinon un rectangle déborde derrière la caméra. + /// + /// `transparency` = the camera layer's `layer_fx.x`: the box fades with the camera. unsafe fn draw_webcam_bg( &self, enc: &metal::RenderCommandEncoderRef, @@ -1153,14 +1249,17 @@ impl Compositor { dst: [f32; 4], quad_px: [f32; 2], radius_px: f32, + transparency: f32, ) { const BLACK: [f32; 4] = [0.0, 0.0, 0.0, 1.0]; + let layer_fx = [transparency, 0.0, 0.0, 0.0]; let solid = |color: [f32; 4]| LayerCB { dst, quad_px, radius_px, mode: 1.0, color, + layer_fx, ..Default::default() }; match bg { @@ -1178,6 +1277,7 @@ impl Compositor { quad_px, radius_px, fx: [a.sin(), -a.cos(), 0.0, 0.0], + layer_fx, ..crate::frame_geometry::gradient_layer(stops, offsets, BLACK) }, ); @@ -1188,7 +1288,9 @@ impl Compositor { let aspect = if quad_px[1] > 0.0 { quad_px[0] / quad_px[1] } else { 1.0 }; let still = WallpaperMotion::None; if let Err(e) = - self.draw_image_in(enc, path, dst, quad_px, radius_px, aspect, still, 0.0) + self.draw_image_in( + enc, path, dst, quad_px, radius_px, aspect, still, 0.0, transparency, + ) { eprintln!("[compositor] fond webcam \"{path}\" : {e:#}"); self.draw_solid(enc, &solid(BLACK)); @@ -1417,6 +1519,37 @@ impl Compositor { } + /// Le fond gardé (`bg_cache`) : copie le RT dans `tex`, ou `tex` dans le RT (`into_rt`). + unsafe fn blit_background(&self, cmd: &metal::CommandBufferRef, tex: &metal::Texture, into_rt: bool) { + let (src, dst) = if into_rt { (tex, &self.rt) } else { (&self.rt, tex) }; + let blit = cmd.new_blit_command_encoder(); + blit.copy_from_texture( + src, 0, 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + metal::MTLSize { width: self.render_w as u64, height: self.render_h as u64, depth: 1 }, + dst, 0, 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + ); + blit.end_encoding(); + } + + /// Fige l'image composée dans `ann_copy` (mip 0), qu'y lisent les flous (mips dérivés, + /// `mips`) et le cristal de Prism Glow. + unsafe fn snapshot_rt(&self, cmd: &metal::CommandBufferRef, mips: bool) { + let blit = cmd.new_blit_command_encoder(); + blit.copy_from_texture( + &self.rt, 0, 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + metal::MTLSize { width: self.render_w as u64, height: self.render_h as u64, depth: 1 }, + &self.ann_copy, 0, 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + ); + if mips { + blit.generate_mipmaps(&self.ann_copy); + } + blit.end_encoding(); + } + /// Annotations : calque le plus haut, ancré sur `s_ann` — le rect écran SANS ZOOM, le /// conteneur que reçoit l'overlay web. Port de `compositor_windows::draw_annotations`. /// @@ -1425,6 +1558,9 @@ impl Compositor { /// grossir les sous-titres sous un zoom (issue #179, puis #397 sur Linux). L'arithmétique /// elle-même vit dans `frame_geometry::annotation_dst_in`, partagée par les trois backends. /// Le flou, lui, se place par `g.privacy_mask` : un masque doit rester sur ce qu'il cache. + /// + /// `privacy` : les flous de confidentialité seuls, dessinés sur le métrage avant le curseur ; + /// sinon toutes les autres annotations, le calque le plus haut. unsafe fn draw_annotations( &self, cmd: &metal::CommandBufferRef, @@ -1432,30 +1568,21 @@ impl Compositor { t: f32, s_ann: [f32; 4], g: &crate::frame_geometry::FrameGeometry, + privacy: bool, ) -> Result<()> { let Some(scene) = scene else { return Ok(()) }; - if scene.annotations.is_empty() { - return Ok(()); - } let (rw, rh) = (self.render_w as f32, self.render_h as f32); let visible = |a: &crate::scene::SceneAnnotation| { - t >= a.start_sec as f32 && t < a.end_sec as f32 + t >= a.start_sec as f32 && t < a.end_sec as f32 && (a.kind == "blur") == privacy }; + if !scene.annotations.iter().any(visible) { + return Ok(()); + } // UNE seule recopie pour toutes les annotations flou de la frame : leur lecture doit // voir l'image composée SANS les flous eux-mêmes, sinon deux zones qui se recouvrent // s'échantillonneraient l'une l'autre selon l'ordre de dessin. - if scene.annotations.iter().any(|a| a.kind == "blur" && visible(a)) { - let blit = cmd.new_blit_command_encoder(); - blit.copy_from_texture( - &self.rt, 0, 0, - metal::MTLOrigin { x: 0, y: 0, z: 0 }, - metal::MTLSize { width: rw as u64, height: rh as u64, depth: 1 }, - &self.ann_copy, 0, 0, - metal::MTLOrigin { x: 0, y: 0, z: 0 }, - ); - // Seul le mip 0 est rempli ; le GPU dérive le reste. - blit.generate_mipmaps(&self.ann_copy); - blit.end_encoding(); + if privacy { + self.snapshot_rt(cmd, true); } let enc = self.begin_pass(cmd, &self.rt, None, &self.pipeline_main)?; @@ -1585,6 +1712,13 @@ impl Compositor { if text.content.trim().is_empty() { continue; } + // The desk label shows only while the camera is covered: skip the rest of + // its section before rasterizing anything. + if crate::text_anim::is_desk_cover(text.animation.as_deref()) + && g.webcam_cover <= 0.0 + { + continue; + } let spec = crate::text::TextSpec { content: text.content.clone(), color: parse_hex(&text.color).unwrap_or([1.0, 1.0, 1.0, 1.0]), @@ -1618,10 +1752,11 @@ impl Compositor { }) else { continue; }; - let anim = crate::text_anim::text_animation_state( + let anim = crate::text_anim::annotation_text_state( text.animation.as_deref(), (t - a.start_sec as f32) * 1000.0, ((a.end_sec - a.start_sec) * 1000.0) as f32, + g.webcam_cover, ); let anim_px = rh / crate::text_anim::ANIMATION_REFERENCE_HEIGHT; let (mut ax, mut ay, mut aw, mut ah) = ( @@ -2058,6 +2193,8 @@ impl Compositor { ca.set_store_action(metal::MTLStoreAction::Store); let enc = cmd.new_render_command_encoder(&desc); enc.set_render_pipeline_state(pipeline); + // La famille du pipeline de calque de cette passe, pour `draw_layer`. + self.layer_add.set(std::ptr::eq(pipeline, &self.pipeline_add)); Ok(enc) } @@ -2066,7 +2203,8 @@ impl Compositor { /// `frame_geometry::cursor_sprite_cb`, partagée avec Windows et Linux. /// /// Avec `model`, le même sprite extrudé (mode 15, `cursor_model_cb`) : le sprite en - /// texture(2), son champ de distance en texture(4). + /// texture(2), son champ de distance en texture(4), l'enregistrement que l'appelant a lié en + /// texture(0) et (1). #[allow(clippy::too_many_arguments)] unsafe fn draw_cursor_sprite( &self, @@ -2077,21 +2215,22 @@ impl Compositor { sprite: &crate::scene::SceneCursorSprite, clip: [f32; 4], model: Option, - ) -> Result<()> { + ) -> Result> { let (tex, iw, ih) = self.cached_image(sprite.path.as_str())?; // Sans champ de distance, repli sur le sprite plat plutôt qu'aucun curseur. Parité Linux. if let Some(pose) = model { match self.cursor_sdf(sprite.path.as_str()) { Ok((sdf, shape)) => { let shape = crate::frame_geometry::model_shape(sprite, shape); - if let Some(cb) = crate::frame_geometry::cursor_model_cb( + let Some(cb) = crate::frame_geometry::cursor_model_cb( placement, size_px, pose, shape, a, clip, - ) { - enc.set_fragment_texture(2, Some(&tex)); - enc.set_fragment_texture(4, Some(&sdf)); - self.draw_solid(enc, &cb); - } - return Ok(()); + ) else { + return Ok(None); + }; + enc.set_fragment_texture(2, Some(&tex)); + enc.set_fragment_texture(4, Some(&sdf)); + self.draw_solid(enc, &cb); + return Ok(Some(cb.dst)); } Err(e) => eprintln!("[curseur] champ de \"{}\" : {e:#}", sprite.path), } @@ -2109,7 +2248,7 @@ impl Compositor { ); enc.set_fragment_texture(2, Some(&tex)); self.draw_solid(enc, &cb); - Ok(()) + Ok(Some(cb.dst)) } /// Curseur thématisé : le sprite de l'état courant, sinon la flèche, sinon rien. @@ -2132,14 +2271,17 @@ impl Compositor { a: f32, clip: [f32; 4], model: Option, - ) { - let sprite = cursor_type.and_then(|t| sprites.get(t)).or_else(|| sprites.get("arrow")); - if let Some(sprite) = sprite { - if let Err(e) = self.draw_cursor_sprite(enc, placement, size_px, a, sprite, clip, model) - { - eprintln!("[compositor] sprite curseur \"{}\" : {e:#}", sprite.path); - } - } + ) -> Option<[f32; 4]> { + let sprite = cursor_type.and_then(|t| sprites.get(t)).or_else(|| sprites.get("arrow"))?; + self.draw_cursor_sprite(enc, placement, size_px, a, sprite, clip, model).unwrap_or_else(|e| { + eprintln!("[compositor] sprite curseur \"{}\" : {e:#}", sprite.path); + None + }) + } + + /// Le métrage dans la dernière image composée : ses coins et son warp (`FootageQuad`). + pub fn footage_quad(&self) -> Option { + self.footage.get() } /// Compose la frame : fond, ombre écran, écran, ombre caméra, caméra — puis miroir @@ -2211,6 +2353,7 @@ impl Compositor { timeline_t_override: *self.timeline_time.borrow(), programme_time: *self.programme_time.borrow(), }); + self.footage.set(Some(g.footage_quad([rw, rh]))); let cmd_buf = self.gpu.context.new_command_buffer(); // Profondeur de champ : pyramide remplie seulement sur une frame inclinée qui la lit, @@ -2220,46 +2363,75 @@ impl Compositor { } else { None }; - let enc = self.begin_pass( - cmd_buf, - &self.rt, - Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 1.0)), - &self.pipeline_main, - )?; - // Les deux plans écran restent liés par défaut : les quads de couleur ne les - // échantillonnent pas, mais Metal veut des slots renseignés pour les draws qui, eux, - // le font. - enc.set_fragment_texture(0, Some(&sy)); - enc.set_fragment_texture(1, Some(&suv)); - - // --- fond --- (parité `compositor_windows.rs`, section « fond ») - match scene_ref.as_ref().map(|s| s.background.clone()) { - Some(SceneBackground::Color { color }) => { - let c = parse_hex(&color).unwrap_or(lp.bg_color); - self.draw_solid( - enc, - &LayerCB { dst: [0.0, 0.0, 1.0, 1.0], mode: 1.0, color: c, ..Default::default() }, - ); - } - Some(SceneBackground::Gradient { angle_deg, stops, offsets, motion }) => { - let a = angle_deg.to_radians(); - let (anim, mb) = - crate::frame_geometry::wallpaper_motion_slots(motion, g.programme_t, rw / rh); - self.draw_solid( - enc, - &LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - fx: [a.sin(), -a.cos(), anim[0], anim[1]], - mb, - ..crate::frame_geometry::gradient_layer(&stops, &offsets, lp.bg_color) - }, - ); + // Le fond de la frame d'avant, flou compris, s'il est celui qu'on demande + // (`BackgroundKey`) : recopié au lieu d'être redessiné. + let bg_desc = scene_ref.as_ref().map(|s| &s.background); + let bg_blur = scene_ref.as_ref().map_or(0.0, |s| s.effects.blur); + let bg_cached = self + .bg_cache + .borrow() + .as_ref() + .is_some_and(|c| c.key.matches(bg_desc, bg_blur, lp.bg_color, [rw, rh])); + if bg_cached { + if let Some(c) = self.bg_cache.borrow().as_ref() { + self.blit_background(cmd_buf, &c.tex, true); } - Some(SceneBackground::Image { path, motion }) => { - // Repli couleur en cas d'échec, mais LOGGÉ : un fallback silencieux masquerait - // un chemin cassé. - if let Err(e) = self.draw_image_bg(enc, &path, rw / rh, motion, g.programme_t) { - eprintln!("[compositor] wallpaper image \"{path}\" : {e:#}"); + } else { + let enc = self.begin_pass( + cmd_buf, + &self.rt, + Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 1.0)), + &self.pipeline_main, + )?; + // Les deux plans écran restent liés par défaut : les quads de couleur ne les + // échantillonnent pas, mais Metal veut des slots renseignés pour les draws qui, eux, + // le font. + enc.set_fragment_texture(0, Some(&sy)); + enc.set_fragment_texture(1, Some(&suv)); + + // --- fond --- (parité `compositor_windows.rs`, section « fond ») + // Faux si l'image du fond n'a pas pu se charger : son repli n'est pas gardé. + let mut bg_drawn = true; + match scene_ref.as_ref().map(|s| s.background.clone()) { + Some(SceneBackground::Color { color }) => { + let c = parse_hex(&color).unwrap_or(lp.bg_color); + self.draw_solid( + enc, + &LayerCB { dst: [0.0, 0.0, 1.0, 1.0], mode: 1.0, color: c, ..Default::default() }, + ); + } + Some(SceneBackground::Gradient { angle_deg, stops, offsets, motion }) => { + let a = angle_deg.to_radians(); + let (anim, mb) = + crate::frame_geometry::wallpaper_motion_slots(motion, g.programme_t, rw / rh); + self.draw_solid( + enc, + &LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + fx: [a.sin(), -a.cos(), anim[0], anim[1]], + mb, + ..crate::frame_geometry::gradient_layer(&stops, &offsets, lp.bg_color) + }, + ); + } + Some(SceneBackground::Image { path, motion }) => { + // Repli couleur en cas d'échec, mais LOGGÉ : un fallback silencieux masquerait + // un chemin cassé. + if let Err(e) = self.draw_image_bg(enc, &path, rw / rh, motion, g.programme_t) { + eprintln!("[compositor] wallpaper image \"{path}\" : {e:#}"); + bg_drawn = false; + self.draw_solid( + enc, + &LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + mode: 1.0, + color: lp.bg_color, + ..Default::default() + }, + ); + } + } + None => { self.draw_solid( enc, &LayerCB { @@ -2271,25 +2443,34 @@ impl Compositor { ); } } - None => { - self.draw_solid( - enc, - &LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - mode: 1.0, - color: lp.bg_color, - ..Default::default() - }, - ); - } - } - // « Blur BG » (parité web `blurredBackgroundLayer`) : floute CE wallpaper qu'on vient - // de dessiner, pas la vidéo. No-op visuel sur une couleur plate, effet réel sur un - // gradient ou une image. Il lui faut ses propres passes, d'où la coupure ici. - enc.end_encoding(); - if let Some(blur) = scene_ref.as_ref().map(|s| s.effects.blur) { - self.blur_bg(cmd_buf, blur)?; + // « Blur BG » (parité web `blurredBackgroundLayer`) : floute CE wallpaper qu'on vient + // de dessiner, pas la vidéo. No-op visuel sur une couleur plate, effet réel sur un + // gradient ou une image. Il lui faut ses propres passes, d'où la coupure ici. + enc.end_encoding(); + if let Some(blur) = scene_ref.as_ref().map(|s| s.effects.blur) { + self.blur_bg(cmd_buf, blur)?; + } + // Pas de cache pour un fond animé (`of` rend `None`). + let key = bg_drawn + .then(|| crate::frame_geometry::BackgroundKey::of(bg_desc, bg_blur, lp.bg_color, [rw, rh])) + .flatten(); + if let Some(key) = key { + // L'ancienne texture est reprise : `resized` vide le cache, sa taille est la bonne. + let old = self.bg_cache.borrow_mut().take().map(|c| c.tex); + let tex = old.unwrap_or_else(|| { + make_texture( + &self.gpu.device, + metal::MTLPixelFormat::RGBA8Unorm, + self.render_w, + self.render_h, + metal::MTLStorageMode::Private, + metal::MTLTextureUsage::ShaderRead, + ) + }); + self.blit_background(cmd_buf, &tex, false); + *self.bg_cache.borrow_mut() = Some(BgCache { key, tex }); + } } // --- écran : ombre puis vidéo --- let s_px = [g.s_dst[2] * rw, g.s_dst[3] * rh]; @@ -2400,6 +2581,11 @@ impl Compositor { enc.end_encoding(); + // --- flous de confidentialité : sur le métrage, AVANT le curseur (parité Windows). Le + // curseur reste net par-dessus, et le cristal de Prism Glow, qui réfracte l'image + // composée, n'y voit que des pixels déjà floutés. + self.draw_annotations(cmd_buf, scene_ref.as_ref(), g.source_t, g.s_ann, &g, true)?; + // --- curseur --- (parité `compositor_windows.rs`, section « curseur custom ») if let Some(track) = cursor_ref.as_ref() { let plan = crate::frame_geometry::plan_cursor( @@ -2422,6 +2608,12 @@ impl Compositor { .map(|s| s.cursor.cursor_sprites.clone()) .unwrap_or_default(); let kind = plan.cursor_type.as_deref(); + // Le cristal de Prism Glow réfracte l'image telle qu'elle est composée à cet + // instant, flous compris : sa copie en texture(5), une fois pour toutes les copies + // de la traînée (`ann_copy` est libre, les flous l'ont déjà lue). + if plan.glass { + self.snapshot_rt(cmd_buf, false); + } // L'impact des clics (mode 16, sans texture), posé sur l'écran SOUS le curseur. if !plan.impacts.is_empty() { let e = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)?; @@ -2432,6 +2624,9 @@ impl Compositor { } if plan.taps <= 1 { let e = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)?; + if plan.glass { + e.set_fragment_texture(5, Some(&self.ann_copy)); + } self.draw_cur_themed( e, &sprites, @@ -2454,11 +2649,17 @@ impl Compositor { Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 0.0)), &self.pipeline_add, )?; + if plan.glass { + e.set_fragment_texture(5, Some(&self.ann_copy)); + } + // L'union des quads peints dans `accum` (x0, y0, x1, y1 en fractions de la + // sortie) : la seule région que la recopie doit relire. + let mut bounds = [f32::MAX, f32::MAX, f32::MIN, f32::MIN]; for k in 0..plan.taps { let f = k as f32 / (plan.taps - 1) as f32; let w = crate::frame_geometry::cursor_tap_weight(k, plan.taps); e.set_blend_color(w, w, w, w); - self.draw_cur_themed( + let drawn = self.draw_cur_themed( e, &sprites, kind, @@ -2468,12 +2669,36 @@ impl Compositor { plan.clip, plan.model, ); + if let Some(d) = drawn { + bounds = [ + bounds[0].min(d[0]), + bounds[1].min(d[1]), + bounds[2].max(d[0] + d[2]), + bounds[3].max(d[1] + d[3]), + ]; + } } e.end_encoding(); let c = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_fs_tex)?; c.set_fragment_texture(0, Some(&self.accum)); - c.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 3); + // Rien n'a été peint dans `accum` hors des quads de la traînée : la recopie + // s'y borne au lieu de repasser sur toute la sortie (0,5 ms par frame 1080p + // mesurée sur Linux, Radeon 610M). + let (w, h) = (self.render_w as f32, self.render_h as f32); + let x0 = (bounds[0] * w).floor().clamp(0.0, w) as u64; + let y0 = (bounds[1] * h).floor().clamp(0.0, h) as u64; + let x1 = (bounds[2] * w).ceil().clamp(0.0, w) as u64; + let y1 = (bounds[3] * h).ceil().clamp(0.0, h) as u64; + if x1 > x0 && y1 > y0 { + c.set_scissor_rect(metal::MTLScissorRect { + x: x0, + y: y0, + width: x1 - x0, + height: y1 - y0, + }); + c.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 3); + } c.end_encoding(); } } @@ -2481,105 +2706,150 @@ impl Compositor { // --- caméra : ombre PiP puis vidéo --- let enc = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)?; - if let (true, Some((wy, wuv))) = (lp.has_webcam, webcam_tex.as_ref()) { - let [cu0, cv0, cu1, cv1] = crate::frame_geometry::webcam_source_rect( - [wcw, wch], - [wtw as f32, wth as f32], - scene_ref.as_ref().and_then(|scene| scene.layout.webcam_crop), - g.w_px[0] / g.w_px[1].max(0.0001), - ); - let (u0, u1) = if lp.webcam_mirror { (cu1, cu0) } else { (cu0, cu1) }; - let webcam_is_block = matches!( - g.scene_preset.as_deref(), - Some("dual-frame") | Some("vertical-stack") - ); - // Effet d'arrière-plan : le mode vient de la scène, le masque par pixel de - // l'inférence. Les DEUX sont requis — un mode sans masque rendrait la webcam - // invisible en détourage, donc tant que rien n'a été segmenté on dessine la - // piste telle quelle. C'est aussi ce qui rend le premier lancement gracieux. - let mask = self.webcam_mask.borrow(); - let effect = scene_ref - .as_ref() - .and_then(|s| s.webcam_effect.as_ref()) - .filter(|_| mask.is_some()) - .map(|e| (e.shader_code(), e)) - .filter(|(code, _)| *code > 0.0); - - // L'ombre appartient à la bulle PiP. En détourage il n'y a plus de bulle — une - // ombre portée par un rectangle invisible se lit comme un artefact. Le test porte - // sur le code de la SCÈNE et non sur celui envoyé au shader : le fond personnalisé - // part lui aussi en détourage ci-dessous, mais sa bulle, elle, est bien peinte et - // garde donc son ombre. - let is_cutout = matches!(effect, Some((code, _)) if code == 1.0); - if cfg.shadow && !webcam_is_block && !is_cutout && g.shape_fade > 0.0 { - self.draw_shadow( - enc, - g.w_dst, - g.w_px, - g.w_radius, - WEBCAM_SHADOW_SPREAD_FRAC * g.frame_min_px, - [0.0, WEBCAM_SHADOW_OFFSET_FRAC * g.frame_min_px], - WEBCAM_SHADOW_OPACITY * g.shape_fade, + // Camera 0's crop, mirror and desk turn as camera settings, plus its homography + // (`Scene::camera(0)`): with one, `src` is not read and the crop is dropped. + let cam0 = g.camera0_settings(scene_ref.as_ref()); + let webcam_is_block = matches!( + g.scene_preset.as_deref(), + Some("dual-frame") | Some("vertical-stack") + ); + // Effet d'arrière-plan : le mode vient de la scène, le masque par pixel de + // l'inférence. Les DEUX sont requis — un mode sans masque rendrait la webcam + // invisible en détourage, donc tant que rien n'a été segmenté on dessine la + // piste telle quelle. C'est aussi ce qui rend le premier lancement gracieux. + let mask = self.webcam_mask.borrow(); + let effect = scene_ref + .as_ref() + .and_then(|s| s.webcam_effect.as_ref()) + .filter(|_| mask.is_some()) + .map(|e| (e.shader_code(), e)) + .filter(|(code, _)| *code > 0.0); + // L'ombre appartient à la bulle PiP. En détourage il n'y a plus de bulle — une + // ombre portée par un rectangle invisible se lit comme un artefact. Le test porte + // sur le code de la SCÈNE et non sur celui envoyé au shader : le fond personnalisé + // part lui aussi en détourage ci-dessous, mais sa bulle, elle, est bien peinte et + // garde donc son ombre. + let is_cutout = matches!(effect, Some((code, _)) if code == 1.0); + // Fond personnalisé : on PEINT le fond dans la bulle, puis on y découpe la caméra + // par-dessus — le mélange alpha donne `lerp(fond, caméra, personne)`, soit exactement + // ce que la branche « mode 3 » du shader calculait, mais pour les TROIS sortes de + // fond. Le shader ne sait peindre qu'une couleur plate sous le masque ; dégradés et + // images y tombaient sur du noir, et le défaut EST une image. L'ordre est imposé : + // ombre, puis fond, puis caméra. `custom_bg` = Some(background) in that mode. + let (effect_code, blur_intensity, custom_bg) = match effect { + Some((code, e)) if code > 2.5 => (1.0, 0.0, Some(e.background.as_ref())), + Some((code, e)) => (code, e.blur_intensity.clamp(0.0, 1.0), None), + None => (0.0, 0.0, None), + }; + // Metal tolère l'index 3 non lié tant que `fx.z` reste à 0 : la branche n'est + // pas prise, la texture n'est pas échantillonnée. Dès qu'il monte, elle doit + // l'être sur TOUT draw capable de la prendre — ici seul celui de la caméra 0. L'état + // d'un encodeur est rémanent, donc lier avant le draw suffit, et l'ombre puis le + // fond qui précèdent sont en modes 1/2/5/6, que `ps_main` garde hors de la branche + // (`mode < 0.5`). The extra cameras draw with `fx.z = 0` and never read it. + // + // Pas de déliaison après coup, contrairement au chemin Windows qui remet le slot + // t3 à `None` : cet état meurt avec l'encodeur, et les annotations en ouvrent un + // autre. Il n'y a rien sur quoi fuir. + if let Some(m) = mask.as_ref() { + enc.set_fragment_texture(3, Some(&m.tex)); + } + // The extra cameras' textures, held to the end of `compose_frame` like `webcam_tex`. + let mut extra_textures = Vec::new(); + if g.camera_layers.is_empty() { + // Without layout regions: camera 0 from `w_dst`, as before the layers. + if let (true, Some((wy, wuv))) = (lp.has_webcam, webcam_tex.as_ref()) { + // Mirror and the desk-shot turn are both bound swaps: u for horizontal, v for + // vertical. + let src = crate::frame_geometry::camera_source_rect( + Some(&cam0), + [wcw, wch], + [wtw as f32, wth as f32], + g.w_px[0] / g.w_px[1].max(0.0001), ); + if cfg.shadow && !webcam_is_block && !is_cutout && g.shape_fade > 0.0 { + self.draw_shadow( + enc, + g.w_dst, + g.w_px, + g.w_radius, + WEBCAM_SHADOW_SPREAD_FRAC * g.frame_min_px, + [0.0, WEBCAM_SHADOW_OFFSET_FRAC * g.frame_min_px], + WEBCAM_SHADOW_OPACITY * g.shape_fade, + ); + } + if let Some(bg) = custom_bg { + self.draw_webcam_bg(enc, bg, g.w_dst, g.w_px, g.w_radius, 0.0); + } + let cb = g.webcam_video_cb(src, w_valid, effect_code, blur_intensity); + let cb = crate::frame_geometry::with_camera_homography(cb, Some(&cam0)); + self.draw_video(enc, &cb, wy, wuv); } - - // Fond personnalisé : on PEINT le fond dans la bulle, puis on y découpe la caméra - // par-dessus — le mélange alpha donne `lerp(fond, caméra, personne)`, soit exactement - // ce que la branche « mode 3 » du shader calculait, mais pour les TROIS sortes de - // fond. Le shader ne sait peindre qu'une couleur plate sous le masque ; dégradés et - // images y tombaient sur du noir, et le défaut EST une image. L'ordre est imposé : - // ombre, puis fond, puis caméra. - let (effect_code, blur_intensity) = match effect { - Some((code, e)) if code > 2.5 => { - self.draw_webcam_bg(enc, e.background.as_ref(), g.w_dst, g.w_px, g.w_radius); - (1.0, 0.0) + } else { + // Every planned layer, in order: shadow, then (camera 0) the custom background, + // then the video. Camera 0 keeps its frame, effects and mask; the extra cameras + // draw from `set_extra_camera_frames` without effects, and a missing one is skipped. + let render = [rw, rh]; + let cam0_base = g.webcam_video_cb([0.0; 4], w_valid, effect_code, blur_intensity); + for plan in &g.camera_layers { + if plan.camera == 0 { + let (true, Some((wy, wuv))) = (lp.has_webcam, webcam_tex.as_ref()) else { + continue; + }; + let video = crate::frame_geometry::camera_layer_cb( + plan, + Some(&cam0), + [wcw, wch], + [wtw as f32, wth as f32], + render, + &cam0_base, + ); + if let Some(shadow) = + g.camera_layer_shadow(plan, &video, render, cfg.shadow, is_cutout) + { + self.draw_solid(enc, &shadow); + } + if let Some(bg) = custom_bg { + let (dst, quad_px, radius_px) = (video.dst, video.quad_px, video.radius_px); + self.draw_webcam_bg(enc, bg, dst, quad_px, radius_px, video.layer_fx[0]); + } + self.draw_video(enc, &video, wy, wuv); + } else { + // Tolerant like camera 0: no frame, or no pixel buffer in it, skips the layer. + let frame = self.extra_camera_frame(plan.camera); + let Ok((ey, euv)) = self.nv12_srvs(frame) else { continue }; + let (tw, th) = self.tex_dims(frame); + let visible = [(*frame).width as f32, (*frame).height as f32]; + let tex = [tw as f32, th as f32]; + let valid = [visible[0] / tex[0].max(1.0), visible[1] / tex[1].max(1.0)]; + let video = crate::frame_geometry::camera_layer_cb( + plan, + scene_ref.as_ref().and_then(|s| s.camera(plan.camera)), + visible, + tex, + render, + &crate::frame_geometry::extra_camera_base_cb(valid), + ); + if let Some(shadow) = + g.camera_layer_shadow(plan, &video, render, cfg.shadow, false) + { + self.draw_solid(enc, &shadow); + } + self.draw_video(enc, &video, &ey, &euv); + extra_textures.push((ey, euv)); } - Some((code, e)) => (code, e.blur_intensity.clamp(0.0, 1.0)), - None => (0.0, 0.0), - }; - - // Metal tolère l'index 3 non lié tant que `fx.z` reste à 0 : la branche n'est - // pas prise, la texture n'est pas échantillonnée. Dès qu'il monte, elle doit - // l'être sur TOUT draw capable de la prendre — ici il n'y en a qu'un. L'état - // d'un encodeur est rémanent, donc lier avant le draw suffit, et l'ombre puis le - // fond qui précèdent sont en modes 1/2/5/6, que `ps_main` garde hors de la branche - // (`mode < 0.5`). - // - // Pas de déliaison après coup, contrairement au chemin Windows qui remet le slot - // t3 à `None` : cet état meurt avec l'encodeur, et les annotations en ouvrent un - // autre. Il n'y a rien sur quoi fuir. - if let Some(m) = mask.as_ref() { - enc.set_fragment_texture(3, Some(&m.tex)); } - self.draw_video( - enc, - &LayerCB { - dst: g.w_dst, - src: [u0, cv0, u1, cv1], - quad_px: g.w_px, - radius_px: g.w_radius, - mode: 0.0, - // `color.a` porte l'alpha du découpage (`color.a * personne`) ; le RGB n'est - // plus lu, le fond ayant déjà été peint sous la caméra. - color: [0.0, 0.0, 0.0, 1.0], - fx: [w_valid[0], w_valid[1], effect_code, blur_intensity], - src_prev: [u0, cv0, u1, cv1], - dst_prev: g.w_dst_prev, - mb: [g.mb_taps, g.mb_amount, 1.0, 0.0], - ..Default::default() - }, - wy, - wuv, - ); } + drop(mask); enc.end_encoding(); // --- annotations : calque le plus haut, ancré sur le rect ÉCRAN SANS ZOOM --- // `s_ann`, pas `s_dst` : le zoom vit dans la boîte depuis l'issue #179, donc `s_dst` // grandit avec lui et emmenait annotations et sous-titres dans le mouvement. Le flou de - // confidentialité est l'exception : il suit le contenu, d'où `&g`. - self.draw_annotations(cmd_buf, scene_ref.as_ref(), g.source_t, g.s_ann, &g)?; + // confidentialité est l'exception : il suit le contenu, d'où `&g` ; il est passé avant le + // curseur (plus haut). + self.draw_annotations(cmd_buf, scene_ref.as_ref(), g.source_t, g.s_ann, &g, false)?; // Ni miroir RGBA ni attente ici : le miroir ne sert qu'à `readback_direct` (la // preview), et l'export ne lit jamais le RGBA — le blit pleine résolution était payé @@ -4267,6 +4537,12 @@ mod tests { &metal::CompileOptions::new(), ) .expect("shaders.metal doit compiler"); + // La variante des modèles 3D (`LAYER_MODELS`) : la seule à compiler leurs branches. + device + .new_library_with_source(&super::msl_source_with_models(), &metal::CompileOptions::new()) + .expect("shaders.metal doit compiler avec LAYER_MODELS") + .get_function("ps_main", None) + .expect("ps_main absent de la variante LAYER_MODELS"); for name in [ "vs_main", "vs_fs", diff --git a/crates/compositor/src/compositor_windows.rs b/crates/compositor/src/compositor_windows.rs index 59431e383..74c60b07f 100644 --- a/crates/compositor/src/compositor_windows.rs +++ b/crates/compositor/src/compositor_windows.rs @@ -24,8 +24,9 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; use std::ffi::c_void; use windows::core::Interface; +use windows::Win32::Foundation::RECT; use windows::Win32::Graphics::Direct3D::{ - D3D11_SRV_DIMENSION_TEXTURE2DARRAY, D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP, + D3D11_SRV_DIMENSION_TEXTURE2D, D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP, }; use windows::Win32::Graphics::Direct3D11::*; use windows::Win32::Graphics::Dxgi::Common::*; @@ -78,6 +79,12 @@ struct WebcamMask { /// demi-résolution de la texture DÉCODEUR, avec sa chaîne de mips. Dimensionnée sur la texture /// décodeur et remplie en UV plein (0..1) : ses UV sont ceux de t0/t1, et le calcul d'UV du /// mode 8 ne change pas. +/// Le fond tel que la passe de fond (et son flou) l'a laissé dans le RT, et la clé qui le décrit. +struct BgCache { + key: crate::frame_geometry::BackgroundKey, + tex: ID3D11Texture2D, +} + struct DofPyramid { rtv: ID3D11RenderTargetView, srv: ID3D11ShaderResourceView, @@ -85,6 +92,15 @@ struct DofPyramid { height: u32, } +/// An extra camera's frame for this `compose_frame`: its private copy's plane views, and the +/// visible and texture sizes `camera_layer_cb` crops against. +struct ExtraCamera { + y: ID3D11ShaderResourceView, + uv: ID3D11ShaderResourceView, + visible: [f32; 2], + tex: [f32; 2], +} + pub struct Compositor { dev: ID3D11Device, ctx: ID3D11DeviceContext, @@ -93,7 +109,10 @@ pub struct Compositor { rt_srv: ID3D11ShaderResourceView, staging: ID3D11Texture2D, vs: ID3D11VertexShader, + /// Le pixel shader des calques, SANS les modèles 3D (modes 15 à 17) : ils passent par + /// `ps_models`, et `draw_layer` choisit d'après le mode (le haut de `shaders.hlsl`). ps: ID3D11PixelShader, + ps_models: ID3D11PixelShader, vs_fs: ID3D11VertexShader, ps_y: ID3D11PixelShader, ps_uv: ID3D11PixelShader, @@ -129,11 +148,16 @@ pub struct Compositor { accum_rtv: ID3D11RenderTargetView, accum_srv: ID3D11ShaderResourceView, blend_add: ID3D11BlendState, + /// Rastérisation avec scissor, pour la seule recopie de la traînée du curseur. + rs_scissor: ID3D11RasterizerState, /// Rendu isolé de l'écran cadré (ombre, cadre, métrage, appareil), transparent autour, que le /// mode 18 recompose le long de sa trajectoire (`FrameGeometry::screen_trail`). Distinct de /// `accum`, que le curseur et `compose_frame_mb` remplissent dans la même frame. trail_rtv: ID3D11RenderTargetView, trail_srv: ID3D11ShaderResourceView, + /// Le dernier fond fixe composé et sa clé (`BackgroundKey`) : recopié dans le RT tant + /// que la clé tient, au lieu d'être redessiné. + bg_cache: RefCell>, /// RefCell (pas un simple champ) pour que `set_cursor` reste `&self`, comme `set_scene` / /// `set_live_params` — nécessaire pour le rebrancher par clip dans l'export multiclip, qui /// n'a qu'une référence partagée au `Compositor`. @@ -142,6 +166,8 @@ pub struct Compositor { /// (`frame / FPS`). L'export multiclip et le live le positionnent au PTS écran courant, /// c'est-à-dire au temps source absolu du clip actif. cursor_t_override: RefCell>, + /// Le métrage dans la dernière image composée, que l'éditeur lit avec elle (`live.rs`). + footage: std::cell::Cell>, /// Override du temps des zoom/full-camera regions (secondes source du clip actif). Le nom /// `timeline_t_override` est conservé pour l'API existante, mais ce temps n'est plus cumulé /// entre clips : les régions projetées par l'app portent elles aussi des temps source. @@ -149,9 +175,14 @@ pub struct Compositor { timeline_t_override: RefCell>, /// Temps programme (secondes de sortie) — cf. `FrameGeometryInput::programme_time`. programme_time: RefCell>, - // cache des SRV décodeur par (texture array, slice) : le pool réutilise ~32 textures, - // donc après warmup plus aucune création de SRV par frame (overhead CPU supprimé). - srv_cache: RefCell>, + /// A private copy of a decoder surface, with its two plane views (Y, UV). + /// Keyed by the decoder texture's pointer, which pins nothing: `nv12_srvs` re-checks + /// the copy's size and format on every hit, so a new texture landing on an old address + /// is never handed a copy sized for the old one. `clear_srv_cache` frees stale entries. + srv_cache: RefCell>, + /// The frames of cameras 1..=3 (`set_extra_camera_frames`), as addresses (0 = none) so + /// the compositor's auto traits stay what they were without the raw pointers. + extra_camera_frames: Cell<[usize; crate::camera_layers::MAX_EXTRA_CAMERAS]>, live_params: RefCell, /// Scène pilotée par l'app (contrat) : quand présente, remplace le layout fixture de /// `timeline()`. Voir `scene.rs` / `SceneDescription` (TS). @@ -404,6 +435,7 @@ impl Compositor { render_size: Cell::new((w, h)), // Caches dimensionnés à l'ancienne taille : recréés à la demande. resize_target: RefCell::new(None), + bg_cache: RefCell::new(None), live_readback_staging: RefCell::new(None), nv12_readback_staging: RefCell::new(None), ..self @@ -600,6 +632,8 @@ impl Compositor { dev.CreateVertexShader(shader!("vs_main"), None, Some(&mut vs))?; let mut ps: Option = None; dev.CreatePixelShader(shader!("ps_main"), None, Some(&mut ps))?; + let mut ps_models: Option = None; + dev.CreatePixelShader(shader!("ps_main_models"), None, Some(&mut ps_models))?; // shaders RGB->NV12 let mut vs_fs: Option = None; @@ -680,6 +714,17 @@ impl Compositor { }; let mut blend_add: Option = None; dev.CreateBlendState(&bla, Some(&mut blend_add))?; + // L'état par défaut (aucun lié), plus le scissor. Sans culling : le triangle plein écran + // n'a pas à dépendre de son sens de parcours. + let rsd = D3D11_RASTERIZER_DESC { + FillMode: D3D11_FILL_SOLID, + CullMode: D3D11_CULL_NONE, + ScissorEnable: true.into(), + DepthClipEnable: true.into(), + ..Default::default() + }; + let mut rs_scissor: Option = None; + dev.CreateRasterizerState(&rsd, Some(&mut rs_scissor))?; Ok(Compositor { dev, @@ -690,6 +735,7 @@ impl Compositor { staging, vs: vs.unwrap(), ps: ps.unwrap(), + ps_models: ps_models.unwrap(), vs_fs: vs_fs.unwrap(), ps_y: ps_y.unwrap(), ps_uv: ps_uv.unwrap(), @@ -718,13 +764,17 @@ impl Compositor { accum_rtv, accum_srv, blend_add: blend_add.unwrap(), + rs_scissor: rs_scissor.unwrap(), trail_rtv, trail_srv, + bg_cache: RefCell::new(None), cursor: RefCell::new(None), cursor_t_override: RefCell::new(None), + footage: std::cell::Cell::new(None), timeline_t_override: RefCell::new(None), programme_time: RefCell::new(None), srv_cache: RefCell::new(HashMap::new()), + extra_camera_frames: Cell::new([0; crate::camera_layers::MAX_EXTRA_CAMERAS]), live_params: RefCell::new(LiveParams::default()), scene: RefCell::new(None), text_raster: match crate::text::TextRasterizer::new() { @@ -803,43 +853,150 @@ impl Compositor { /// depuis le layout preset au lieu du planning fixture. pub fn set_scene(&self, s: Option) { *self.scene.borrow_mut() = s; + // A new scene can drop the regions that drew an extra camera: forget its frames so a + // stale pointer is never drawn before the next `set_extra_camera_frames`. + self.extra_camera_frames.set([0; crate::camera_layers::MAX_EXTRA_CAMERAS]); + } + + /// Frames for cameras 1..=3 (index 0 = scene camera 1). A null or missing entry means that + /// camera is not drawn this frame. The compositor keeps the pointers and reads them in every + /// `compose_frame` until the next call, so they must stay valid until then. They go through + /// `nv12_srvs` and its cache like camera 0's frame (`clear_srv_cache` clears them alike). + pub unsafe fn set_extra_camera_frames(&self, frames: &[*const AVFrame]) { + let mut slots = [0usize; crate::camera_layers::MAX_EXTRA_CAMERAS]; + for (slot, frame) in slots.iter_mut().zip(frames) { + *slot = *frame as usize; + } + self.extra_camera_frames.set(slots); + } + + /// The frame set for scene camera `camera` (1..=3), null when there is none. + fn extra_camera_frame(&self, camera: usize) -> *const AVFrame { + let slots = self.extra_camera_frames.get(); + camera.checked_sub(1).and_then(|k| slots.get(k).copied()).unwrap_or(0) as *const AVFrame } - /// Crée les SRV Y (R8) et UV (R8G8) sur la tranche d'array de la frame décodeur. + /// The copies (`nv12_srvs`) of every extra camera `layers` draws, with their visible and + /// texture sizes. Tolerant: a camera without a frame, or whose frame has no D3D11 texture, + /// is `None` and simply not drawn. + unsafe fn extra_camera_srvs( + &self, + layers: &[crate::camera_layers::CameraLayerPlan], + ) -> [Option; crate::camera_layers::MAX_EXTRA_CAMERAS] { + let mut out: [Option; crate::camera_layers::MAX_EXTRA_CAMERAS] = + Default::default(); + for (k, slot) in out.iter_mut().enumerate() { + let frame = self.extra_camera_frame(k + 1); + if frame.is_null() || !layers.iter().any(|l| l.camera == k + 1) { + continue; + } + let Ok((y, uv)) = self.nv12_srvs(frame) else { continue }; + let (tw, th) = self.tex_dims(frame); + *slot = Some(ExtraCamera { + y, + uv, + visible: [(*frame).width as f32, (*frame).height as f32], + tex: [tw as f32, th as f32], + }); + } + out + } + + /// The Y (R8) and UV (R8G8) views of the decoder frame — over a private COPY, never + /// over the decoder surface itself. + /// + /// Two documented D3D11 rules rule out the direct path. The first is explicit, on + /// `D3D11_BIND_DECODER`: "you cannot use texture arrays that are created with this + /// flag in calls to ID3D11Device::CreateShaderResourceView". The ffmpeg pool IS such + /// an array (`initial_pool_size` = 32, see `get_hw_format`). The second explains what + /// we were seeing: between the video engine and the 3D pipeline "there is no automatic + /// hazard tracking" — the surface stays the decoder's reference frame, so it may be + /// rewritten WHILE the shader samples it. ffmpeg's `ID3D11VideoContext` is in fact the + /// SAME object as our immediate context (it comes out of a QueryInterface on it), and + /// `SetMultithreadProtected(TRUE)` only makes an individual call atomic, never a + /// sequence. + /// + /// Measured on a Snapdragon X Elite (Adreno X1-85), same build, same recording, the + /// path picked by an environment variable: 768 black frames out of 792 when sampling + /// the decoder surface, 2 out of 545 through the copy. The black was OPAQUE and total, + /// so not "one frame late" but no frame at all. + /// + /// `CopySubresourceRegion` is issued on the immediate context, so it IS ordered against + /// the draws that follow; the destination texture is `ArraySize = 1` and carries only + /// `BIND_SHADER_RESOURCE`, which also takes it out of the restriction above. The cost + /// is one GPU→GPU copy per frame and per source, with nothing going back to system + /// memory. + /// + /// What hid the cause for so long: exporting the SAME scene never produced a black + /// frame (28 342 verified), because the export drains the GPU every frame through the + /// encoder. Everything that slowed the live loop — one more readback, a lock, a pause — + /// cut the black proportionally without ever removing it. pub unsafe fn nv12_srvs( &self, frame: *const AVFrame, ) -> Result<(ID3D11ShaderResourceView, ID3D11ShaderResourceView)> { let tex_ptr = (*frame).data[0] as *mut c_void; let slice = (*frame).data[1] as u32; - // cache hit : le pool réutilise les mêmes textures -> zéro création après warmup - let key = (tex_ptr as usize, slice); - if let Some((y, uv)) = self.srv_cache.borrow().get(&key) { - return Ok((y.clone(), uv.clone())); - } - let tex = ID3D11Texture2D::from_raw_borrowed(&tex_ptr) - .ok_or_else(|| anyhow::anyhow!("frame sans texture D3D11"))? + let src = ID3D11Texture2D::from_raw_borrowed(&tex_ptr) + .ok_or_else(|| anyhow::anyhow!("frame has no D3D11 texture"))? .clone(); - let mk = |fmt: DXGI_FORMAT| -> Result { - let mut d = D3D11_SHADER_RESOURCE_VIEW_DESC { - Format: fmt, - ViewDimension: D3D11_SRV_DIMENSION_TEXTURE2DARRAY, - ..Default::default() - }; - d.Anonymous.Texture2DArray = D3D11_TEX2D_ARRAY_SRV { - MostDetailedMip: 0, - MipLevels: 1, - FirstArraySlice: slice, - ArraySize: 1, - }; - let mut srv: Option = None; - self.dev.CreateShaderResourceView(&tex, Some(&d), Some(&mut srv))?; - Ok(srv.unwrap()) + // Cache hit: the pool reuses the same textures, so this allocates once per decoder + // after warmup. The COPY itself happens on EVERY frame — that is what freezes the + // slice's contents before the decoder takes it back. + // + // The key is a bare address and pins nothing. A decoder replaced without + // `clear_srv_cache` (a mid-stream resolution change, `CpuFrames::ensure_tex`) can + // hand a NEW texture the address of an old one, and a copy into a smaller destination + // is dropped without an error: the picture freezes. So a hit only counts while the + // destination still has the source's size and format. + let mut sd = D3D11_TEXTURE2D_DESC::default(); + src.GetDesc(&mut sd); + let key = tex_ptr as usize; + let cached = self.srv_cache.borrow().get(&key).cloned().filter(|(dst, ..)| { + let mut dd = D3D11_TEXTURE2D_DESC::default(); + dst.GetDesc(&mut dd); + (dd.Width, dd.Height, dd.Format) == (sd.Width, sd.Height, sd.Format) + }); + let (dst, y, uv) = match cached { + Some(v) => v, + None => { + let dd = D3D11_TEXTURE2D_DESC { + Width: sd.Width, + Height: sd.Height, + MipLevels: 1, + ArraySize: 1, + Format: sd.Format, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let mut dst: Option = None; + self.dev.CreateTexture2D(&dd, None, Some(&mut dst))?; + let dst = dst.unwrap(); + let mk = |fmt: DXGI_FORMAT| -> Result { + let mut d = D3D11_SHADER_RESOURCE_VIEW_DESC { + Format: fmt, + ViewDimension: D3D11_SRV_DIMENSION_TEXTURE2D, + ..Default::default() + }; + d.Anonymous.Texture2D = D3D11_TEX2D_SRV { MostDetailedMip: 0, MipLevels: 1 }; + let mut srv: Option = None; + self.dev.CreateShaderResourceView(&dst, Some(&d), Some(&mut srv))?; + Ok(srv.unwrap()) + }; + let y = mk(DXGI_FORMAT_R8_UNORM)?; + let uv = mk(DXGI_FORMAT_R8G8_UNORM)?; + self.srv_cache + .borrow_mut() + .insert(key, (dst.clone(), y.clone(), uv.clone())); + (dst, y, uv) + } }; - let y = mk(DXGI_FORMAT_R8_UNORM)?; - let uv = mk(DXGI_FORMAT_R8G8_UNORM)?; - self.srv_cache.borrow_mut().insert(key, (y.clone(), uv.clone())); + + self.ctx.CopySubresourceRegion(&dst, 0, 0, 0, 0, &src, slice, None); Ok((y, uv)) } @@ -874,6 +1031,26 @@ impl Compositor { self.ctx.ClearRenderTargetView(&self.rtv, &clear); } + /// Garde le fond que le RT porte à cet instant (`bg_cache`), sous `key`. + unsafe fn keep_background(&self, key: crate::frame_geometry::BackgroundKey) -> Result<()> { + // L'ancienne texture est reprise : `resized` vide le cache, sa taille est la bonne. + let old = self.bg_cache.borrow_mut().take().map(|c| c.tex); + let tex = match old { + Some(tex) => tex, + None => { + let mut d = D3D11_TEXTURE2D_DESC::default(); + self.rt.GetDesc(&mut d); + d.BindFlags = 0; + let mut tex: Option = None; + self.dev.CreateTexture2D(&d, None, Some(&mut tex))?; + tex.ok_or_else(|| anyhow::anyhow!("CreateTexture2D (fond gardé)"))? + } + }; + self.ctx.CopyResource(&tex, &self.rt); + *self.bg_cache.borrow_mut() = Some(BgCache { key, tex }); + Ok(()) + } + /// Passe plein écran générique (triangle unique) : `srv` -> `rtv` via `ps`, avec `fx`. unsafe fn fs_pass( &self, @@ -1025,6 +1202,15 @@ impl Compositor { self.ctx.PSSetConstantBuffers(0, Some(&[Some(self.cbuf.clone())])); } + /// Dessine le quad d'un calque, ses ressources déjà liées. Le pixel shader suit son mode : + /// `ps_models` pour un modèle 3D (`LayerCB::needs_models`), `ps` pour tout le reste, qui ne + /// paie pas ainsi les registres de ces modèles. + unsafe fn draw_layer(&self, cb: &LayerCB) { + self.upload_cb(cb); + self.ctx.PSSetShader(if cb.needs_models() { &self.ps_models } else { &self.ps }, None); + self.ctx.Draw(4, 0); + } + /// Calque vidéo NV12. pub unsafe fn draw_video( &self, @@ -1032,16 +1218,14 @@ impl Compositor { srv_y: &ID3D11ShaderResourceView, srv_uv: &ID3D11ShaderResourceView, ) { - self.upload_cb(cb); self.ctx .PSSetShaderResources(0, Some(&[Some(srv_y.clone()), Some(srv_uv.clone())])); - self.ctx.Draw(4, 0); + self.draw_layer(cb); } /// Calque couleur pleine (fond). pub unsafe fn draw_solid(&self, cb: &LayerCB) { - self.upload_cb(cb); - self.ctx.Draw(4, 0); + self.draw_layer(cb); } /// Fond wallpaper image (cover-fit). `path` = chemin absolu (résolu côté app). Décodé et @@ -1103,13 +1287,14 @@ impl Compositor { programme_t: f32, ) -> Result<()> { let full = [0.0, 0.0, 1.0, 1.0]; - self.draw_image_in(path, full, [0.0, 0.0], 0.0, output_aspect, motion, programme_t) + self.draw_image_in(path, full, [0.0, 0.0], 0.0, output_aspect, motion, programme_t, 0.0) } /// `draw_image_bg` pour un rect quelconque — la bulle webcam s'en sert avec ses coins /// arrondis. `output_aspect` est le ratio du RECT visé, pas celui de la sortie : le crop /// « cover » se calcule contre la zone qu'on remplit. `motion` anime l'image au temps - /// programme `programme_t` ; la bulle passe `WallpaperMotion::None`. + /// programme `programme_t` ; la bulle passe `WallpaperMotion::None`. `transparency` = + /// `layer_fx.x` (0 = opaque): a camera layer's background box fades with the layer. #[allow(clippy::too_many_arguments)] unsafe fn draw_image_in( &self, @@ -1120,6 +1305,7 @@ impl Compositor { output_aspect: f32, motion: WallpaperMotion, programme_t: f32, + transparency: f32, ) -> Result<()> { let (srv, iw, ih) = self.cached_image(path)?; let ai = iw as f32 / ih as f32; @@ -1135,7 +1321,8 @@ impl Compositor { (0.0, (1.0 - vis) * 0.5, 1.0, 1.0 - (1.0 - vis) * 0.5) }; let (anim, mb) = crate::frame_geometry::wallpaper_motion_slots(motion, programme_t, ao); - self.upload_cb(&LayerCB { + self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); + self.draw_layer(&LayerCB { dst, src: [u0, v0, u1, v1], quad_px, @@ -1143,10 +1330,9 @@ impl Compositor { mode: 6.0, fx: [0.0, 0.0, anim[0], anim[1]], mb, + layer_fx: [transparency, 0.0, 0.0, 0.0], ..Default::default() }); - self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); - self.ctx.Draw(4, 0); Ok(()) } @@ -1162,20 +1348,25 @@ impl Compositor { /// /// `quad_px` / `radius_px` sont ceux de la bulle : le fond doit épouser ses coins arrondis, /// sinon un rectangle déborde derrière la caméra. + /// + /// `transparency` = the camera layer's `layer_fx.x`: the box fades with the camera. unsafe fn draw_webcam_bg( &self, bg: Option<&SceneBackground>, dst: [f32; 4], quad_px: [f32; 2], radius_px: f32, + transparency: f32, ) { const BLACK: [f32; 4] = [0.0, 0.0, 0.0, 1.0]; + let layer_fx = [transparency, 0.0, 0.0, 0.0]; let solid = |color: [f32; 4]| LayerCB { dst, quad_px, radius_px, mode: 1.0, color, + layer_fx, ..Default::default() }; match bg { @@ -1192,6 +1383,7 @@ impl Compositor { quad_px, radius_px, fx: [dir[0], dir[1], 0.0, 0.0], + layer_fx, ..crate::frame_geometry::gradient_layer(stops, offsets, BLACK) }); } @@ -1201,7 +1393,9 @@ impl Compositor { let aspect = if quad_px[1] > 0.0 { quad_px[0] / quad_px[1] } else { 1.0 }; let still = WallpaperMotion::None; if let Err(e) = - self.draw_image_in(path, dst, quad_px, radius_px, aspect, still, 0.0) + self.draw_image_in( + path, dst, quad_px, radius_px, aspect, still, 0.0, transparency, + ) { eprintln!("[compositor] fond webcam \"{}\" : {:#}", path, e); self.draw_solid(&solid(BLACK)); @@ -1661,7 +1855,7 @@ impl Compositor { /// Curseur custom (dot+ring) centré en `center` (0..1 sortie), taille `size_px`, opacité `a`. /// `clip` = rect "Clip to canvas" en espace sortie [x,y,w,h] ; passer un rect englobant tout /// (ex. [-1,-1,3,3]) pour désactiver l'effet. - unsafe fn draw_cursor(&self, center: [f32; 2], size_px: f32, a: f32, clip: [f32; 4]) { + unsafe fn draw_cursor(&self, center: [f32; 2], size_px: f32, a: f32, clip: [f32; 4]) -> [f32; 4] { let w = size_px / self.rw(); let h = size_px / self.rh(); let dst = [center[0] - w * 0.5, center[1] - h * 0.5, w, h]; @@ -1673,6 +1867,7 @@ impl Compositor { fx: clip, ..Default::default() }); + dst } /// Curseur thème (sprite PNG, ex. arrow.png) dont le PIVOT `hotspot` (fraction 0..1 de @@ -1690,7 +1885,7 @@ impl Compositor { sprite: &SceneCursorSprite, clip: [f32; 4], model: Option, - ) -> Result<()> { + ) -> Result> { let path = sprite.path.as_str(); let (srv, iw, ih) = self.cached_image(path)?; // Sans champ de distance, repli sur le sprite plat plutôt que sur le curseur math. @@ -1699,16 +1894,16 @@ impl Compositor { match self.cursor_sdf(path) { Ok((sdf, shape)) => { let shape = crate::frame_geometry::model_shape(sprite, shape); - if let Some(cb) = crate::frame_geometry::cursor_model_cb( + let Some(cb) = crate::frame_geometry::cursor_model_cb( placement, size_px, pose, shape, a, clip, - ) { - self.upload_cb(&cb); - self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); - self.ctx.PSSetShaderResources(4, Some(&[Some(sdf)])); - self.ctx.Draw(4, 0); - self.ctx.PSSetShaderResources(4, Some(&[None])); - } - return Ok(()); + ) else { + return Ok(None); + }; + self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); + self.ctx.PSSetShaderResources(4, Some(&[Some(sdf)])); + self.draw_layer(&cb); + self.ctx.PSSetShaderResources(4, Some(&[None])); + return Ok(Some(cb.dst)); } Err(e) => eprintln!("[curseur] champ de \"{path}\" : {e:#}"), } @@ -1723,10 +1918,9 @@ impl Compositor { clip, [self.rw(), self.rh()], ); - self.upload_cb(&cb); self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); - self.ctx.Draw(4, 0); - Ok(()) + self.draw_layer(&cb); + Ok(Some(cb.dst)) } /// Sprite de l'état courant (`cursor_type`, ex. `"text"`), à défaut celui de la flèche, @@ -1746,17 +1940,17 @@ impl Compositor { a: f32, clip: [f32; 4], model: Option, - ) { + ) -> Option<[f32; 4]> { let sprite = cursor_type.and_then(|t| sprites.get(t)).or_else(|| sprites.get("arrow")); if let Some(sprite) = sprite { - if self.draw_cursor_sprite(placement, size_px, a, sprite, clip, model).is_ok() { - return; + if let Ok(dst) = self.draw_cursor_sprite(placement, size_px, a, sprite, clip, model) { + return dst; } } // Le repli math reste droit même sur un plan incliné : il ne devrait plus apparaître // maintenant que l'art par défaut existe, et lui donner sa propre passe de warp pour // un cas de secours ne se justifie pas. - self.draw_cursor(placement.upright_center(), size_px, a, clip); + Some(self.draw_cursor(placement.upright_center(), size_px, a, clip)) } /// Ombre portée (§7 E4) sous un quad `dst` (normalisé) de taille `size_px`. @@ -1850,6 +2044,11 @@ impl Compositor { }); } + /// Le métrage dans la dernière image composée : ses coins et son warp (`FootageQuad`). + pub fn footage_quad(&self) -> Option { + self.footage.get() + } + /// Compose une frame animée (§6/§8) : fond flouté + screen zoomé (padding, coins, ombre) /// + webcam crop carré (coins, ombre), placements interpolés A↔B par la timeline. pub unsafe fn compose_frame( @@ -1893,8 +2092,10 @@ impl Compositor { timeline_t_override: *self.timeline_t_override.borrow(), programme_time: *self.programme_time.borrow(), }); + self.footage.set(Some(g.footage_quad([self.rw(), self.rh()]))); + // The extra cameras' copies, made before anything is drawn like the two above. + let extra_cams = self.extra_camera_srvs(&g.camera_layers); let scene_preset = g.scene_preset.clone(); - let mb_taps = g.mb_taps; let mb_amount = g.mb_amount; let source_t = g.source_t; let _padding_scale = g.padding_scale; @@ -1905,7 +2106,6 @@ impl Compositor { let s_radius = g.s_radius; let frame_min_px = g.frame_min_px; let w_dst = g.w_dst; - let w_dst_prev = g.w_dst_prev; let w_px = g.w_px; let w_radius = g.w_radius; let shape_fade = g.shape_fade; @@ -1915,8 +2115,6 @@ impl Compositor { let dof = g.depth_of_field_on(self.cpu_backend); let dof_srv = if dof { Some(self.fill_dof_pyramid(&sy, &suv, stw, sth)?) } else { None }; - self.begin([0.0, 0.0, 0.0, 1.0]); - // --- fond --- // Parité web (frameRenderer.blurredBackgroundLayer) : le fond est le WALLPAPER sélectionné // (image/couleur/gradient) et « Blur BG » floute CE wallpaper, PAS la vidéo. Le natif @@ -1925,86 +2123,120 @@ impl Compositor { // ensuite ; pour une couleur plate le flou est un no-op visuel). Côté fixture/bench // (pas de scène) on garde le fond screen-flouté, dont le coût est mesuré (C4). let scene_bg = self.scene.borrow().as_ref().map(|s| (s.background.clone(), s.effects.blur)); - if let Some((bg, blur_wallpaper)) = scene_bg { - match bg { - SceneBackground::Color { color } => { - let c = parse_hex(&color).unwrap_or(lp.bg_color); - self.draw_solid(&LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - mode: 1.0, - color: c, - ..Default::default() - }); - } - SceneBackground::Gradient { angle_deg, stops, offsets, motion } => { - // angle CSS → direction unitaire (espace sortie, y vers le bas) : - // 0° = vers le haut, 90° = vers la droite. - let a = angle_deg.to_radians(); - let dir = [a.sin(), -a.cos()]; - let (anim, mb) = crate::frame_geometry::wallpaper_motion_slots( - motion, - g.programme_t, - self.rw() / self.rh(), - ); - self.draw_solid(&LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - fx: [dir[0], dir[1], anim[0], anim[1]], - mb, - ..crate::frame_geometry::gradient_layer(&stops, &offsets, lp.bg_color) - }); - } - SceneBackground::Image { path, motion } => { - // image bg (cover-fit, mise en cache) ; fallback couleur si chargement échoue - // (loggé — un fallback silencieux masquerait un chemin cassé, cf. le panic - // borrow qu'on a déjà eu : toute panne doit être visible/traçable). - let aspect = self.rw() / self.rh(); - if let Err(e) = self.draw_image_bg(&path, aspect, motion, g.programme_t) { - eprintln!("[compositor] wallpaper image \"{}\" : {:#}", path, e); + // Le fond de la frame d'avant, s'il est celui qu'on demande (`BackgroundKey`), est recopié + // au lieu d'être redessiné. Côté scène seulement : sans scène, le fond est la vidéo floutée. + let bg_size = [self.rw(), self.rh()]; + let bg_cached = scene_ref.as_ref().is_some_and(|s| { + self.bg_cache + .borrow() + .as_ref() + .is_some_and(|c| { + c.key.matches(Some(&s.background), s.effects.blur, lp.bg_color, bg_size) + }) + }); + if bg_cached { + self.bind_compose_state(); + if let Some(c) = self.bg_cache.borrow().as_ref() { + self.ctx.CopyResource(&self.rt, &c.tex); + } + } else { + self.begin([0.0, 0.0, 0.0, 1.0]); + // Faux si l'image du fond n'a pas pu se charger : son repli n'est pas gardé. + let mut bg_drawn = true; + if let Some((bg, blur_wallpaper)) = scene_bg { + match bg { + SceneBackground::Color { color } => { + let c = parse_hex(&color).unwrap_or(lp.bg_color); self.draw_solid(&LayerCB { dst: [0.0, 0.0, 1.0, 1.0], mode: 1.0, - color: lp.bg_color, + color: c, ..Default::default() }); } + SceneBackground::Gradient { angle_deg, stops, offsets, motion } => { + // angle CSS → direction unitaire (espace sortie, y vers le bas) : + // 0° = vers le haut, 90° = vers la droite. + let a = angle_deg.to_radians(); + let dir = [a.sin(), -a.cos()]; + let (anim, mb) = crate::frame_geometry::wallpaper_motion_slots( + motion, + g.programme_t, + self.rw() / self.rh(), + ); + self.draw_solid(&LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + fx: [dir[0], dir[1], anim[0], anim[1]], + mb, + ..crate::frame_geometry::gradient_layer(&stops, &offsets, lp.bg_color) + }); + } + SceneBackground::Image { path, motion } => { + // image bg (cover-fit, mise en cache) ; fallback couleur si chargement échoue + // (loggé — un fallback silencieux masquerait un chemin cassé, cf. le panic + // borrow qu'on a déjà eu : toute panne doit être visible/traçable). + let aspect = self.rw() / self.rh(); + if let Err(e) = self.draw_image_bg(&path, aspect, motion, g.programme_t) { + eprintln!("[compositor] wallpaper image \"{}\" : {:#}", path, e); + bg_drawn = false; + self.draw_solid(&LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + mode: 1.0, + color: lp.bg_color, + ..Default::default() + }); + } + } } - } - // « Blur BG » (parité web blurredBackgroundLayer) : floute CE wallpaper qu'on vient - // de dessiner (dual-Kawase, déjà utilisé pour le fond fixture ci-dessous). No-op - // visuel sur une couleur plate, effet réel sur gradient/image. - if blur_wallpaper > 0.0 { - self.blur_bg(blur_wallpaper); + // « Blur BG » (parité web blurredBackgroundLayer) : floute CE wallpaper qu'on vient + // de dessiner (dual-Kawase, déjà utilisé pour le fond fixture ci-dessous). No-op + // visuel sur une couleur plate, effet réel sur gradient/image. + if blur_wallpaper > 0.0 { + self.blur_bg(blur_wallpaper); + self.bind_compose_state(); + } + } else if cfg.bg_blur > 0.0 { + let over = 0.06; + self.draw_video( + &LayerCB { + dst: [-over, -over, 1.0 + 2.0 * over, 1.0 + 2.0 * over], + src: [0.0, 0.0, u_max, v_max], + quad_px: [self.rw(), self.rh()], + mode: 0.0, + color: [1.0, 1.0, 1.0, 1.0], + ..Default::default() + }, + &sy, + &suv, + ); + self.blur_bg(cfg.bg_blur); self.bind_compose_state(); - } - } else if cfg.bg_blur > 0.0 { - let over = 0.06; - self.draw_video( - &LayerCB { - dst: [-over, -over, 1.0 + 2.0 * over, 1.0 + 2.0 * over], - src: [0.0, 0.0, u_max, v_max], - quad_px: [self.rw(), self.rh()], - mode: 0.0, - color: [1.0, 1.0, 1.0, 1.0], + self.draw_solid(&LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + mode: 1.0, + color: [0.0, 0.0, 0.0, 0.35], ..Default::default() - }, - &sy, - &suv, - ); - self.blur_bg(cfg.bg_blur); - self.bind_compose_state(); - self.draw_solid(&LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - mode: 1.0, - color: [0.0, 0.0, 0.0, 0.35], - ..Default::default() - }); - } else { - self.draw_solid(&LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - mode: 1.0, - color: lp.bg_color, - ..Default::default() + }); + } else { + self.draw_solid(&LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + mode: 1.0, + color: lp.bg_color, + ..Default::default() + }); + } + // Pas de cache pour un fond animé (`of` rend `None`). + let key = scene_ref.as_ref().filter(|_| bg_drawn).and_then(|s| { + crate::frame_geometry::BackgroundKey::of( + Some(&s.background), + s.effects.blur, + lp.bg_color, + bg_size, + ) }); + if let Some(key) = key { + self.keep_background(key)?; + } } // --- screen : crop du clip actif, puis zoom appliqué dans ce rect source (§8) --- @@ -2138,6 +2370,11 @@ impl Compositor { self.ctx.PSSetShaderResources(5, Some(&[None])); } + // --- flous de confidentialité : sur le métrage, AVANT le curseur. Le curseur reste net + // par-dessus, et le cristal de Prism Glow, qui réfracte l'image composée, n'y voit que des + // pixels déjà floutés. Les autres annotations restent le calque le plus haut (plus bas). + self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, true); + // --- curseur custom : suit le mapping src/dst (zoom+layout), click bounce, // et flou de mouvement (parité `compositor_macos.rs` et `compositor_linux.rs`) --- if let Some(track) = cursor_ref.as_ref() { @@ -2161,6 +2398,13 @@ impl Compositor { .map(|s| s.cursor.cursor_sprites.clone()) .unwrap_or_default(); let cursor_type = plan.cursor_type.as_deref(); + // Le cristal de Prism Glow réfracte l'image telle qu'elle est composée à cet + // instant, flous compris : sa copie en t5, une fois pour toutes les copies de la + // traînée (`ann_copy` est libre, les flous l'ont déjà lue). + if plan.glass { + self.ctx.CopySubresourceRegion(&self.ann_copy, 0, 0, 0, 0, &self.rt, 0, None); + self.ctx.PSSetShaderResources(5, Some(&[Some(self.ann_copy_srv.clone())])); + } // L'impact des clics (mode 16, sans texture), posé sur l'écran SOUS le curseur. for cb in &plan.impacts { self.draw_solid(cb); @@ -2180,11 +2424,14 @@ impl Compositor { // buffer ISOLÉ (transparent), pas directement sur la scène déjà composée. self.ctx.ClearRenderTargetView(&self.accum_rtv, &[0.0, 0.0, 0.0, 0.0]); self.ctx.OMSetRenderTargets(Some(&[Some(self.accum_rtv.clone())]), None); + // L'union des quads peints dans `accum` (x0, y0, x1, y1 en fractions de la + // sortie) : la seule région que la recopie doit relire. + let mut bounds = [f32::MAX, f32::MAX, f32::MIN, f32::MIN]; for k in 0..plan.taps { let f = k as f32 / (plan.taps - 1) as f32; let w = crate::frame_geometry::cursor_tap_weight(k, plan.taps); self.ctx.OMSetBlendState(&self.blend_add, Some(&[w, w, w, w]), 0xffffffff); - self.draw_cur_themed( + let drawn = self.draw_cur_themed( &cursor_sprites, cursor_type, plan.prev_placement.lerp(plan.placement, f), @@ -2193,6 +2440,14 @@ impl Compositor { plan.clip, plan.model, ); + if let Some(d) = drawn { + bounds = [ + bounds[0].min(d[0]), + bounds[1].min(d[1]), + bounds[2].max(d[0] + d[2]), + bounds[3].max(d[1] + d[3]), + ]; + } } // composite le buffer accumulé sur la scène (blend "over" normal, prémultiplié). self.ctx.OMSetRenderTargets(Some(&[Some(self.rtv.clone())]), None); @@ -2210,12 +2465,30 @@ impl Compositor { }; self.ctx.RSSetViewports(Some(&[vp])); self.ctx.OMSetBlendState(&self.blend, None, 0xffffffff); - self.ctx.Draw(3, 0); + // Rien n'a été peint dans `accum` hors des quads de la traînée : la recopie + // s'y borne au lieu de repasser sur toute la sortie (0,5 ms par frame 1080p + // mesurée sur Linux, Radeon 610M). + let (rw, rh) = (self.rw(), self.rh()); + let rect = RECT { + left: (bounds[0] * rw).floor().clamp(0.0, rw) as i32, + top: (bounds[1] * rh).floor().clamp(0.0, rh) as i32, + right: (bounds[2] * rw).ceil().clamp(0.0, rw) as i32, + bottom: (bounds[3] * rh).ceil().clamp(0.0, rh) as i32, + }; + if rect.right > rect.left && rect.bottom > rect.top { + self.ctx.RSSetState(&self.rs_scissor); + self.ctx.RSSetScissorRects(Some(&[rect])); + self.ctx.Draw(3, 0); + self.ctx.RSSetState(None); + } self.ctx.PSSetShaderResources(0, Some(&[None])); // restaure l'état de composition standard (VS/PS/topologie quad-strip) pour // le dessin de la webcam qui suit juste après. self.bind_compose_state(); } + if plan.glass { + self.ctx.PSSetShaderResources(5, Some(&[None])); + } } } @@ -2233,99 +2506,144 @@ impl Compositor { // // Le center-crop carré de square/circle en est un cas particulier (boîte 1:1) — il n'a // plus besoin d'être traité à part. - let [su0, sv0, su1, sv1] = crate::frame_geometry::webcam_source_rect( - [wcw, wch], - [wtw as f32, wth as f32], - scene_ref - .as_ref() - .and_then(|scene| scene.layout.webcam_crop), - w_px[0] / w_px[1].max(0.0001), + // Camera 0's crop, mirror and desk turn as camera settings, plus its homography + // (`Scene::camera(0)`): with one, `src` is not read and the crop is dropped. + let cam0 = g.camera0_settings(scene_ref.as_ref()); + // L'ombre portée appartient à la bulle flottante PiP : elle se retire avec elle + // (`shape_fade`), pour qu'au plein écran plus rien n'encadre la caméra. C'est une + // ombre légère NON paramétrable — indépendante du slider Shadow, qui ne pilote plus + // que l'écran (`WEBCAM_SHADOW_OPACITY`, pas `shadow_scale`) — et propre au PiP : les + // blocs side-by-side / top-bottom soudent la caméra à l'écran et n'en portent aucune + // (parité `preset.shadow` web, `null` hors PiP dans `compositeLayout.ts`). + let webcam_is_block = matches!( + scene_preset.as_deref(), + Some("dual-frame") | Some("vertical-stack"), ); - // miroir = échanger les bornes u du rect source (flip horizontal). - let (u0, u1) = if lp.webcam_mirror { (su1, su0) } else { (su0, su1) }; - if lp.has_webcam { - // L'ombre portée appartient à la bulle flottante PiP : elle se retire avec elle - // (`shape_fade`), pour qu'au plein écran plus rien n'encadre la caméra. C'est une - // ombre légère NON paramétrable — indépendante du slider Shadow, qui ne pilote plus - // que l'écran (`WEBCAM_SHADOW_OPACITY`, pas `shadow_scale`) — et propre au PiP : les - // blocs side-by-side / top-bottom soudent la caméra à l'écran et n'en portent aucune - // (parité `preset.shadow` web, `null` hors PiP dans `compositeLayout.ts`). - let webcam_is_block = matches!( - scene_preset.as_deref(), - Some("dual-frame") | Some("vertical-stack"), - ); - // L'ombre appartient à la bulle PiP. En détourage il n'y a plus de bulle — une - // ombre portée par un rectangle invisible se lit comme un artefact. - let is_cutout = matches!( - scene_ref.as_ref().and_then(|s| s.webcam_effect.as_ref()), - Some(e) if e.shader_code() == 1.0 - ) && self.webcam_mask.borrow().is_some(); - if cfg.shadow && !webcam_is_block && !is_cutout && shape_fade > 0.0 { - let strength = WEBCAM_SHADOW_OPACITY * shape_fade; - self.draw_shadow( - w_dst, - w_px, - w_radius, - WEBCAM_SHADOW_SPREAD_FRAC * frame_min_px, - [0.0, WEBCAM_SHADOW_OFFSET_FRAC * frame_min_px], - strength, - ); - } - // Effet d'arrière-plan : le mode vient de la scène, le masque par pixel de - // l'inférence. Les DEUX sont requis — un mode sans masque rendrait la webcam - // invisible en détourage, donc tant que rien n'a été segmenté on dessine la piste - // telle quelle. C'est aussi ce qui rend le premier lancement gracieux. - let mask = self.webcam_mask.borrow(); - let effect = scene_ref - .as_ref() - .and_then(|s| s.webcam_effect.as_ref()) - .filter(|_| mask.is_some()) - .map(|e| (e.shader_code(), e)) - .filter(|(code, _)| *code > 0.0); - - // Fond personnalisé : on PEINT le fond dans la bulle, puis on y découpe la caméra - // par-dessus — le mélange alpha donne `lerp(fond, caméra, personne)`, soit exactement - // ce que la branche « mode 3 » du shader calculait, mais pour les TROIS sortes de - // fond. Le shader ne sait peindre qu'une couleur plate sous le masque ; dégradés et - // images y tombaient sur du noir, et le défaut EST une image. - let (effect_code, blur_intensity) = match effect { - Some((code, e)) if code > 2.5 => { - self.draw_webcam_bg(e.background.as_ref(), w_dst, w_px, w_radius); - (1.0, 0.0) - } - Some((code, e)) => (code, e.blur_intensity.clamp(0.0, 1.0)), - None => (0.0, 0.0), - }; - + // L'ombre appartient à la bulle PiP. En détourage il n'y a plus de bulle — une + // ombre portée par un rectangle invisible se lit comme un artefact. + let is_cutout = matches!( + scene_ref.as_ref().and_then(|s| s.webcam_effect.as_ref()), + Some(e) if e.shader_code() == 1.0 + ) && self.webcam_mask.borrow().is_some(); + // Effet d'arrière-plan : le mode vient de la scène, le masque par pixel de + // l'inférence. Les DEUX sont requis — un mode sans masque rendrait la webcam + // invisible en détourage, donc tant que rien n'a été segmenté on dessine la piste + // telle quelle. C'est aussi ce qui rend le premier lancement gracieux. + let mask = self.webcam_mask.borrow(); + let effect = scene_ref + .as_ref() + .and_then(|s| s.webcam_effect.as_ref()) + .filter(|_| mask.is_some()) + .map(|e| (e.shader_code(), e)) + .filter(|(code, _)| *code > 0.0); + // Fond personnalisé : on PEINT le fond dans la bulle, puis on y découpe la caméra + // par-dessus — le mélange alpha donne `lerp(fond, caméra, personne)`, soit exactement + // ce que la branche « mode 3 » du shader calculait, mais pour les TROIS sortes de + // fond. Le shader ne sait peindre qu'une couleur plate sous le masque ; dégradés et + // images y tombaient sur du noir, et le défaut EST une image. + // `custom_bg` = Some(background) in that mode; it is painted between shadow and camera. + let (effect_code, blur_intensity, custom_bg) = match effect { + Some((code, e)) if code > 2.5 => (1.0, 0.0, Some(e.background.as_ref())), + Some((code, e)) => (code, e.blur_intensity.clamp(0.0, 1.0), None), + None => (0.0, 0.0, None), + }; + // Camera 0's video draw: the mask, bound to slot 3 for this draw only. `draw_video` ne + // lie que les slots 0-1, donc le masque posé ici tient pour l'appel qui suit. Il est + // délié juste après pour ne pas fuir sur les calques d'annotation, qui utilisent eux + // aussi le slot 2 et au-delà. + let draw_camera0 = |cb: &LayerCB| { if let Some(m) = mask.as_ref() { - // `draw_video` ne lie que les slots 0-1, donc le masque posé ici tient pour - // l'appel qui suit. Il est délié juste après pour ne pas fuir sur les calques - // d'annotation, qui utilisent eux aussi le slot 2 et au-delà. self.ctx.PSSetShaderResources(3, Some(&[Some(m.srv.clone())])); } - self.draw_video( - &LayerCB { - dst: w_dst, - src: [u0, sv0, u1, sv1], - quad_px: w_px, - radius_px: w_radius, - mode: 0.0, - // `color.a` porte l'alpha du découpage (`color.a * personne`) ; le RGB n'est - // plus lu, le fond ayant déjà été peint sous la caméra. - color: [0.0, 0.0, 0.0, 1.0], - fx: [w_valid[0], w_valid[1], effect_code, blur_intensity], - src_prev: [u0, sv0, u1, sv1], // src fixe (pas de zoom webcam) - dst_prev: w_dst_prev, - mb: [mb_taps, mb_amount, 1.0, 0.0], - ..Default::default() - }, - &wy, - &wuv, - ); + self.draw_video(cb, &wy, &wuv); if mask.is_some() { self.ctx.PSSetShaderResources(3, Some(&[None])); } + }; + + if g.camera_layers.is_empty() { + // Without layout regions: camera 0 from `w_dst`, as before the layers. + let [u0, v0, u1, v1] = crate::frame_geometry::camera_source_rect( + Some(&cam0), + [wcw, wch], + [wtw as f32, wth as f32], + w_px[0] / w_px[1].max(0.0001), + ); + if lp.has_webcam { + if cfg.shadow && !webcam_is_block && !is_cutout && shape_fade > 0.0 { + let strength = WEBCAM_SHADOW_OPACITY * shape_fade; + self.draw_shadow( + w_dst, + w_px, + w_radius, + WEBCAM_SHADOW_SPREAD_FRAC * frame_min_px, + [0.0, WEBCAM_SHADOW_OFFSET_FRAC * frame_min_px], + strength, + ); + } + if let Some(bg) = custom_bg { + self.draw_webcam_bg(bg, w_dst, w_px, w_radius, 0.0); + } + let cb = g.webcam_video_cb([u0, v0, u1, v1], w_valid, effect_code, blur_intensity); + draw_camera0(&crate::frame_geometry::with_camera_homography(cb, Some(&cam0))); + } + } else { + // Every planned layer, in order: shadow, then (camera 0) the custom background, + // then the video. Camera 0 keeps its frame, effects and mask; the extra cameras + // draw from `set_extra_camera_frames` without effects, and a missing one is skipped. + let render = [self.rw(), self.rh()]; + let cam0_base = g.webcam_video_cb([0.0; 4], w_valid, effect_code, blur_intensity); + for plan in &g.camera_layers { + if plan.camera == 0 { + if !lp.has_webcam { + continue; + } + let video = crate::frame_geometry::camera_layer_cb( + plan, + Some(&cam0), + [wcw, wch], + [wtw as f32, wth as f32], + render, + &cam0_base, + ); + if let Some(shadow) = + g.camera_layer_shadow(plan, &video, render, cfg.shadow, is_cutout) + { + self.draw_solid(&shadow); + } + if let Some(bg) = custom_bg { + let t = video.layer_fx[0]; + self.draw_webcam_bg(bg, video.dst, video.quad_px, video.radius_px, t); + } + draw_camera0(&video); + } else { + let Some(cam) = extra_cams.get(plan.camera - 1).and_then(|c| c.as_ref()) + else { + continue; + }; + let settings = scene_ref.as_ref().and_then(|s| s.camera(plan.camera)); + let valid = [ + cam.visible[0] / cam.tex[0].max(1.0), + cam.visible[1] / cam.tex[1].max(1.0), + ]; + let video = crate::frame_geometry::camera_layer_cb( + plan, + settings, + cam.visible, + cam.tex, + render, + &crate::frame_geometry::extra_camera_base_cb(valid), + ); + if let Some(shadow) = + g.camera_layer_shadow(plan, &video, render, cfg.shadow, false) + { + self.draw_solid(&shadow); + } + self.draw_video(&video, &cam.y, &cam.uv); + } + } } + drop(mask); // --- annotations : calque le plus haut, comme dans le DOM de la preview (le calque y est // monté après la vidéo). Ancrées sur `s_ann`, le rect ÉCRAN SANS ZOOM — c'est le conteneur @@ -2334,12 +2652,13 @@ impl Compositor { // la transform, donc les annotations restent en place pendant que le contenu zoome dessous. // Ce fut `s_dst` tant que le zoom vivait dans la coupe source ; depuis l'issue #179 il vit // dans la BOÎTE, et `s_dst` emmenait annotations et sous-titres avec lui. - // Exception : le flou de confidentialité suit le contenu (`FrameGeometry::privacy_mask`), - // d'où la géométrie entière passée en plus de `s_ann`. + // Les flous de confidentialité, eux, sont passés avant le curseur (plus haut) : ils suivent + // le contenu (`FrameGeometry::privacy_mask`), d'où la géométrie entière passée en plus de + // `s_ann`. // `source_t`, la même base de temps que les zoom/speed regions : le temps SOURCE du clip, // pas le compteur de frames. C'est ce qui garde une annotation alignée sur l'image quand // une speed region répète ou saute des frames. - self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g); + self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, false); Ok(()) } @@ -2351,19 +2670,23 @@ impl Compositor { /// sous-titres sous un zoom (issue #179, puis #397 sur Linux). L'arithmétique elle-même vit /// dans `frame_geometry::annotation_dst_in`, partagée par les trois backends. Le flou, lui, /// se place par `g.privacy_mask` : un masque doit rester sur ce qu'il cache. + /// + /// `privacy` : les flous de confidentialité seuls, dessinés sur le métrage avant le curseur ; + /// sinon toutes les autres annotations, le calque le plus haut. unsafe fn draw_annotations( &self, scene: Option<&Scene>, t: f32, s_ann: [f32; 4], g: &crate::frame_geometry::FrameGeometry, + privacy: bool, ) { let Some(scene) = scene else { return }; if scene.annotations.is_empty() { return; } let visible = |a: &crate::scene::SceneAnnotation| { - t >= a.start_sec as f32 && t < a.end_sec as f32 + t >= a.start_sec as f32 && t < a.end_sec as f32 && (a.kind == "blur") == privacy }; // Une seule recopie du render target pour TOUTES les annotations flou de la frame — leur // lecture doit voir l'image composée sans les flous eux-mêmes, sinon deux zones qui se @@ -2534,6 +2857,13 @@ impl Compositor { if text.content.trim().is_empty() { continue; } + // The desk label shows only while the camera is covered: skip the rest of + // its section before rasterizing anything. + if crate::text_anim::is_desk_cover(text.animation.as_deref()) + && g.webcam_cover <= 0.0 + { + continue; + } // `font_size_rel` est une fraction de la HAUTEUR DE LA BOÎTE D'ANCRAGE — rect // écran, ou cadre de sortie pour un sous-titre (cf. le contrat et // `annotationScale.ts`) : on la ramène en pixels de sortie ici, avec le même @@ -2579,10 +2909,13 @@ impl Compositor { // dispose ici) : dans une région accélérée, elle défile donc au rythme du // clip. À vitesse 1 — le cas de toutes les annotations existantes — c'est // exactement le timing de l'aperçu DOM. - let anim = crate::text_anim::text_animation_state( + // The desk label is the exception: its opacity is the camera cover, which + // runs on the screen clock (`annotation_text_state`). + let anim = crate::text_anim::annotation_text_state( text.animation.as_deref(), (t - annotation.start_sec as f32) * 1000.0, ((annotation.end_sec - annotation.start_sec) * 1000.0) as f32, + g.webcam_cover, ); // Les décalages sont donnés à la hauteur de référence : on les ramène à la // sortie, comme la taille de police, pour que l'animation ait la même @@ -2917,6 +3250,12 @@ impl Compositor { Ok((rw, rh, out)) } + /// Le RT de la dernière composition, à `render_size()`, pour qui le copie ailleurs sans + /// passer par la RAM : l'anneau de textures partagées de la preview (`shared_frames`). + pub fn render_target(&self) -> &ID3D11Texture2D { + &self.rt + } + /// Comme `rgb_to_nv12`, mais vers `target_w`×`target_h` : si la cible diffère de la taille /// de rendu, le RT composé est d'abord redimensionné (bilinéaire, `ps_tex`/`sampler` déjà /// utilisés partout ailleurs dans le fichier, cf. `blit_resized`). Le RT suivant la sortie, @@ -3162,6 +3501,9 @@ impl Compositor { /// (p.ex. après un export) pour ne pas retenir indéfiniment des textures de pool. pub fn clear_srv_cache(&self) { self.srv_cache.borrow_mut().clear(); + // The extra cameras' frames belong to the decoders just closed: forget them too, so + // a stale pointer is never read before the next `set_extra_camera_frames`. + self.extra_camera_frames.set([0; crate::camera_layers::MAX_EXTRA_CAMERAS]); } } @@ -3249,6 +3591,62 @@ mod tests { ); } + /// Un NV12 sans données, à la taille voulue : tout ce que `nv12_srvs` lit d'une frame. + fn nv12_frame(gpu: &crate::d3d::Gpu, w: u32, h: u32) -> (Box, ID3D11Texture2D) { + let desc = D3D11_TEXTURE2D_DESC { + Width: w, + Height: h, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_NV12, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + unsafe { + let mut tex: Option = None; + gpu.device.CreateTexture2D(&desc, None, Some(&mut tex)).expect("texture NV12"); + let tex = tex.expect("texture NV12"); + let mut frame: Box = Box::new(std::mem::zeroed()); + frame.data[0] = tex.as_raw() as *mut u8; + frame.data[1] = std::ptr::null_mut(); + (frame, tex) + } + } + + /// Un décodeur remplacé sans `clear_srv_cache` (changement de résolution en cours de flux, + /// `CpuFrames::ensure_tex`) peut donner à une texture NEUVE l'adresse d'une ancienne. La + /// copie privée gardée pour l'ancienne est alors trop petite, `CopySubresourceRegion` la + /// saute sans erreur et l'image gèle. On ne peut pas choisir l'adresse d'une texture ; + /// l'entrée de la petite est donc replacée sous la clé de la grande, l'état exact où le + /// cache se retrouve après un tel recyclage. + #[test] + fn a_larger_source_at_a_cached_address_gets_a_copy_of_its_own_size() { + let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else { + eprintln!("pas de device D3D11 — test sauté"); + return; + }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + let (small, _small_tex) = nv12_frame(&gpu, 64, 64); + let (large, _large_tex) = nv12_frame(&gpu, 128, 96); + let (small_key, large_key) = (small.data[0] as usize, large.data[0] as usize); + + unsafe { comp.nv12_srvs(&*small) }.expect("source de 64x64"); + { + let mut cache = comp.srv_cache.borrow_mut(); + let stale = cache.remove(&small_key).expect("entrée de la petite source"); + cache.insert(large_key, stale); + } + unsafe { comp.nv12_srvs(&*large) }.expect("source de 128x96"); + + let (dst, ..) = comp.srv_cache.borrow().get(&large_key).cloned().expect("entrée remplacée"); + let mut dd = D3D11_TEXTURE2D_DESC::default(); + unsafe { dst.GetDesc(&mut dd) }; + assert_eq!((dd.Width, dd.Height), (128, 96), "la copie a gardé la taille de l'ancienne texture"); + } + /// Un cran de padding en format Auto change la taille de rendu de la preview. `resized` /// réalloue les cibles et garde le reste : le masque webcam, et la boîte aux lettres où le /// worker de segmentation dépose les suivants. diff --git a/crates/compositor/src/cpu_frames_windows.rs b/crates/compositor/src/cpu_frames_windows.rs index 790f23a7f..ea0e921fb 100644 --- a/crates/compositor/src/cpu_frames_windows.rs +++ b/crates/compositor/src/cpu_frames_windows.rs @@ -41,8 +41,9 @@ pub(crate) struct CpuFrames { /// NV12 en mémoire système : la cible de swscale, la source de l'upload. nv12: *mut AVFrame, /// La texture NV12 échantillonnée par les shaders. UNE seule, réécrite à chaque - /// frame — le `srv_cache` du compositeur (clé `(ptr, slice)`) n'a donc qu'une entrée - /// et ne recrée jamais de SRV, contrairement au pool tournant de D3D11VA. + /// frame — le `srv_cache` du compositeur (clé : le pointeur de texture) n'a donc qu'une + /// entrée et ne recrée jamais de SRV, contrairement au pool tournant de D3D11VA. Quand + /// `ensure_tex` la réalloue à une autre taille, `nv12_srvs` voit l'écart et refait la copie. // ponytail: une seule texture = le CPU peut attendre que le GPU ait fini de lire la // frame précédente. Sur WARP tout est CPU et le pilote sérialise déjà ; si un backend // GPU réutilise ce chemin un jour et que le Map bloque, double-bufferiser ici. diff --git a/crates/compositor/src/extra_cameras.rs b/crates/compositor/src/extra_cameras.rs new file mode 100644 index 000000000..0b2833c84 --- /dev/null +++ b/crates/compositor/src/extra_cameras.rs @@ -0,0 +1,241 @@ +//! The extra cameras (k >= 1) a camera layout region draws, shared by the live preview +//! (`live.rs`) and the export walk (`timeline_walk.rs`). +//! +//! Index convention: camera 0 is the clip's `webcam`; camera k >= 1 is +//! `additional_cameras[k - 1]`. Only the cameras a non-empty layout region shows are opened, +//! and each is decoded only near those regions (`extra_camera_active`). Unlike camera 0, an +//! extra camera never shortens a clip and never holds its last picture: past its end its +//! layer disappears. + +use crate::camera_layers::MAX_EXTRA_CAMERAS; +use crate::ffi::AVFrame; +use crate::live::PREFETCH_LEAD_SEC; +use crate::pipeline::Decoder; +use crate::scene::{SceneCameraLayoutRegion, SceneClipCamera}; +use crate::timeline_walk::{frame_step, FrameStep, NextFrameTime}; +use anyhow::Result; +use std::collections::{HashMap, HashSet}; + +/// An extra camera of an export clip: its file and its offset +/// (camera source time = screen source time - `offset_sec`). +pub type ClipCamera = SceneClipCamera; + +/// What a clip opens for its extra cameras: index k-1 = camera k, `None` = not opened (no +/// layout region of the clip shows it, or the clip has no file for it). No extra camera at all +/// is the empty list. +pub(crate) type ExtraCameraKeys = Vec>; + +/// Re-seek an extra camera instead of decoding forward when its target is further ahead than +/// this (it sat idle between two regions). +pub(crate) const EXTRA_RESEEK_SEC: f64 = 1.0; + +/// Camera indices >= 1 that the non-empty `regions` draw, sorted, without duplicates. +pub(crate) fn cameras_in_regions(regions: &[SceneCameraLayoutRegion]) -> Vec { + let mut cameras: Vec = regions + .iter() + .filter(|r| r.end_sec > r.start_sec) + .flat_map(|r| r.layers.iter().map(|l| l.camera)) + .filter(|camera| (1..=MAX_EXTRA_CAMERAS).contains(camera)) + .collect(); + cameras.sort_unstable(); + cameras.dedup(); + cameras +} + +pub(crate) fn has_camera_file(camera: Option<&SceneClipCamera>) -> bool { + camera.is_some_and(|c| !c.path.trim().is_empty()) +} + +/// The slots to open for `cameras` from a clip's camera files (`sources[k-1]` = camera k). +pub(crate) fn extra_camera_keys(cameras: &[usize], sources: &[SceneClipCamera]) -> ExtraCameraKeys { + let mut keys: ExtraCameraKeys = (1..=MAX_EXTRA_CAMERAS) + .map(|k| { + let source = sources.get(k - 1); + if cameras.contains(&k) && has_camera_file(source) { + source.cloned() + } else { + None + } + }) + .collect(); + while keys.last().is_some_and(|k| k.is_none()) { + keys.pop(); + } + keys +} + +/// Is extra camera `camera` near one of its layout regions at screen source time `t` — from +/// `PREFETCH_LEAD_SEC` before the region to its end? Outside, its decoder stays idle. +pub(crate) fn extra_camera_active(regions: &[SceneCameraLayoutRegion], camera: usize, t: f64) -> bool { + regions.iter().any(|r| { + r.end_sec > r.start_sec + && r.layers.iter().any(|l| l.camera == camera) + && (r.start_sec - PREFETCH_LEAD_SEC..=r.end_sec).contains(&t) + }) +} + +/// A camera's source time at screen source time `screen_t` (`camera = screen - offset`), +/// never before the file's start. Camera 0 (`live.rs`) and the extra cameras share it. +pub(crate) fn camera_source_time(screen_t: f64, offset_sec: f64) -> f64 { + (screen_t - offset_sec).max(0.0) +} + +/// An extra camera's open result: a file that will not open is `None` with one warning line, +/// and its layer is skipped — never drawn from another source. +pub(crate) fn opened_or_skipped(path: &str, opened: Result) -> Option { + match opened { + Ok(dec) => Some(dec), + Err(e) => { + eprintln!("WARNING: extra camera unreadable ({path}): {e:#}. Its layer will not be drawn."); + None + } + } +} + +/// Opens `path` into `decs` once. A file that will not open is remembered in `unreadable` +/// and never tried again, so its layer is skipped for the whole export. `true` when `decs` +/// holds a decoder for `path`. +pub(crate) fn open_once( + decs: &mut HashMap, + unreadable: &mut HashSet, + path: &str, + open: impl FnOnce(&str) -> Result, +) -> bool { + if decs.contains_key(path) { + return true; + } + if unreadable.contains(path) { + return false; + } + match opened_or_skipped(path, open(path)) { + Some(dec) => { + decs.insert(path.to_string(), dec); + true + } + None => { + unreadable.insert(path.to_string()); + false + } + } +} + +/// The list `set_extra_camera_frames` takes: null for an empty slot, else `frame` of the camera +/// (itself null when it has nothing to show). +pub(crate) fn extra_frame_list( + extra: &[Option], + frame: impl Fn(&T) -> *const AVFrame, +) -> Vec<*const AVFrame> { + extra.iter().map(|slot| slot.as_ref().map_or(std::ptr::null(), &frame)).collect() +} + +/// Advances `dec` to `target` (camera source time) with the webcam's hold semantics +/// (`frame_step`); re-seeks after an idle stretch or a jump back. Null past the camera's last +/// frame. `ended_at` is the camera source time past which the file has no frame left: the +/// decoder is not asked again until the target moves back before it. +pub(crate) unsafe fn step_extra_camera( + dec: &mut Decoder, + ended_at: &mut Option, + target: f64, +) -> Result<*const AVFrame> { + if ended_at.is_some_and(|end| target >= end) { + return Ok(std::ptr::null()); + } + let frame_dur = 1.0 / dec.fps().max(1.0); + let cur = dec.cur_frame(); + let far = cur.is_null() || { + let t = dec.cur_time_sec(); + target < t - frame_dur * 0.5 || target > t + EXTRA_RESEEK_SEC + }; + let frame = if far { + dec.seek_to(target)? + } else { + let mut frame = cur; + let mut guard = 0u32; + loop { + let next = dec.peek_next_time_sec()?; + // Unlike camera 0, an extra camera does not hold its last picture: past its + // end (one frame's worth of slack) its layer disappears. + if matches!(next, NextFrameTime::Eof) && target > dec.cur_time_sec() + frame_dur { + frame = std::ptr::null_mut(); + break; + } + match frame_step(next, 0.0, target) { + FrameStep::Commit => frame = dec.commit_peek()?, + FrameStep::CommitAndStop => { + frame = dec.commit_peek()?; + break; + } + FrameStep::Hold => break, + } + guard += 1; + if guard > 1000 { + break; + } + } + frame + }; + Ok(settle(ended_at, frame, target)) +} + +/// Seeks `dec` to `target` (camera source time). Null past the camera's last frame. +pub(crate) unsafe fn seek_extra_camera( + dec: &mut Decoder, + ended_at: &mut Option, + target: f64, +) -> Result<*const AVFrame> { + if ended_at.is_some_and(|end| target >= end) { + return Ok(std::ptr::null()); + } + let frame = dec.seek_to(target)?; + Ok(settle(ended_at, frame, target)) +} + +fn settle(ended_at: &mut Option, frame: *mut AVFrame, target: f64) -> *const AVFrame { + *ended_at = if frame.is_null() { Some(target) } else { None }; + frame +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::anyhow; + + #[test] + fn export_skips_an_extra_camera_that_will_not_open() { + let mut decs: HashMap = HashMap::new(); + let mut unreadable = HashSet::new(); + let mut attempts = 0; + let mut refuse = |_: &str| -> Result { + attempts += 1; + Err(anyhow!("no such file")) + }; + assert!(!open_once(&mut decs, &mut unreadable, "/cam2.mp4", &mut refuse)); + // A later clip of the same export does not try the file again. + assert!(!open_once(&mut decs, &mut unreadable, "/cam2.mp4", &mut refuse)); + assert_eq!(attempts, 1); + assert!(decs.is_empty()); + + // A readable camera opens once and is reused by the next clip. + let mut opens = 0; + let mut accept = |_: &str| -> Result { + opens += 1; + Ok(7) + }; + assert!(open_once(&mut decs, &mut unreadable, "/cam3.mp4", &mut accept)); + assert!(open_once(&mut decs, &mut unreadable, "/cam3.mp4", &mut accept)); + assert_eq!(opens, 1); + assert_eq!(decs.get("/cam3.mp4"), Some(&7)); + } + + #[test] + fn extra_offsets_move_into_the_screen_clock() { + // camera + offset = screen: at offset 2 s, a camera frame at 0.5 s is due at 2.5 s + // of screen time, not before. + let held = camera_source_time(2.4, 2.0); + let due = camera_source_time(2.5, 2.0); + assert_eq!(frame_step(NextFrameTime::At(0.5), 0.0, held), FrameStep::Hold); + assert_eq!(frame_step(NextFrameTime::At(0.5), 0.0, due), FrameStep::Commit); + // Before the camera started, its source time clamps to its first frame. + assert_eq!(camera_source_time(1.0, 2.0), 0.0); + } +} diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index c788d92b8..9cba80d82 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -27,7 +27,7 @@ use crate::config::Cfg; use crate::scene::{Scene, SceneCrop}; -/// Constant buffer d'un calque : **176 octets**, un par draw. +/// Constant buffer d'un calque : **256 octets**, un par draw. /// /// C'est le contrat partagé par les trois côtés — `cbuffer Layer` dans `shaders.hlsl`, /// `struct Layer` dans `shaders.metal` et `vk_shaders/layer.wgsl`, et ce struct. Ils doivent @@ -35,13 +35,15 @@ use crate::scene::{Scene, SceneCrop}; /// produit un shader qui lit `color` là où on a écrit `fx`. /// /// `align(16)` vient de la version macOS ; sous `repr(C)` seul, les offsets sont déjà -/// 0/16/32/40/44/48/64/80/96/112/128/144/160 des deux côtés — l'alignement Rust ne change que +/// 0/16/32/40/44/48/64/80/96/112/128/144/160/176 des deux côtés — l'alignement Rust ne change que /// l'adresse du struct, pas son contenu, et Windows le `copy_nonoverlapping` dans un /// constant buffer mappé où l'alignement source est sans effet. Les deux formes étaient /// donc compatibles ; les unifier évite qu'elles cessent de l'être. /// /// (Le commentaire d'origine annonçait « 64 octets ». Il n'a jamais été juste : dix champs, -/// trente-deux `f32`. Les trois derniers, le flou de mouvement de l'écran incliné, en font 176.) +/// trente-deux `f32`. Les trois suivants, le flou de mouvement de l'écran incliné, en font 176, et `cover`, le voile de la vue bureau, 192.) +/// (The camera layers' homography `persp` and transparency `layer_fx` make it 256: offsets 192 +/// and 240.) #[repr(C, align(16))] #[derive(Clone, Copy, Default)] pub struct LayerCB { @@ -62,6 +64,28 @@ pub struct LayerCB { pub trail_a: [f32; 4], pub trail_b: [f32; 4], pub trail_mb: [f32; 4], + /// Desk-view cover of the webcam layer: x = strength 0..1, y = blur radius (quad px), + /// z = dim factor, w unused. Zero everywhere else. + pub cover: [f32; 4], + /// Camera (mode 0) only: the rows of the homography H that maps a point of the layer's + /// `dst` quad (0..1, top-left origin) to the camera frame (0..1 of its valid part): + /// `[h0, h1, h2, 0], [h3, h4, h5, 0], [h6, h7, h8, 0]`. Read only when `layer_fx.y` is 1; + /// `src` is then ignored. Every other mode ignores it. + pub persp: [[f32; 4]; 3], + /// x = transparency 0..1 (0 = opaque), applied to the output of every mode; y = 1 when + /// `persp` applies; z, w = 0. All zero = the layer draws exactly as before the lanes. + pub layer_fx: [f32; 4], +} + +impl LayerCB { + /// Le calque est l'un des modèles 3D — le curseur modelé (mode 15), l'impact de son clic + /// (16), l'appareil et son ombre (17) — que seule la variante « modèles » du shader de calque + /// compile. Dans le même shader que les autres, leurs branches fixaient l'occupation de TOUS + /// les calques (168 VGPR sur RADV) ; les trois backends décident donc du shader d'un calque + /// par ce test, au moment de le dessiner (cf. le haut de `vk_shaders/layer.wgsl`). + pub fn needs_models(&self) -> bool { + self.mode > 14.5 && self.mode < 17.5 + } } pub const OUT_W: u32 = 1920; @@ -319,6 +343,33 @@ pub(crate) fn cover_crop_uv(visible: [f32; 2], tex: [f32; 2], box_ar: f32) -> (f (u0, v0, u1, v1) } +/// How the webcam texture is laid onto its quad this frame. The backends swap the source +/// rect's u bounds for `flip_u` and its v bounds for `flip_v` — 180° is both — and drop the +/// webcam crop for `full_frame`. Derived per frame because a Full Camera region can turn +/// the camera for a desk shot. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct WebcamOrientation { + pub flip_u: bool, + pub flip_v: bool, + pub full_frame: bool, +} + +pub fn webcam_orientation( + region: Option<&crate::scene::SceneCameraFullscreenRegion>, + layout_mirror: bool, +) -> WebcamOrientation { + let Some(r) = region else { + return WebcamOrientation { flip_u: layout_mirror, flip_v: false, full_frame: false }; + }; + let turned = r.rotation == 180; + let mirror = r.mirror.unwrap_or(layout_mirror); + WebcamOrientation { + flip_u: mirror ^ turned, + flip_v: turned, + full_frame: turned && r.full_frame, + } +} + /// Camera equivalent of the screen crop pipeline: apply the user crop first, then a centred /// cover-crop inside that authored window so arbitrary layout slots never stretch the image. pub(crate) fn webcam_source_rect( @@ -335,6 +386,140 @@ pub(crate) fn webcam_source_rect( box_ar, ) } + +/// The camera's perspective correction, when it has one whose nine values are all finite. A +/// matrix with a NaN or an infinity would draw garbage (or nothing), so it counts as none. +pub(crate) fn camera_homography(camera: Option<&crate::scene::SceneCamera>) -> Option<[f32; 9]> { + camera + .and_then(|c| c.homography) + .filter(|h| h.iter().all(|v| v.is_finite())) +} + +/// The source rect a camera is drawn from: its crop, then a cover-crop to `box_ar`, then its +/// rotation (180° = both bounds swapped) and mirror (u bounds swapped), `flip_u = mirror ^ +/// turned` as `webcam_orientation` does. A camera without settings is drawn upright and +/// unmirrored. With a homography, crop, rotation and mirror are all ignored: the corner order +/// already defines the picture (the shader then does not read `src` at all). +pub(crate) fn camera_source_rect( + camera: Option<&crate::scene::SceneCamera>, + visible: [f32; 2], + tex: [f32; 2], + box_ar: f32, +) -> [f32; 4] { + let corrected = camera_homography(camera).is_some(); + let settings = camera.filter(|_| !corrected); + let [u0, v0, u1, v1] = webcam_source_rect(visible, tex, settings.and_then(|c| c.crop), box_ar); + let turned = settings.is_some_and(|c| c.rotation == 180); + let mirror = settings.and_then(|c| c.mirror).unwrap_or(false); + let (u0, u1) = if mirror ^ turned { (u1, u0) } else { (u0, u1) }; + let (v0, v1) = if turned { (v1, v0) } else { (v0, v1) }; + [u0, v0, u1, v1] +} + +/// The window of a corrected picture (`aspect` = its width/height) a box of ratio `box_ar` +/// shows, as `[u0, v0, du, dv]` of the picture's unit square: centred and cover-fitted, so the +/// picture is cropped, never stretched — what `cover_uv_rect` does for an uncorrected camera, +/// whose `src` the shader stops reading once a homography is set. The whole picture without a +/// usable aspect or box. +pub(crate) fn corrected_cover_window(aspect: Option, box_ar: f32) -> [f32; 4] { + let Some(aspect) = aspect.filter(|a| a.is_finite() && *a > 0.0) else { + return [0.0, 0.0, 1.0, 1.0]; + }; + if !(box_ar.is_finite() && box_ar > 0.0) { + return [0.0, 0.0, 1.0, 1.0]; + } + if box_ar > aspect { + let dv = aspect / box_ar; + [0.0, (1.0 - dv) * 0.5, 1.0, dv] + } else { + let du = box_ar / aspect; + [(1.0 - du) * 0.5, 0.0, du, 1.0] + } +} + +/// `cb` with the camera's homography in `persp` and `layer_fx.y = 1`; unchanged without one. +/// The matrix is composed with `corrected_cover_window` for the box `cb.quad_px`, so a box +/// whose ratio is not the camera's `aspect` (a frame-filling layer, a half of side-by-side) +/// crops the corrected picture instead of stretching it. +pub(crate) fn with_camera_homography( + mut cb: LayerCB, + camera: Option<&crate::scene::SceneCamera>, +) -> LayerCB { + if let Some(h) = camera_homography(camera) { + let box_ar = cb.quad_px[0] / cb.quad_px[1]; + let [u0, v0, du, dv] = corrected_cover_window(camera.and_then(|c| c.aspect), box_ar); + // H * C, with C = [[du, 0, u0], [0, dv, v0], [0, 0, 1]] taking box uv to picture uv. + let h: [f32; 9] = std::array::from_fn(|i| { + let (r, c) = (i / 3, i % 3); + match c { + 0 => h[r * 3] * du, + 1 => h[r * 3 + 1] * dv, + _ => h[r * 3] * u0 + h[r * 3 + 1] * v0 + h[r * 3 + 2], + } + }); + cb.persp = [ + [h[0], h[1], h[2], 0.0], + [h[3], h[4], h[5], 0.0], + [h[6], h[7], h[8], 0.0], + ]; + cb.layer_fx[1] = 1.0; + } + cb +} + +/// The video draw of an extra camera before `camera_layer_cb` places it: mode 0, no background +/// effect (`fx.zw = 0`), `valid` = the valid fraction of its decoder texture, no motion trail. +pub(crate) fn extra_camera_base_cb(valid: [f32; 2]) -> LayerCB { + LayerCB { + mode: 0.0, + color: [0.0, 0.0, 0.0, 1.0], + fx: [valid[0], valid[1], 0.0, 0.0], + mb: [1.0, 0.0, 1.0, 0.0], + ..Default::default() + } +} + +/// The video draw of one planned camera layer: `base` (camera 0's `webcam_video_cb`, or +/// `extra_camera_base_cb` for the others) moved to the plan's rect, with its corners, its +/// source rect (`camera_source_rect`), its homography and its transparency (`1 - opacity`). +/// Planned layers have no motion trail: `dst_prev = dst` and one tap. The exception is camera +/// 0's untouched default layer (the plan's rect is `base.dst`, fully opaque — what a frame +/// outside every layout region plans): it keeps `base`'s trail, as on today's path. +pub(crate) fn camera_layer_cb( + plan: &crate::camera_layers::CameraLayerPlan, + camera: Option<&crate::scene::SceneCamera>, + visible_px: [f32; 2], + tex_px: [f32; 2], + render: [f32; 2], + base: &LayerCB, +) -> LayerCB { + let quad_px = [plan.dst[2] * render[0], plan.dst[3] * render[1]]; + let min_px = quad_px[0].min(quad_px[1]); + let src = camera_source_rect(camera, visible_px, tex_px, quad_px[0] / quad_px[1].max(0.0001)); + let cover = base.cover[0]; + let untouched = plan.dst == base.dst && plan.opacity >= 1.0; + let (dst_prev, mb) = if untouched { + (base.dst_prev, base.mb) + } else { + (plan.dst, [1.0, 0.0, base.mb[2], base.mb[3]]) + }; + with_camera_homography( + LayerCB { + dst: plan.dst, + src, + quad_px, + radius_px: plan.radius_frac * min_px, + src_prev: src, + dst_prev, + mb, + cover: [cover, 0.04 * min_px * cover, base.cover[2], base.cover[3]], + persp: [[0.0; 4]; 3], + layer_fx: [1.0 - plan.opacity.clamp(0.0, 1.0), 0.0, 0.0, 0.0], + ..*base + }, + camera, + ) +} /// Rétrécit un rect SOURCE déjà exprimé en UV (`[u0, v0, u1, v1]`) autour de son /// centre pour qu'il porte le ratio `box_ar` une fois rapporté aux pixels de la /// texture. C'est la forme générale de `object-fit: cover`, et LA primitive qui @@ -1590,6 +1775,54 @@ fn same_source_path(a: &str, b: &str) -> bool { /// shader un temps borné, là où un `f32` de plusieurs heures perdrait la finesse du bruit. pub const WALLPAPER_MOTION_PERIOD_S: f32 = 120.0; +/// Ce qui décide à lui seul du fond composé — sa description, la force de son flou, la couleur +/// de repli, la taille de rendu —, gardé avec sa copie (`bg_cache`) : tant que le fond demandé +/// lui est égal, le backend recopie le fond de la frame d'avant au lieu de le redessiner, ce qui +/// coûtait plus que l'écran lui-même (1,65 ms par frame 1080p sur une Radeon 610M, un wallpaper +/// 4K échantillonné sans mips). +/// +/// La comparaison de chaque frame (`matches`) ne construit ni ne copie rien : une image en data +/// URI pèse plusieurs Mo. Seul `of`, au moment de garder un nouveau fond, la clone. +#[derive(Clone, PartialEq)] +pub struct BackgroundKey { + background: Option, + blur: f32, + fallback: [f32; 4], + render_px: [f32; 2], +} + +impl BackgroundKey { + /// La clé d'un fond fixe ; `None` pour un fond animé, qui change à chaque frame. + pub fn of( + background: Option<&crate::scene::SceneBackground>, + blur: f32, + fallback: [f32; 4], + render_px: [f32; 2], + ) -> Option { + use crate::scene::{SceneBackground as B, WallpaperMotion}; + if let Some(B::Gradient { motion, .. } | B::Image { motion, .. }) = background { + if *motion != WallpaperMotion::None { + return None; + } + } + Some(BackgroundKey { background: background.cloned(), blur, fallback, render_px }) + } + + /// Le fond demandé est celui-ci. + pub fn matches( + &self, + background: Option<&crate::scene::SceneBackground>, + blur: f32, + fallback: [f32; 4], + render_px: [f32; 2], + ) -> bool { + self.background.as_ref() == background + && self.blur == blur + && self.fallback == fallback + && self.render_px == render_px + } +} + /// Emplacements libres des modes 5 (dégradé) et 6 (image) pour le fond animé : `fx.zw` = (temps /// programme replié, indice du mouvement), `mb.x` = aspect w/h du rect rempli (les nappes de /// l'aurore restent rondes). @@ -1849,6 +2082,14 @@ pub struct FrameGeometry { pub w_px: [f32; 2], pub w_radius: f32, pub shape_fade: f32, + /// Orientation of the webcam this frame (mirror, desk-shot turn, crop bypass). + pub webcam: WebcamOrientation, + /// Desk-view cover strength of the webcam this frame, 0..1 (`camera_fullscreen_cover_at`). + pub webcam_cover: f32, + /// Every camera layer to draw this frame, in draw order (`camera_layers::camera_layers_at`). + /// Empty when the scene has no camera layout regions: the backends then draw camera 0 from + /// `w_dst` as before. Otherwise camera 0 is in here too, and its `dst`/opacity come from it. + pub camera_layers: Vec, /// Cadre autour de l'écran : chrome de fenêtre plat (mode 14) ou appareil modelé (mode 17). /// `None` : aucun, et le rendu est celui d'avant le cadre, à l'octet. `Some` : `s_dst` est /// déjà la boîte rétrécie, et `s_radius` le rayon des coins de l'écran — des seuls coins BAS @@ -2087,6 +2328,109 @@ impl FrameGeometry { self.tilt_trail(render_px).filter(|_| !self.screen_trail(render_px)) } + /// The camera-0 video draw (mode 0), the same on every backend. `src` = the source rect with + /// the mirror / desk turn already applied as swapped bounds (it is also `src_prev`: only + /// `dst_prev` carries the motion), `valid` = the valid fraction of the decoder texture, + /// `effect_code` / `blur_intensity` = the background effect sent to the shader. + pub fn webcam_video_cb( + &self, + src: [f32; 4], + valid: [f32; 2], + effect_code: f32, + blur_intensity: f32, + ) -> LayerCB { + LayerCB { + dst: self.w_dst, + src, + quad_px: self.w_px, + radius_px: self.w_radius, + mode: 0.0, + // `color.a` carries the cutout alpha (`color.a * person`); the RGB is not read, the + // background has already been painted under the camera. + color: [0.0, 0.0, 0.0, 1.0], + fx: [valid[0], valid[1], effect_code, blur_intensity], + src_prev: src, + dst_prev: self.w_dst_prev, + mb: [self.mb_taps, self.mb_amount, 1.0, 0.0], + cover: [ + self.webcam_cover, + 0.04 * self.w_px[0].min(self.w_px[1]) * self.webcam_cover, + 0.35, + 0.0, + ], + ..Default::default() + } + } + + /// Camera 0's settings as a `SceneCamera`, for `camera_source_rect` / `camera_layer_cb`: + /// its orientation is the project mirror and the desk-view turn (`self.webcam`, expressed so + /// that `mirror ^ turned` gives back `flip_u`), its crop is the layout's (none for a + /// full-frame desk shot), and only its homography and the corrected picture's aspect (so a + /// box of another ratio cover-crops it, `with_camera_homography`) come from + /// `Scene::camera(0)`. + pub(crate) fn camera0_settings(&self, scene: Option<&Scene>) -> crate::scene::SceneCamera { + let turned = self.webcam.flip_v; + let settings = scene.and_then(|s| s.camera(0)); + crate::scene::SceneCamera { + index: 0, + rotation: if turned { 180 } else { 0 }, + mirror: Some(self.webcam.flip_u ^ turned), + crop: if self.webcam.full_frame { + None + } else { + scene.and_then(|s| s.layout.webcam_crop) + }, + homography: settings.and_then(|c| c.homography), + aspect: settings.and_then(|c| c.aspect), + } + } + + /// The drop shadow under a planned camera layer, `video` being its `camera_layer_cb`. Same + /// constants as camera 0's PiP shadow, and it fades with the layer (`video.layer_fx.x`). + /// None when `shadows_on` is off or the layer fills the frame. Camera 0 also keeps today's + /// rules: no shadow in the block presets nor in cutout (`camera0_cutout`), and it leaves + /// with the bubble (`shape_fade`). + pub(crate) fn camera_layer_shadow( + &self, + plan: &crate::camera_layers::CameraLayerPlan, + video: &LayerCB, + render: [f32; 2], + shadows_on: bool, + camera0_cutout: bool, + ) -> Option { + let strength = if plan.camera == 0 { + let block = matches!( + self.scene_preset.as_deref(), + Some("dual-frame") | Some("vertical-stack") + ); + if block || camera0_cutout { + return None; + } + WEBCAM_SHADOW_OPACITY * self.shape_fade + } else { + WEBCAM_SHADOW_OPACITY + }; + if !shadows_on || plan.fills_frame || strength <= 0.0 { + return None; + } + let spread = WEBCAM_SHADOW_SPREAD_FRAC * self.frame_min_px; + let offset = WEBCAM_SHADOW_OFFSET_FRAC * self.frame_min_px; + let (sx, sy) = (spread / render[0].max(1.0), spread / render[1].max(1.0)); + let oy = offset / render[1].max(1.0); + let dst = video.dst; + Some(LayerCB { + dst: [dst[0] - sx, dst[1] - sy + oy, dst[2] + 2.0 * sx, dst[3] + 2.0 * sy], + quad_px: [video.quad_px[0] + 2.0 * spread, video.quad_px[1] + 2.0 * spread], + radius_px: video.radius_px, + mode: 2.0, + color: [0.0, 0.0, 0.0, strength], + fx: [spread, 0.0, 0.0, 0.0], + mb: [0.0, 1.0, 1.0, 0.0], + layer_fx: [video.layer_fx[0], 0.0, 0.0, 0.0], + ..Default::default() + }) + } + /// Le calque du mode 18 : le rendu isolé de l'écran cadré (t2), recomposé le long de sa /// trajectoire. `src` = la coupe (le métrage relu directement là où le rendu isolé s'arrête /// au bord de la sortie), `quad_px` et `radius_px` = l'écran et ses coins : ce repli ne lit le @@ -2658,10 +3002,6 @@ impl FrameGeometry { [centre[0] + px, centre[1] + py] }; let pts = [at(x0, y0), at(x1, y0), at(x1, y1), at(x0, y1)]; - // Le mode 8 étale aussi chaque pixel vers là où le plan était une frame plus tôt - // (`tilt_trail`) : une copie traînée du secret sort du quad courant. Le masque couvre alors - // la boîte englobante du secret aux deux frames, en rect droit, comme le chemin droit. - // Même warp que le shader sur les coins d'avant (le drapeau projectif est celui du plan). // La perspective grossit localement le côté proche : la force suit l'arête la plus // agrandie par rapport au rect droit zoomé, pour que nulle part le masque ne soit plus // fin, rapporté au contenu, qu'au repos. @@ -2672,28 +3012,29 @@ impl FrameGeometry { .max(len(pts[0], pts[3]) / eh) .max(len(pts[1], pts[2]) / eh); let strength = (zoom_k * tilt_k).max(1.0); - // Sans mouvement, la traînée EST le quad (au bit près) : rien de plus à couvrir. - if let Some(trail) = self.tilt_trail(render_px).filter(|t| t.corners != quad.corners) { - let before = crate::regions::TiltedQuad { corners: trail.corners, ..quad }; - let then = |fx: f32, fy: f32| { - let (px, py) = before.point_px(fx, fy); - [centre[0] + px, centre[1] + py] - }; - let (mut lo, mut hi) = ([f32::MAX; 2], [f32::MIN; 2]); - for [px, py] in pts.into_iter().chain([then(x0, y0), then(x1, y0), then(x1, y1), then(x0, y1)]) { - (lo, hi) = ([lo[0].min(px), lo[1].min(py)], [hi[0].max(px), hi[1].max(py)]); + // Le mode 8 étale aussi chaque pixel vers là où le plan était une frame plus tôt + // (`tilt_trail`) : une copie traînée du secret sort du quad courant. Le masque garde la + // perspective du quad courant, élargi de la plus grande avance d'un de ses coins : chaque + // coin d'avant tombe dedans, donc tout le secret d'avant aussi (un quad convexe contient + // l'enveloppe de ses coins), et la traînée, qui va de l'un à l'autre. Même warp que le + // shader sur les coins d'avant (le drapeau projectif est celui du plan). Sans mouvement, + // la traînée EST le quad (au bit près) : rien de plus à couvrir. + let trail = self.tilt_trail(render_px).filter(|t| t.corners != quad.corners); + let pts = match trail { + Some(trail) => { + let before = crate::regions::TiltedQuad { corners: trail.corners, ..quad }; + let then = |fx: f32, fy: f32| { + let (px, py) = before.point_px(fx, fy); + [centre[0] + px, centre[1] + py] + }; + let prev = [then(x0, y0), then(x1, y0), then(x1, y1), then(x0, y1)]; + let reach = (0..4) + .map(|i| (prev[i][0] - pts[i][0]).hypot(prev[i][1] - pts[i][1])) + .fold(0.0f32, f32::max); + grow_quad(pts, reach + 0.5) } - let rect = - [lo[0] / rw, lo[1] / rh, (hi[0] - lo[0]).max(1.0) / rw, (hi[1] - lo[1]).max(1.0) / rh]; - let rect = match self.screen_mask { - Some(m) => m.clip(rect)?, - None => rect, - }; - let mut mask = PrivacyMask::upright(rect, render_px, strength); - // Un ovale inscrit dans la boîte élargie ne couvrirait plus l'ovale courant. - mask.oval_ok = false; - return Some(mask); - } + None => pts, + }; let (mut min_x, mut min_y) = (f32::MAX, f32::MAX); let (mut max_x, mut max_y) = (f32::MIN, f32::MIN); for [px, py] in pts { @@ -2721,9 +3062,65 @@ impl FrameGeometry { warp: Some(local), projective: quad.projective, strength, - oval_ok: true, + // Un ovale inscrit dans le quad élargi ne couvrirait plus sûrement l'ovale d'avant. + oval_ok: trail.is_none(), }) } + + /// Où tombe le métrage dans l'image (`FootageQuad`) : là où `privacy_mask` pose un masque, + /// donc là où l'éditeur doit poser le gimbal d'un flou pour qu'il couvre ce qu'il cache. + pub fn footage_quad(&self, render_px: [f32; 2]) -> FootageQuad { + let [rw, rh] = render_px; + let s_px = [self.s_dst[2] * rw, self.s_dst[3] * rh]; + if let Some(quad) = self.screen_tilt(s_px).filter(|_| self.tilted() && rw > 0.0 && rh > 0.0) { + let centre = self.screen_center_px(render_px); + let at = |fx: f32, fy: f32| { + let (px, py) = quad.point_px(fx, fy); + [(centre[0] + px) / rw, (centre[1] + py) / rh] + }; + return FootageQuad { + corners: [at(0.0, 0.0), at(1.0, 0.0), at(1.0, 1.0), at(0.0, 1.0)], + projective: quad.projective, + }; + } + let [x, y, w, h] = self.s_dst; + FootageQuad { corners: [[x, y], [x + w, y], [x + w, y + h], [x, y + h]], projective: false } + } +} + +/// Le métrage dans l'image : ses coins TL, TR, BR, BL en fractions de la sortie, et la +/// correspondance d'un point `(u, v)` du métrage à l'image, bilinéaire entre eux ou, sous la +/// caméra réelle, l'homographie qu'ils définissent (`TiltedQuad::point_px`). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FootageQuad { + pub corners: [[f32; 2]; 4], + pub projective: bool, +} + +/// Le quad convexe `pts` (TL, TR, BR, BL) dont chaque côté recule de `d` px vers l'extérieur : +/// ses côtés restent parallèles aux siens, donc sa perspective aussi, et il contient tout point +/// à moins de `d` de lui. +fn grow_quad(pts: [[f32; 2]; 4], d: f32) -> [[f32; 2]; 4] { + // L'extérieur d'un côté est d'un bord ou de l'autre selon le sens de parcours. + let area: f32 = (0..4) + .map(|i| { + let (a, b) = (pts[i], pts[(i + 1) % 4]); + a[0] * b[1] - b[0] * a[1] + }) + .sum(); + let side = area.signum(); + let normal = |i: usize| { + let (a, b) = (pts[i], pts[(i + 1) % 4]); + let (ex, ey) = (b[0] - a[0], b[1] - a[1]); + let len = ex.hypot(ey).max(1e-6); + [side * ey / len, -side * ex / len] + }; + std::array::from_fn(|i| { + // Le coin glisse sur la bissectrice jusqu'à la rencontre des deux côtés reculés. + let (na, nb) = (normal((i + 3) % 4), normal(i)); + let k = d / (1.0 + na[0] * nb[0] + na[1] * nb[1]).max(0.2); + [pts[i][0] + (na[0] + nb[0]) * k, pts[i][1] + (na[1] + nb[1]) * k] + }) } /// Marge ajoutée autour d'un masque de confidentialité, en px de la boîte écran AU REPOS : un @@ -2873,6 +3270,9 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { let zoom_regions = scene.map(|s| &s.zoom_regions).unwrap_or(&empty_zoom); let cam_regions = scene.map(|s| &s.camera_fullscreen_regions).unwrap_or(&empty_cam); + // The layout regions, for the seams a Full Camera region shares with them. + let empty_layouts: Vec = Vec::new(); + let layouts = scene.map(|s| &s.camera_layout_regions).unwrap_or(&empty_layouts); let webcam_reactive = scene.map(|s| s.layout.webcam_reactive_zoom).unwrap_or(false); let source_t = input.timeline_t_override.unwrap_or(frame / FPS); // Les transitions se mesurent à l'écran (`ScreenClock`), la frame précédente aussi : une @@ -2891,6 +3291,8 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { let mut zoom_tilt = 0.0f32; let mut zoom_camera = 0.0f32; let mut zoom_camera_prev = 0.0f32; + let mut zoom_strength = 0.0f32; + let mut zoom_strength_prev = 0.0f32; let mut zoom_aim = [0.5f32; 2]; let mut zoom_orbit = [0.5f32; 2]; let (mut zoom_aim_prev, mut zoom_orbit_prev) = ([0.5f32; 2], [0.5f32; 2]); @@ -2919,6 +3321,7 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { zoom_rotation = zs.rotation; zoom_tilt = zs.tilt; zoom_camera = zs.camera; + zoom_strength = zs.strength; zoom_aim = zs.aim; zoom_orbit = zs.orbit; // Même `CameraFrame` qu'à la frame courante : la caméra réelle d'avant (poids, visée, @@ -2934,24 +3337,38 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { pp.focus = zs_p.focus; zoom_rotation_prev = zs_p.rotation; zoom_camera_prev = zs_p.camera; + zoom_strength_prev = zs_p.strength; zoom_aim_prev = zs_p.aim; zoom_orbit_prev = zs_p.orbit; } // Full Camera ignore le rétrécissement réactif de la webcam (design web : mélanger // "rétrécit pour le zoom" et "grandit en plein cadre" dans la même frame n'a pas de sens). let cam_progress = - crate::regions::camera_fullscreen_progress_at(cam_regions, source_t, &clock); + crate::regions::camera_fullscreen_progress_at(cam_regions, source_t, &clock, layouts); let cam_progress_prev = - crate::regions::camera_fullscreen_progress_at(cam_regions, source_t_prev, &clock); + crate::regions::camera_fullscreen_progress_at( + cam_regions, + source_t_prev, + &clock, + layouts, + ); let shape_fade = - crate::regions::camera_fullscreen_shape_at(cam_regions, source_t, &clock); - // rétrécissement réactif : la webcam rétrécit pendant un zoom actif, de la MOITIÉ de ce - // que prendrait 1/zoom : ×0,78 au zoom par défaut (1,8), ×0,6 au zoom maximal (5). En - // 1/zoom elle tombait à ×0,56 au zoom par défaut et un visage ne se lisait plus. Ignoré - // pendant Full Camera (voir ci-dessus). - let reactive_scale = |zoom: f32, progress: f32| -> f32 { - if webcam_reactive && progress <= 0.0 && zoom.is_finite() && zoom > 0.0 { - ((1.0 + 1.0 / zoom) / 2.0).min(1.0) + crate::regions::camera_fullscreen_shape_at(cam_regions, source_t, &clock, layouts); + let webcam = webcam_orientation( + crate::regions::camera_fullscreen_region_at(cam_regions, source_t, &clock, layouts), + lp.webcam_mirror, + ); + let webcam_cover = + crate::regions::camera_fullscreen_cover_at(cam_regions, source_t, &clock, layouts); + // rétrécissement réactif : la webcam garde 70 % de sa taille pendant un zoom actif, quel + // que soit son niveau (elle suivait 1/zoom, et rétrécissait donc d'autant plus que le zoom + // était profond : ×0,6 au zoom maximal). L'enveloppe est celle de la région : elle descend + // avec le ease-in du zoom, tient 0,7 sur le span, remonte avec le ease-out. Une région à + // échelle 1 ne zoome rien : elle laisse la caméra à sa taille. Ignoré pendant Full Camera + // (voir ci-dessus). + let reactive_scale = |strength: f32, zoom: f32, progress: f32| -> f32 { + if webcam_reactive && progress <= 0.0 && zoom > 1.0 && strength.is_finite() { + (1.0 - 0.3 * strength.clamp(0.0, 1.0)).clamp(0.7, 1.0) } else { 1.0 } @@ -2965,8 +3382,9 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { // Seul `reactive_scale` (rétrécissement pendant un zoom, une valeur ANIMÉE par frame que // le rect statique de l'app ne capture pas) doit encore s'appliquer dans ce cas. let base_size_scale = if app_webcam_rect.is_some() { 1.0 } else { lp.webcam_size_scale }; - let webcam_size_scale = base_size_scale * reactive_scale(p.zoom, cam_progress); - let webcam_size_scale_prev = base_size_scale * reactive_scale(pp.zoom, cam_progress_prev); + let webcam_size_scale = base_size_scale * reactive_scale(zoom_strength, p.zoom, cam_progress); + let webcam_size_scale_prev = + base_size_scale * reactive_scale(zoom_strength_prev, pp.zoom, cam_progress_prev); // padding : échelle globale du layout autour du centre du cadre (parité web frameRenderer : // paddingScale = 1 - padding*0.4 → padding 0 = plein cadre). S'applique à TOUS les presets : @@ -3354,6 +3772,30 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { _ => 0.12, }, }; + // Camera layout regions: every camera layer of this frame, camera 0 included, on the + // same time and clock as Full Camera. Camera 0's default is the layer just planned + // above, so a region glides from and back to exactly where `w_dst` puts it. Without + // regions nothing is planned, and the backends keep drawing camera 0 from `w_dst`. + let camera_layers = match scene { + Some(s) if !s.camera_layout_regions.is_empty() => { + let default_cam0 = lp.has_webcam.then(|| crate::camera_layers::CameraLayerPlan { + camera: 0, + dst: w_dst, + radius_frac: w_radius / w_px[0].min(w_px[1]).max(1.0), + shape: lp.webcam_shape, + opacity: 1.0, + fills_frame: cam_progress >= 1.0, + }); + crate::camera_layers::camera_layers_at( + &s.camera_layout_regions, + &s.camera_fullscreen_regions, + source_t, + &clock, + default_cam0, + ) + } + _ => Vec::new(), + }; FrameGeometry { scene_preset, @@ -3385,6 +3827,9 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { w_px, w_radius, shape_fade, + webcam, + webcam_cover, + camera_layers, window_frame, screen_mask, } @@ -3419,6 +3864,10 @@ pub struct CursorPlan { /// L'impact des clics récents sur l'écran (mode 16), à dessiner SOUS le curseur. Vide sans /// curseur modélisé, sans clic récent ou à `clickBounce` nul. pub impacts: Vec, + /// Le curseur modélisé est le cristal de Prism Glow : il réfracte l'image déjà composée, le + /// métrage et ses flous de confidentialité, que le backend lui copie juste avant de le + /// dessiner. Seuls des pixels floutés passent ainsi à travers le verre. + pub glass: bool, } impl CursorPlan { @@ -3720,6 +4169,7 @@ pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option Option, x: f32, y: f32| { let anchor = anchor.map(|a| format!(r#","webcamAnchor":"{a}""#)).unwrap_or_default(); let layout = format!( @@ -6490,19 +6985,56 @@ mod tests { plan_frame(&golden_input(&scene, &cfg)).w_dst }; for (anchor, x, y, want) in [ - (Some("bottom-right"), 0.77, 0.77, [0.82, 0.82]), + (Some("bottom-right"), 0.77, 0.77, [0.83, 0.83]), (Some("top-left"), 0.03, 0.03, [0.03, 0.03]), - (Some("bottom"), 0.4, 0.77, [0.425, 0.82]), - (Some("left"), 0.03, 0.4, [0.03, 0.425]), - (None, 0.77, 0.77, [0.795, 0.795]), + (Some("bottom"), 0.4, 0.77, [0.43, 0.83]), + (Some("left"), 0.03, 0.4, [0.03, 0.43]), + (None, 0.77, 0.77, [0.80, 0.80]), ] { let w = camera_at(anchor, x, y); let got = [w[0], w[1], w[2], w[3]]; - let want = [want[0], want[1], 0.15, 0.15]; + let want = [want[0], want[1], 0.14, 0.14]; assert!(got.iter().zip(want).all(|(g, w)| (g - w).abs() < 1e-5), "{anchor:?} : {got:?} au lieu de {want:?}"); } } + /// Le rétrécissement réactif vise 70 % QUEL QUE SOIT le niveau de zoom : la caméra garde la + /// même taille à 1,25×, 2× et 5× (elle suivait 1/zoom, et rétrécissait d'autant plus que le + /// zoom était profond). + #[test] + fn the_camera_shrink_is_the_same_at_every_zoom_level() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let camera_at = |scale: &str| { + let layout = r#""webcamReactiveZoom":true,"webcamAnchor":"bottom-right","webcamRect":{"x":0.77,"y":0.77,"width":0.2,"height":0.2}}"#; + let json = zoomed_golden_scene_json() + .replace(r#""webcamReactiveZoom":false}"#, layout) + .replace(r#""scale":2.0"#, scale); + let scene = Scene::from_json(&json).expect("scène"); + plan_frame(&golden_input(&scene, &cfg)).w_dst + }; + for scale in [r#""scale":1.25"#, r#""scale":2.0"#, r#""scale":5.0"#] { + let w = camera_at(scale); + let got = [w[0], w[1], w[2], w[3]]; + let want = [0.83, 0.83, 0.14, 0.14]; + assert!(got.iter().zip(want).all(|(g, w)| (g - w).abs() < 1e-5), "{scale} : {got:?} au lieu de {want:?}"); + } + } + + /// Une région de zoom à échelle 1 ne zoome rien : elle ne rétrécit pas la caméra. + #[test] + fn a_one_x_zoom_region_keeps_the_camera_at_its_size() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let layout = r#""webcamReactiveZoom":true,"webcamAnchor":"bottom-right","webcamRect":{"x":0.77,"y":0.77,"width":0.2,"height":0.2}}"#; + let json = zoomed_golden_scene_json() + .replace(r#""webcamReactiveZoom":false}"#, layout) + .replace(r#""scale":2.0"#, r#""scale":1.0"#); + let scene = Scene::from_json(&json).expect("scène"); + let w = plan_frame(&golden_input(&scene, &cfg)).w_dst; + let got = [w[0], w[1], w[2], w[3]]; + let want = [0.77, 0.77, 0.2, 0.2]; + assert!(got.iter().zip(want).all(|(g, w)| (g - w).abs() < 1e-5), "{got:?} au lieu de {want:?}"); + } + /// La vitesse se mesure dans la coupe VISIBLE, zoom compris : sous un x2, le même geste /// traverse deux fois plus d'écran et penche donc plus le plan (hors saturation). #[test] @@ -6839,6 +7371,87 @@ mod tests { } } + /// Le point `p` est dans le quad convexe `q` (TL, TR, BR, BL, px de sortie). + fn in_convex_quad(q: [[f32; 2]; 4], p: [f32; 2]) -> bool { + let side = |i: usize| { + let (a, b) = (q[i], q[(i + 1) % 4]); + (b[0] - a[0]) * (p[1] - a[1]) - (b[1] - a[1]) * (p[0] - a[0]) + }; + (0..4).all(|i| side(i) >= -1e-3) || (0..4).all(|i| side(i) <= 1e-3) + } + + /// Sous l'orbite, la caméra suit le pointeur et le flou de mouvement étale le contenu vers + /// sa position de la frame d'avant : le masque garde la perspective du contenu (un quad warpé + /// aux côtés parallèles aux siens, pas un rect droit) et couvre le secret aux deux frames. + #[test] + fn a_moving_tilted_mask_keeps_its_perspective() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let json = zoomed_golden_scene_json() + .replace(r#""rotation":"none""#, r#""rotation":"orbit","focusMode":"auto""#); + let scene = Scene::from_json(&json).expect("scène"); + // Le pointeur file vers la droite : l'orbite le suit, la caméra bouge à chaque frame. + let track: &'static crate::cursor::CursorTrack = Box::leak(Box::new(crate::cursor::CursorTrack::new( + (0..=90).map(|i| (i as f32 / 30.0, 0.05 + 0.5 * i as f32 / 90.0, 0.3)).collect(), + vec![], + vec![], + ))); + let g = plan_frame(&FrameGeometryInput { cursor: Some(track), ..golden_input(&scene, &cfg) }); + let render = [1170.0, 658.0]; + let quad = g.screen_tilt([g.s_dst[2] * render[0], g.s_dst[3] * render[1]]).expect("orbite"); + let trail = g.tilt_trail(render).expect("garde : flou de mouvement actif sous l'orbite"); + assert_ne!(trail.corners, quad.corners, "garde : la caméra doit bouger d'une frame à l'autre"); + let a = blur_annotation(""); + let m = g.privacy_mask(&a, render).expect("masque"); + let warp = m.warp.expect("le masque d'un écran incliné qui bouge reste un quad warpé"); + let origin = [m.dst[0] * render[0], m.dst[1] * render[1]]; + let mask = warp.map(|[x, y]| [origin[0] + x, origin[1] + y]); + let centre = [(g.s_dst[0] + g.s_dst[2] * 0.5) * render[0], (g.s_dst[1] + g.s_dst[3] * 0.5) * render[1]]; + let before = crate::regions::TiltedQuad { corners: trail.corners, ..quad }; + for q in [quad, before] { + for (fx, fy) in [(a.x, a.y), (a.x + a.w, a.y), (a.x + a.w, a.y + a.h), (a.x, a.y + a.h)] { + let (px, py) = q.point_px(fx, fy); + let p = [centre[0] + px, centre[1] + py]; + assert!(in_convex_quad(mask, p), "{p:?} hors du masque {mask:?}"); + } + } + let (tl, tr) = (quad.point_px(a.x, a.y), quad.point_px(a.x + a.w, a.y)); + let content = (tr.1 - tl.1) / (tr.0 - tl.0); + assert!(content.abs() > 1e-3, "garde : l'orbite doit incliner le haut du secret ({content})"); + let top = (mask[1][1] - mask[0][1]) / (mask[1][0] - mask[0][0]); + assert!((top - content).abs() < 1e-3, "le haut du masque ({top}) doit suivre celui du contenu ({content})"); + assert!(!m.oval_ok, "un masque élargi à la trace doit refuser l'ovale"); + } + + /// Le métrage que l'éditeur lit avec l'image : le rect zoomé à plat, le quad incliné du mode 8 + /// sous un préset, l'homographie de ses coins sous la caméra réelle. + #[test] + fn the_footage_quad_is_where_masks_land() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let render = [1170.0, 658.0]; + let flat = plan_frame(&golden_input(&zoomed_golden_scene(), &cfg)); + let [x, y, w, h] = flat.s_dst; + assert_eq!( + flat.footage_quad(render), + FootageQuad { corners: [[x, y], [x + w, y], [x + w, y + h], [x, y + h]], projective: false } + ); + let tilted = plan_frame(&golden_input(&tilted_golden_scene(), &cfg)); + let q = tilted.footage_quad(render); + let quad = tilted.screen_tilt([tilted.s_dst[2] * render[0], tilted.s_dst[3] * render[1]]).expect("iso"); + let centre = [ + (tilted.s_dst[0] + tilted.s_dst[2] * 0.5) * render[0], + (tilted.s_dst[1] + tilted.s_dst[3] * 0.5) * render[1], + ]; + for (k, (cx, cy)) in quad.corners.iter().enumerate() { + let want = [(centre[0] + cx) / render[0], (centre[1] + cy) / render[1]]; + assert!((q.corners[k][0] - want[0]).abs() < 1e-5 && (q.corners[k][1] - want[1]).abs() < 1e-5, "coin {k}"); + } + assert!(!q.projective); + let json = zoomed_golden_scene_json() + .replace(r#""rotation":"none""#, r#""rotation":"orbit","focusMode":"auto""#); + let orbit = plan_frame(&golden_input(&Scene::from_json(&json).expect("scène"), &cfg)); + assert!(orbit.footage_quad(render).projective, "la caméra réelle warpe par homographie"); + } + /// Une entrée mesurée sur le cadre de sortie n'est pas concernée par le zoom. #[test] fn a_privacy_mask_in_frame_space_ignores_the_zoom() { @@ -7016,7 +7629,7 @@ mod tests { #[test] fn layer_cb_matches_the_shader_constant_buffer() { use std::mem::{align_of, offset_of, size_of}; - assert_eq!(size_of::(), 176); + assert_eq!(size_of::(), 256); assert_eq!(align_of::(), 16); for (name, got, want) in [ ("dst", offset_of!(LayerCB, dst), 0), @@ -7032,11 +7645,527 @@ mod tests { ("trail_a", offset_of!(LayerCB, trail_a), 128), ("trail_b", offset_of!(LayerCB, trail_b), 144), ("trail_mb", offset_of!(LayerCB, trail_mb), 160), + ("cover", offset_of!(LayerCB, cover), 176), + ("persp", offset_of!(LayerCB, persp), 192), + ("layer_fx", offset_of!(LayerCB, layer_fx), 240), ] { assert_eq!(got, want, "offset de `{name}`"); } } + /// The camera-0 draw as every backend builds it today leaves both new lanes at zero: no + /// homography (`layer_fx.y = 0`) and opaque (`layer_fx.x = 0`), so the shaders take exactly + /// the path they took before the lanes existed. The other fields are the ones the three + /// backends used to spell out inline. + #[test] + fn the_webcam_draw_leaves_the_homography_and_transparency_lanes_at_zero() { + let cfg = crate::config::all().pop().expect("cfg"); + let scene = golden_scene(); + let g = plan_frame(&golden_input(&scene, &cfg)); + let src = [0.1, 0.2, 0.9, 0.8]; + let cb = g.webcam_video_cb(src, [1.0, 0.75], 2.0, 0.4); + assert_eq!(cb.persp, [[0.0; 4]; 3]); + assert_eq!(cb.layer_fx, [0.0; 4]); + assert_eq!(cb.dst, g.w_dst); + assert_eq!(cb.src, src); + assert_eq!(cb.src_prev, src); + assert_eq!(cb.quad_px, g.w_px); + assert_eq!(cb.radius_px, g.w_radius); + assert_eq!(cb.mode, 0.0); + assert_eq!(cb.color, [0.0, 0.0, 0.0, 1.0]); + assert_eq!(cb.fx, [1.0, 0.75, 2.0, 0.4]); + assert_eq!(cb.dst_prev, g.w_dst_prev); + assert_eq!(cb.mb, [g.mb_taps, g.mb_amount, 1.0, 0.0]); + let min_px = g.w_px[0].min(g.w_px[1]); + assert_eq!(cb.cover, [g.webcam_cover, 0.04 * min_px * g.webcam_cover, 0.35, 0.0]); + } + + fn extra_camera(index: usize) -> crate::scene::SceneCamera { + crate::scene::SceneCamera { + index, + rotation: 0, + mirror: None, + crop: None, + homography: None, + aspect: None, + } + } + + fn planned(camera: usize, opacity: f32) -> crate::camera_layers::CameraLayerPlan { + crate::camera_layers::CameraLayerPlan { + camera, + dst: [0.5, 0.25, 0.25, 0.5], + radius_frac: 0.2, + shape: 0, + opacity, + fills_frame: false, + } + } + + /// One planned layer's video draw: placed at the plan's rect with its corners, no trail, + /// the transparency is `1 - opacity`, and the camera's homography replaces its crop. + #[test] + fn camera_layer_cb_places_the_layer_and_carries_its_lanes() { + let render = [1920.0, 1080.0]; + let (visible, tex) = ([1280.0, 720.0], [1280.0, 736.0]); + let base = extra_camera_base_cb([1.0, 720.0 / 736.0]); + let plan = planned(1, 0.25); + let plain = camera_layer_cb(&plan, None, visible, tex, render, &base); + assert_eq!(plain.dst, plan.dst); + assert_eq!(plain.dst_prev, plan.dst); + assert_eq!(plain.quad_px, [480.0, 540.0]); + assert_eq!(plain.radius_px, 0.2 * 480.0); + assert_eq!(plain.layer_fx, [0.75, 0.0, 0.0, 0.0]); + assert_eq!(plain.persp, [[0.0; 4]; 3]); + assert_eq!(plain.src_prev, plain.src); + assert_eq!(plain.mb[0], 1.0); + assert_eq!(plain.fx, base.fx); + + // A crop moves the source rect; with a homography as well, the crop is ignored. + let crop = SceneCrop { x: 0.5, y: 0.0, width: 0.5, height: 1.0 }; + let cropped = extra_camera(1); + let cropped = crate::scene::SceneCamera { crop: Some(crop), ..cropped }; + let with_crop = camera_layer_cb(&plan, Some(&cropped), visible, tex, render, &base); + assert_ne!(with_crop.src, plain.src); + let h = [-1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]; + let corrected = crate::scene::SceneCamera { homography: Some(h), ..cropped.clone() }; + let warped = camera_layer_cb(&plan, Some(&corrected), visible, tex, render, &base); + assert_eq!(warped.layer_fx[1], 1.0); + assert_eq!(warped.layer_fx[0], 0.75); + let rows = [[-1.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]]; + assert_eq!(warped.persp, rows); + assert_eq!(warped.src, plain.src); + + // Mirror swaps the u bounds only; 180° swaps both, and mirror on top of it un-swaps u. + let mirrored = crate::scene::SceneCamera { mirror: Some(true), ..extra_camera(1) }; + let m = camera_layer_cb(&plan, Some(&mirrored), visible, tex, render, &base).src; + assert_eq!(m, [plain.src[2], plain.src[1], plain.src[0], plain.src[3]]); + let turned = crate::scene::SceneCamera { rotation: 180, ..extra_camera(1) }; + let t = camera_layer_cb(&plan, Some(&turned), visible, tex, render, &base).src; + assert_eq!(t, [plain.src[2], plain.src[3], plain.src[0], plain.src[1]]); + let both = crate::scene::SceneCamera { mirror: Some(true), ..turned.clone() }; + let b = camera_layer_cb(&plan, Some(&both), visible, tex, render, &base).src; + assert_eq!(b, [plain.src[0], plain.src[3], plain.src[2], plain.src[1]]); + } + + /// Camera 0's default layer, planned outside every layout region at its own rect and fully + /// opaque, keeps its motion trail; moved or faded, it draws without one (R8). + #[test] + fn camera_0s_untouched_layer_keeps_its_trail() { + let render = [1920.0, 1080.0]; + let base = LayerCB { + dst: [0.7, 0.7, 0.2, 0.2], + dst_prev: [0.65, 0.7, 0.2, 0.2], + mb: [8.0, 0.5, 1.0, 0.0], + ..Default::default() + }; + let plan = crate::camera_layers::CameraLayerPlan { + camera: 0, + dst: base.dst, + radius_frac: 0.2, + shape: 0, + opacity: 1.0, + fills_frame: false, + }; + let kept = camera_layer_cb(&plan, None, [64.0; 2], [64.0; 2], render, &base); + assert_eq!(kept.dst_prev, base.dst_prev); + assert_eq!(kept.mb, base.mb); + + let moved = crate::camera_layers::CameraLayerPlan { dst: [0.1, 0.1, 0.2, 0.2], ..plan }; + let m = camera_layer_cb(&moved, None, [64.0; 2], [64.0; 2], render, &base); + assert_eq!(m.dst_prev, moved.dst); + assert_eq!(m.mb, [1.0, 0.0, 1.0, 0.0]); + + let faded = crate::camera_layers::CameraLayerPlan { opacity: 0.5, ..plan }; + let f = camera_layer_cb(&faded, None, [64.0; 2], [64.0; 2], render, &base); + assert_eq!(f.dst_prev, faded.dst); + assert_eq!(f.mb, [1.0, 0.0, 1.0, 0.0]); + } + + /// A corrected camera in a box of another ratio is cover-cropped, not stretched: the box's + /// local uv lands on a centred window of the corrected picture with the box's own ratio. + #[test] + fn a_corrected_camera_is_cover_fitted_into_its_box() { + let identity = [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]; + let cam = crate::scene::SceneCamera { + homography: Some(identity), + aspect: Some(16.0 / 9.0), + ..extra_camera(2) + }; + let row = |cb: &LayerCB, r: usize| [cb.persp[r][0], cb.persp[r][1], cb.persp[r][2]]; + let near = |a: [f32; 3], b: [f32; 3]| a.iter().zip(b).all(|(x, y)| (x - y).abs() < 1e-5); + + // A box narrower than 16:9 (1752x1080, a 1.62 export): the full height, a centred + // slice of the width with the box's ratio. + let narrow = LayerCB { quad_px: [1752.0, 1080.0], ..Default::default() }; + let cb = with_camera_homography(narrow, Some(&cam)); + let du = (1752.0 / 1080.0) / (16.0 / 9.0); + assert!(near(row(&cb, 0), [du, 0.0, (1.0 - du) * 0.5]), "{:?}", cb.persp); + assert!(near(row(&cb, 1), [0.0, 1.0, 0.0]), "{:?}", cb.persp); + assert!(near(row(&cb, 2), [0.0, 0.0, 1.0]), "{:?}", cb.persp); + + // A wider box (a 16:9 picture in a 21:9 box): full width, a centred band of height. + let wide = LayerCB { quad_px: [2100.0, 900.0], ..Default::default() }; + let cb = with_camera_homography(wide, Some(&cam)); + let dv = (16.0 / 9.0) / (2100.0 / 900.0); + assert!(near(row(&cb, 0), [1.0, 0.0, 0.0]), "{:?}", cb.persp); + assert!(near(row(&cb, 1), [0.0, dv, (1.0 - dv) * 0.5]), "{:?}", cb.persp); + + // The box the template gives a PiP of this camera has its aspect: nothing is cropped. + let pip = LayerCB { quad_px: [422.4, 237.6], ..Default::default() }; + let cb = with_camera_homography(pip, Some(&cam)); + assert!(near(row(&cb, 0), [1.0, 0.0, 0.0]) && near(row(&cb, 1), [0.0, 1.0, 0.0])); + + // A real (projective) matrix is composed on the right: H * C, so H's third row picks + // up the window too. + let h = [0.8, 0.1, 0.05, -0.02, 0.9, 0.04, 0.1, -0.2, 1.0]; + let cam = crate::scene::SceneCamera { homography: Some(h), ..cam }; + let cb = with_camera_homography(narrow, Some(&cam)); + let u0 = (1.0 - du) * 0.5; + assert!(near(row(&cb, 2), [0.1 * du, -0.2, 0.1 * u0 + 1.0]), "{:?}", cb.persp); + assert_eq!(cb.layer_fx[1], 1.0); + } + + /// Without an aspect, or with a degenerate box, the whole corrected picture is shown. + #[test] + fn the_cover_window_falls_back_to_the_whole_picture() { + let whole = [0.0, 0.0, 1.0, 1.0]; + assert_eq!(corrected_cover_window(None, 1.5), whole); + assert_eq!(corrected_cover_window(Some(0.0), 1.5), whole); + assert_eq!(corrected_cover_window(Some(1.5), f32::NAN), whole); + assert_eq!(corrected_cover_window(Some(1.5), f32::INFINITY), whole); + assert_eq!(corrected_cover_window(Some(1.5), 0.0), whole); + assert_eq!(corrected_cover_window(Some(1.5), 1.5), whole); + } + + /// A homography with a NaN or an infinity in it counts as none. + #[test] + fn a_non_finite_homography_is_ignored() { + let mut h = [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]; + let ok = crate::scene::SceneCamera { homography: Some(h), ..extra_camera(1) }; + assert_eq!(camera_homography(Some(&ok)), Some(h)); + for bad in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { + h[7] = bad; + let cam = crate::scene::SceneCamera { homography: Some(h), ..extra_camera(1) }; + assert_eq!(camera_homography(Some(&cam)), None); + let cb = with_camera_homography(LayerCB::default(), Some(&cam)); + assert_eq!(cb.layer_fx, [0.0; 4]); + assert_eq!(cb.persp, [[0.0; 4]; 3]); + } + } + + /// Camera 0's settings give back exactly today's source rect: layout crop, project mirror + /// and desk turn, through `camera_source_rect`. + #[test] + fn camera_0_settings_reproduce_todays_source_rect() { + let crop = SceneCrop { x: 0.1, y: 0.2, width: 0.6, height: 0.7 }; + let (visible, tex, box_ar) = ([1280.0, 720.0], [1280.0, 736.0], 1.3); + for (flip_u, flip_v, full_frame) in [ + (false, false, false), + (true, false, false), + (false, true, false), + (true, true, false), + (false, true, true), + ] { + let mut g = plan_frame(&golden_input(&golden_scene(), &crate::config::all()[0])); + g.webcam = WebcamOrientation { flip_u, flip_v, full_frame }; + let mut scene = golden_scene(); + scene.layout.webcam_crop = Some(crop); + let [cu0, cv0, cu1, cv1] = webcam_source_rect( + visible, + tex, + if full_frame { None } else { Some(crop) }, + box_ar, + ); + let (u0, u1) = if flip_u { (cu1, cu0) } else { (cu0, cu1) }; + let (v0, v1) = if flip_v { (cv1, cv0) } else { (cv0, cv1) }; + let cam0 = g.camera0_settings(Some(&scene)); + assert!(cam0.homography.is_none()); + assert_eq!(camera_source_rect(Some(&cam0), visible, tex, box_ar), [u0, v0, u1, v1]); + } + } + + /// Camera 0 with a perspective is cover-fitted into a box of another ratio, on both paths: + /// its default draw in a Full Camera section (no layout region, the box fills the frame) + /// and a planned frame-filling layer. Its PiP box, which has the corrected aspect, shows + /// the whole corrected picture. + #[test] + fn camera_0_with_a_perspective_is_cover_fitted_on_both_paths() { + let cfg = crate::config::all().pop().expect("cfg"); + let identity = "[1,0,0,0,1,0,0,0,1]"; + let aspect = 4.0_f32 / 3.0; + let cameras = format!(r#""cameras":[{{"index":0,"homography":{identity},"aspect":{aspect}}}]"#); + let zoom = r#""zoomRegions":[{"clipIndex":0,"startSec":0.0,"endSec":5.0,"scale":2.0,"focusX":0.5,"focusY":0.3,"rotation":"none"}]"#; + let near = |a: f32, b: f32| (a - b).abs() < 1e-4; + let render = [1170.0_f32, 658.0]; + let box_ar = render[0] / render[1]; + // A 4:3 picture in a wider box: the full width, a centred band of the box's ratio. + let dv = aspect / box_ar; + let check = |cb: &LayerCB| { + assert_eq!(cb.layer_fx[1], 1.0); + assert!(near(cb.persp[0][0], 1.0) && near(cb.persp[0][2], 0.0), "{:?}", cb.persp); + assert!(near(cb.persp[1][1], dv), "{:?}", cb.persp); + assert!(near(cb.persp[1][2], (1.0 - dv) * 0.5), "{:?}", cb.persp); + }; + + // No layout region, inside a Full Camera section: the default box fills the frame. + let json = zoomed_golden_scene_json().replace( + zoom, + &format!( + r#""zoomRegions":[],{cameras},"cameraFullscreenRegions":[{{"clipIndex":0,"startSec":0.0,"endSec":9.0}}]"# + ), + ); + let scene = Scene::from_json(&json).expect("scene"); + let g = plan_frame(&FrameGeometryInput { + timeline_t_override: Some(4.0), + ..golden_input(&scene, &cfg) + }); + assert!(g.camera_layers.is_empty()); + assert!(near(g.w_px[0], render[0]) && near(g.w_px[1], render[1]), "{:?}", g.w_px); + let cam0 = g.camera0_settings(Some(&scene)); + assert_eq!(cam0.aspect, Some(aspect)); + check(&with_camera_homography( + g.webcam_video_cb([0.0, 0.0, 1.0, 1.0], [1.0, 1.0], 0.0, 0.0), + Some(&cam0), + )); + + // A planned layer that fills the frame. + let json = zoomed_golden_scene_json().replace( + zoom, + &format!( + r#""zoomRegions":[],{cameras},"cameraLayoutRegions":[{{"clipIndex":0,"startSec":0.0,"endSec":4.0,"layers":[ + {{"camera":0,"rect":{{"x":0,"y":0,"width":1,"height":1}},"fillsFrame":true}}]}}]"# + ), + ); + let scene = Scene::from_json(&json).expect("scene"); + let g = plan_frame(&golden_input(&scene, &cfg)); + assert_eq!(g.camera_layers.len(), 1); + let cam0 = g.camera0_settings(Some(&scene)); + let base = g.webcam_video_cb([0.0; 4], [1.0, 1.0], 0.0, 0.0); + let plan = g.camera_layers[0]; + check(&camera_layer_cb(&plan, Some(&cam0), [1280.0, 720.0], [1280.0, 720.0], render, &base)); + + // The PiP box the app gives camera 0 has the corrected aspect: nothing is cropped. + let pip = LayerCB { quad_px: [400.0, 300.0], ..Default::default() }; + let cb = with_camera_homography(pip, Some(&cam0)); + assert!(near(cb.persp[0][0], 1.0) && near(cb.persp[1][1], 1.0), "{:?}", cb.persp); + } + + /// The shadow of a planned layer fades with it, and a frame-filling layer has none. + #[test] + fn a_planned_layers_shadow_fades_with_it() { + let cfg = crate::config::all().pop().expect("cfg"); + let g = plan_frame(&golden_input(&golden_scene(), &cfg)); + let render = [1920.0, 1080.0]; + let plan = planned(2, 0.4); + let base = extra_camera_base_cb([1.0; 2]); + let video = camera_layer_cb(&plan, None, [64.0; 2], [64.0; 2], render, &base); + let shadow = g.camera_layer_shadow(&plan, &video, render, true, false).expect("shadow"); + assert_eq!(shadow.mode, 2.0); + assert_eq!(shadow.layer_fx[0], video.layer_fx[0]); + assert_eq!(shadow.color[3], WEBCAM_SHADOW_OPACITY); + assert!(g.camera_layer_shadow(&plan, &video, render, false, false).is_none()); + let full = crate::camera_layers::CameraLayerPlan { fills_frame: true, ..plan }; + assert!(g.camera_layer_shadow(&full, &video, render, true, false).is_none()); + // Cutout removes camera 0's bubble, not the others'. + assert!(g.camera_layer_shadow(&plan, &video, render, true, true).is_some()); + let cam0 = crate::camera_layers::CameraLayerPlan { camera: 0, ..plan }; + assert!(g.camera_layer_shadow(&cam0, &video, render, true, true).is_none()); + } + + /// A turned Full Camera section from 1 s to 9 s: the plan carries the cover strength, full in + /// the hold after the start and zero in the steady part. + #[test] + fn the_frame_plan_carries_the_cover() { + let cfg = crate::config::all().pop().expect("cfg"); + let json = zoomed_golden_scene_json().replace( + r#""zoomRegions":[{"clipIndex":0,"startSec":0.0,"endSec":5.0,"scale":2.0,"focusX":0.5,"focusY":0.3,"rotation":"none"}]"#, + r#""zoomRegions":[],"cameraFullscreenRegions":[{"clipIndex":0,"startSec":1.0,"endSec":9.0,"rotation":180,"fullFrame":true}]"#, + ); + let scene = Scene::from_json(&json).expect("scene"); + let cover_at = |t: f32| { + plan_frame(&FrameGeometryInput { timeline_t_override: Some(t), ..golden_input(&scene, &cfg) }) + .webcam_cover + }; + assert_eq!(cover_at(1.5), 1.0); + assert_eq!(cover_at(5.0), 0.0); + } + + /// A scene without layout regions plans no camera layers, and camera 0 lands exactly where + /// it did: the new (empty) keys change nothing. + #[test] + fn without_layout_regions_the_plan_has_no_camera_layers() { + let cfg = crate::config::all().pop().expect("cfg"); + let before = plan_frame(&golden_input(&golden_scene(), &cfg)); + let json = zoomed_golden_scene_json().replace( + r#""zoomRegions":[{"clipIndex":0,"startSec":0.0,"endSec":5.0,"scale":2.0,"focusX":0.5,"focusY":0.3,"rotation":"none"}]"#, + r#""zoomRegions":[],"cameras":[],"cameraLayoutRegions":[]"#, + ); + let scene = Scene::from_json(&json).expect("scene"); + let after = plan_frame(&golden_input(&scene, &cfg)); + assert!(before.camera_layers.is_empty()); + assert!(after.camera_layers.is_empty()); + assert_eq!(after.w_dst, before.w_dst); + assert_eq!(after.w_dst_prev, before.w_dst_prev); + assert_eq!(after.w_px, before.w_px); + assert_eq!(after.w_radius, before.w_radius); + assert_eq!(after.shape_fade, before.shape_fade); + assert_eq!(after.webcam, before.webcam); + assert_eq!(after.webcam_cover, before.webcam_cover); + } + + /// Inside a layout region camera 0 is one of the planned layers, at the region's rect. + #[test] + fn with_a_layout_region_camera_0_comes_from_the_plan() { + let cfg = crate::config::all().pop().expect("cfg"); + let json = zoomed_golden_scene_json().replace( + r#""zoomRegions":[{"clipIndex":0,"startSec":0.0,"endSec":5.0,"scale":2.0,"focusX":0.5,"focusY":0.3,"rotation":"none"}]"#, + r#""zoomRegions":[],"cameraLayoutRegions":[{"clipIndex":0,"startSec":0.0,"endSec":4.0,"layers":[ + {"camera":0,"rect":{"x":0.1,"y":0.2,"width":0.3,"height":0.4},"radiusFrac":0.1,"shape":"rectangle"}]}]"#, + ); + let scene = Scene::from_json(&json).expect("scene"); + // `golden_input` samples t = 1.5 s: past the lead-in, well before the lead-out. + let g = plan_frame(&golden_input(&scene, &cfg)); + assert_eq!(g.camera_layers.len(), 1); + let cam0 = g.camera_layers[0]; + assert_eq!(cam0.camera, 0); + assert_eq!(cam0.dst, [0.1, 0.2, 0.3, 0.4]); + assert_eq!(cam0.opacity, 1.0); + assert_eq!(cam0.shape, webcam_shape_code("rectangle")); + assert!(!cam0.fills_frame); + // Outside the region camera 0 is the default PiP, where `w_dst` puts it. + let outside = plan_frame(&FrameGeometryInput { + timeline_t_override: Some(6.0), + ..golden_input(&scene, &cfg) + }); + assert_eq!(outside.camera_layers.len(), 1); + assert_eq!(outside.camera_layers[0].dst, outside.w_dst); + let r = outside.w_radius / outside.w_px[0].min(outside.w_px[1]).max(1.0); + assert_eq!(outside.camera_layers[0].radius_frac, r); + } + + /// The desk label's opacity at the compositor caller boundary, against the cover of the same + /// frame plan: equal at every sampled frame, and the label (spanned over its whole section by + /// the app) is on screen wherever the cover is not 0. Three sections: the review's 2.6 s one + /// at 1×, where the label's old fade (from its own 1.15 s window) was half gone while the + /// cover still held 1; a long one; and a section inside a 2× speed region, whose cover runs + /// on the screen clock. + #[test] + fn the_desk_label_fades_with_the_cover() { + use crate::text_anim::{annotation_text_state, DESK_COVER_ANIMATION}; + let cfg = crate::config::all().pop().expect("cfg"); + let zoom = r#""zoomRegions":[{"clipIndex":0,"startSec":0.0,"endSec":5.0,"scale":2.0,"focusX":0.5,"focusY":0.3,"rotation":"none"}]"#; + let section = |start: f32, end: f32, speed: Option<&str>| { + let regions = format!( + r#""zoomRegions":[],"cameraFullscreenRegions":[{{"clipIndex":0,"startSec":{start},"endSec":{end},"rotation":180,"fullFrame":true}}]{}"#, + speed.map(|s| format!(",{s}")).unwrap_or_default() + ); + Scene::from_json(&zoomed_golden_scene_json().replace(zoom, ®ions)).expect("scene") + }; + // (label, cover) at source time `t` of a label spanning [start, end]. + let sample = |scene: &Scene, start: f32, end: f32, t: f32| { + let g = plan_frame(&FrameGeometryInput { + timeline_t_override: Some(t), + ..golden_input(scene, &cfg) + }); + let label = annotation_text_state( + Some(DESK_COVER_ANIMATION), + (t - start) * 1000.0, + (end - start) * 1000.0, + g.webcam_cover, + ); + (label.opacity, g.webcam_cover) + }; + let speed = r#""speedRegions":[{"clipIndex":0,"startSec":1.0,"endSec":9.0,"speed":2.0}]"#; + let cases = [ + ("2.6 s at 1x", section(1.0, 3.6, None), 1.0f32, 3.6f32), + ("long at 1x", section(1.0, 9.0, None), 1.0, 9.0), + ("2.6 s on screen inside 2x", section(2.0, 7.2, Some(speed)), 2.0, 7.2), + ]; + for (name, scene, start, end) in &cases { + let mut t = start - 0.1; + while t < end + 0.1 { + let (label, cover) = sample(scene, *start, *end, t); + assert_eq!(label, cover, "{name}: t = {t}"); + if cover > 0.0 { + assert!(t >= *start && t < *end, "{name}: cover {cover} outside the label at {t}"); + } + t += 1.0 / 120.0; + } + } + // The review's frames: the cover still holds 1 at 0.9 s and 1.0 s (the old label was at + // ~0.5 and below). The 0.28 s left between the holds is two 0.14 s fades meeting at + // +1.1575 s, where label and cover touch 0 together before rising into the end hold. + let (_, short, s0, s1) = &cases[0]; + for at in [0.5, 0.9, 1.0] { + assert_eq!(sample(short, *s0, *s1, s0 + at), (1.0, 1.0), "at +{at} s"); + } + let (mid, _) = sample(short, *s0, *s1, s0 + 1.09); + assert!(mid > 0.0 && mid < 1.0, "mid-fade {mid}"); + let (low, low_cover) = sample(short, *s0, *s1, s0 + 1.1575); + assert!(low < 1e-3 && low == low_cover, "junction {low} vs {low_cover}"); + assert_eq!(sample(short, *s0, *s1, s1 - 1.25), (1.0, 1.0), "end hold"); + // Inside 2x the same 2.6 s of screen time takes 5.2 s of source: every screen instant + // matches the 1x section's, which the source-time label could not do. + let (_, fast, f0, f1) = &cases[2]; + for k in 0..=26 { + let screen = k as f32 * 0.1; + let slow = sample(short, *s0, *s1, s0 + screen); + let quick = sample(fast, *f0, *f1, f0 + 2.0 * screen); + assert!((slow.0 - quick.0).abs() < 1e-3, "screen +{screen} s: {slow:?} vs {quick:?}"); + } + } + + /// A turned Full Camera section that meets a layout region: the cover follows the seam rule + /// (no hold before the layout region, only its own fade), and the label, spanned over the + /// whole section by the app, follows the cover there frame for frame without any seam + /// logic of its own. + #[test] + fn the_desk_label_follows_the_cover_across_a_seam_with_a_layout_region() { + use crate::regions::DESK_COVER_FADE_S; + use crate::text_anim::{annotation_text_state, DESK_COVER_ANIMATION}; + let cfg = crate::config::all().pop().expect("cfg"); + let zoom = r#""zoomRegions":[{"clipIndex":0,"startSec":0.0,"endSec":5.0,"scale":2.0,"focusX":0.5,"focusY":0.3,"rotation":"none"}]"#; + let full = r#""zoomRegions":[],"cameraFullscreenRegions":[{"clipIndex":0,"startSec":1.0,"endSec":6.0,"rotation":180,"fullFrame":true}]"#; + let layout = r#","cameraLayoutRegions":[{"clipIndex":0,"startSec":6.0,"endSec":9.0,"layers":[ + {"camera":0,"rect":{"x":0.1,"y":0.2,"width":0.3,"height":0.4},"radiusFrac":0.1,"shape":"rectangle"}]}]"#; + let json = zoomed_golden_scene_json(); + let seam = + Scene::from_json(&json.replace(zoom, &format!("{full}{layout}"))).expect("scene"); + let lone = Scene::from_json(&json.replace(zoom, full)).expect("scene"); + let (start, end) = (1.0f32, 6.0f32); + let sample = |scene: &Scene, t: f32| { + let g = plan_frame(&FrameGeometryInput { + timeline_t_override: Some(t), + ..golden_input(scene, &cfg) + }); + let label = annotation_text_state( + Some(DESK_COVER_ANIMATION), + (t - start) * 1000.0, + (end - start) * 1000.0, + g.webcam_cover, + ); + (label.opacity, g.webcam_cover) + }; + let mut t = start - 0.1; + while t < 9.1 { + let (label, cover) = sample(&seam, t); + assert_eq!(label, cover, "t = {t}"); + if cover > 0.0 { + assert!(t >= start && t < end, "cover {cover} outside the label at {t}"); + } + t += 1.0 / 120.0; + } + // Before the seam the cover (and so the label) is only its fade: half way up at + // end - fade/2 and sharp just before the fade, where a lone section still holds 1. + let (mid, _) = sample(&seam, end - DESK_COVER_FADE_S / 2.0); + assert!((mid - 0.5).abs() < 1e-3, "half way up at the seam: {mid}"); + let before_fade = end - DESK_COVER_FADE_S - 0.05; + assert_eq!(sample(&seam, before_fade), (0.0, 0.0), "sharp before the seam's fade"); + assert_eq!(sample(&lone, before_fade), (1.0, 1.0), "a lone section still holds"); + assert!(sample(&seam, end - 0.001).0 > 0.99, "covered at the seam"); + } + /// Le pivot doit rester collé à `center` quand le sprite grandit — c'est exactement ce qui /// était cassé (ancrage centré en dur : la pointe s'éloignait proportionnellement à la /// taille). On dessine la même flèche à deux tailles et on vérifie que le point désigné @@ -7444,6 +8573,9 @@ mod tests { w_px: [0.0, 0.0], w_radius: 0.0, shape_fade: 0.0, + webcam: WebcamOrientation::default(), + webcam_cover: 0.0, + camera_layers: Vec::new(), window_frame: None, screen_mask: None, } @@ -8508,4 +9640,117 @@ mod tests { println!("left ×2 : {:.3} côté proche, {:.3} côté lointain", near / at_focus, far / at_focus); assert!(near > 1.04 * at_focus && far < 0.96 * at_focus, "{near} {at_focus} {far}"); } + + use crate::scene::SceneCameraFullscreenRegion; + + fn cam_region(rotation: u16, mirror: Option) -> SceneCameraFullscreenRegion { + SceneCameraFullscreenRegion { + clip_index: None, + start_sec: 0.0, + end_sec: 1.0, + rotation, + mirror, + full_frame: rotation == 180, + } + } + + #[test] + fn without_a_region_the_webcam_is_laid_as_before() { + for m in [false, true] { + assert_eq!( + webcam_orientation(None, m), + WebcamOrientation { flip_u: m, flip_v: false, full_frame: false } + ); + } + } + + /// 180° is both axes swapped; a mirror on top cancels the horizontal one. + #[test] + fn a_turned_region_swaps_both_axes_and_a_mirror_cancels_one() { + let turned = cam_region(180, Some(false)); + assert_eq!( + webcam_orientation(Some(&turned), true), + WebcamOrientation { flip_u: true, flip_v: true, full_frame: true } + ); + let turned_mirrored = cam_region(180, Some(true)); + assert_eq!( + webcam_orientation(Some(&turned_mirrored), false), + WebcamOrientation { flip_u: false, flip_v: true, full_frame: true } + ); + } + + #[test] + fn a_plain_region_keeps_the_layout_mirror_and_an_unknown_rotation_is_none() { + assert!(webcam_orientation(Some(&cam_region(0, None)), true).flip_u); + let odd = cam_region(90, None); + assert_eq!( + webcam_orientation(Some(&odd), false), + WebcamOrientation { flip_u: false, flip_v: false, full_frame: false } + ); + } + + /// `layer.wgsl` is only compiled on Linux, in CI, where a name or syntax error breaks every + /// draw. Parse and validate it on every host through wgpu's own naga, and pin the desk-view + /// cover: the fragment entry point must call the radius-taking blur kernel, and the kernel + /// must keep its taps inside the picture's valid area. + /// + /// The file does not declare `LAYER_MODELS`: `compositor_linux::layer_source` prefixes it, + /// once per pipeline. Both variants are checked here the same way, the one without the 3D + /// models first, since that is the pipeline the webcam (mode 0) is drawn with. + #[test] + fn layer_wgsl_validates_and_its_fragment_applies_the_desk_view_cover() { + use wgpu::naga; + fn calls(block: &naga::Block, target: naga::Handle) -> usize { + block + .iter() + .map(|st| match st { + naga::Statement::Call { function, .. } => usize::from(*function == target), + naga::Statement::Block(b) => calls(b, target), + naga::Statement::If { accept, reject, .. } => calls(accept, target) + calls(reject, target), + naga::Statement::Loop { body, continuing, .. } => calls(body, target) + calls(continuing, target), + naga::Statement::Switch { cases, .. } => cases.iter().map(|c| calls(&c.body, target)).sum(), + _ => 0, + }) + .sum() + } + + for models in [false, true] { + // The same prefix as `compositor_linux::layer_source`, which only builds on Linux. + let source = + format!("const LAYER_MODELS: bool = {models};\n{}", include_str!("vk_shaders/layer.wgsl")); + let module = naga::front::wgsl::parse_str(&source) + .unwrap_or_else(|e| panic!("layer.wgsl (LAYER_MODELS = {models}) parses: {e:?}")); + naga::valid::Validator::new(naga::valid::ValidationFlags::all(), naga::valid::Capabilities::all()) + .validate(&module) + .unwrap_or_else(|e| panic!("layer.wgsl (LAYER_MODELS = {models}) validates: {e:?}")); + let (kernel, f) = module + .functions + .iter() + .find(|(_, f)| f.name.as_deref() == Some("blur_webcam_radius")) + .expect("blur_webcam_radius exists"); + assert_eq!(f.arguments.len(), 5, "uv, max_r_px, qpx, local_px, valid"); + // The taps are clamped to the valid part of an aligned decoder texture, half a texel + // in: the kernel reads the texture size to know what half a texel is. + assert!( + f.expressions.iter().any(|(_, e)| matches!( + e, + naga::Expression::ImageQuery { query: naga::ImageQuery::Size { .. }, .. } + )), + "blur_webcam_radius reads the texture size for its half-texel clamp" + ); + // `fs_main` only applies the layer's transparency to `fs_layer`, which draws it. + let fs = module.entry_points.iter().find(|e| e.name == "fs_main").expect("fs_main"); + let (body, f) = module + .functions + .iter() + .find(|(_, f)| f.name.as_deref() == Some("fs_layer")) + .expect("fs_layer exists"); + assert_eq!(calls(&fs.function.body, body), 1, "fs_main draws through fs_layer"); + assert_eq!( + calls(&f.body, kernel), + 1, + "fs_layer blurs the covered camera once (LAYER_MODELS = {models})" + ); + } + } } diff --git a/crates/compositor/src/gif_export.rs b/crates/compositor/src/gif_export.rs index 8215704c2..7ab0c1b6c 100644 --- a/crates/compositor/src/gif_export.rs +++ b/crates/compositor/src/gif_export.rs @@ -256,6 +256,7 @@ fn export_gif_inner( let mut screen_decs: HashMap = HashMap::new(); let mut webcam_decs: HashMap = HashMap::new(); + let mut extra_decs: HashMap = HashMap::new(); screen_decs.insert(clips[0].screen.clone(), unsafe { Decoder::open_for_export(&clips[0].screen, gpu)? }); @@ -270,6 +271,7 @@ fn export_gif_inner( &scene, &mut screen_decs, &mut webcam_decs, + &mut extra_decs, &mut |frame_index| { control.check()?; // CPU readback of the staged RT (RGBA8 tightly-packed, diff --git a/crates/compositor/src/lib.rs b/crates/compositor/src/lib.rs index e26480cc6..67c233e4b 100644 --- a/crates/compositor/src/lib.rs +++ b/crates/compositor/src/lib.rs @@ -30,6 +30,7 @@ pub mod audio; pub mod audio_jobs; pub mod camera; +pub mod camera_layers; pub mod config; pub mod cursor; pub mod cursor_sdf; @@ -43,6 +44,7 @@ pub mod regions; // cibles) et le shim C. Seul Linux l'appelle aujourd'hui, parce que c'est la // seule plateforme dont la capture passe par `MediaRecorder`, mais rien dedans // n'est spécifique à Linux. +pub mod prism_mesh; pub mod remux; pub mod scene; pub mod sculpt; @@ -50,9 +52,12 @@ pub mod sculpt; // `segmentation` ses deux entrées échouent proprement, ce qui garde le reste du crate // indépendant du choix de packaging d'ONNX Runtime. pub mod segmentation; +// Livraison de la preview par textures partagées (Windows) ; la tenue des cases est portable. +pub mod shared_frames; pub mod text_anim; pub mod text_fonts; pub mod text_plate; +pub(crate) mod extra_cameras; pub(crate) mod timeline_walk; // GPU backend : Windows → d3d_windows, macOS → d3d_macos. Ré-exporté sous le nom `d3d` diff --git a/crates/compositor/src/live.rs b/crates/compositor/src/live.rs index ebcb46388..d8e2c4e87 100644 --- a/crates/compositor/src/live.rs +++ b/crates/compositor/src/live.rs @@ -23,13 +23,21 @@ //! le napi — le `gen` est l'identité de la frame (cf. `LatestFrame`). use crate::compositor::{Compositor, LiveParams}; +use crate::ffi::AVFrame; use crate::regions::{speed_at, ProgrammeClock}; -use crate::scene::Scene; +use crate::scene::{Scene, SceneCameraLayoutRegion, SceneClipCamera}; use crate::config::{self, Cfg}; use crate::cursor::CursorTrack; -use crate::d3d::Gpu; +use crate::d3d::{Backend, Gpu}; use crate::frame_geometry::webcam_is_real; use crate::pipeline::Decoder; +#[cfg(windows)] +use crate::shared_frames::SharedRing; +use crate::shared_frames::{SharedFrame, SlotBook}; +use crate::extra_cameras::{ + camera_source_time, cameras_in_regions, extra_camera_active, extra_camera_keys, extra_frame_list, + has_camera_file, opened_or_skipped, seek_extra_camera, step_extra_camera, ExtraCameraKeys, +}; use crate::timeline_walk::{frame_step, FrameStep, NextFrameTime}; use anyhow::Result; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; @@ -49,10 +57,6 @@ fn parse_hex_color(s: &str) -> Option<[f32; 4]> { Some([r, g, b, 1.0]) } -fn webcam_seek_time(screen_source_time_sec: f64, webcam_offset_sec: f64) -> f64 { - (screen_source_time_sec - webcam_offset_sec).max(0.0) -} - /// Décodeurs déjà ouverts ET positionnés au bon playhead pour un clip à venir — le résultat /// d'un préchargement en tâche de fond (voir `open_and_seek_clip`/`maybe_start_prefetch` /// dans `render_thread`). Appliquer ceci à un `Player` (`apply_prefetched`) ne fait plus @@ -69,6 +73,9 @@ struct PrefetchedClip { /// des paires ouvertes plusieurs bascules plus tôt. Cf. `open_webcam_or_stand_in`. webcam_decoder_is_real: bool, webcam_offset_sec: f64, + /// The clip's extra cameras (index k-1 = camera k) and what they were opened for. + extra: Vec>, + extra_keys: ExtraCameraKeys, idx: u32, /// Piste curseur du clip à venir, préchargée ici pour la même raison que les décodeurs : /// sans ça, la bascule à la frontière restait synchrone sur CE point précis (lecture + @@ -106,9 +113,13 @@ struct PrefetchedClip { /// `should_draw_webcam`). /// /// ponytail: on garde le remplaçant plutôt que de passer `wdec` en `Option`, ce qui -/// toucherait 22 sites dont le pool de décodeurs et la boucle de composition `unsafe`. À faire -/// si quelqu'un mesure que le décodeur inutile coûte (VRAM des pools D3D11VA, une ouverture -/// par clip) — l'avertissement ci-dessous dit enfin à quelle fréquence le cas visible arrive. +/// toucherait 22 sites dont le pool de décodeurs et la boucle de composition `unsafe`. Il est +/// ouvert mais n'est plus jamais avancé ni recherché : la composition reçoit la frame écran à +/// sa place (`Player::webcam_frame`), l'image même qu'il aurait décodée. Le décoder à côté de +/// l'écran coûtait la moitié du débit de décodage, sans caméra, c'est-à-dire le cas courant : +/// une source 4K lue à 2× plafonnait à ~46 frames/s pour 97 que le décodeur seul tient, et la +/// preview prenait des secondes de retard dans les régions de vitesse. Reste à faire si la +/// VRAM des pools D3D11VA ou l'ouverture par clip se mesurent. unsafe fn open_webcam_or_stand_in( screen_path: &str, webcam_path: &str, @@ -142,27 +153,139 @@ fn should_draw_webcam(webcam_path: &str, screen_path: &str, decoder_is_real: boo webcam_is_real(webcam_path, screen_path) && decoder_is_real } +/// An extra camera (k >= 1) of the active clip, decoded only near the layout regions that show +/// it (`extra_camera_active`). +struct ExtraCamera { + dec: Decoder, + offset_sec: f64, + /// The last step or seek left a frame to draw. Read by `Player::recompose`. + shown: bool, + /// Camera source time past which the file has no frame left (`step_extra_camera`). + ended_at: Option, +} + +impl ExtraCamera { + /// `step_extra_camera` toward `target` (camera source time). + unsafe fn step_to(&mut self, target: f64) -> Result<*const AVFrame> { + let frame = step_extra_camera(&mut self.dec, &mut self.ended_at, target)?; + self.shown = !frame.is_null(); + Ok(frame) + } + + /// `seek_extra_camera` to `target` (camera source time). + unsafe fn seek(&mut self, target: f64) -> Result<*const AVFrame> { + let frame = seek_extra_camera(&mut self.dec, &mut self.ended_at, target)?; + self.shown = !frame.is_null(); + Ok(frame) + } +} + +/// The extra cameras clip `clip_index` opens: those its own layout regions +/// (`Scene::for_clip_window`) draw and that the clip has a file for. +fn extra_cameras_for_clip(scene: &Scene, clip_index: usize) -> Vec { + let Some(clip) = scene.clips.get(clip_index) else { + return Vec::new(); + }; + // The common case — no layout region anywhere — skips the windowing copy of the scene. + if scene.camera_layout_regions.is_empty() || clip.additional_cameras.is_empty() { + return Vec::new(); + } + cameras_in_regions(&scene_for_clip(scene, clip_index).camera_layout_regions) + .into_iter() + .filter(|&k| has_camera_file(clip.additional_cameras.get(k - 1))) + .collect() +} + +/// `extra_camera_keys` for clip `clip_index` of `scene`, from the clip's own camera files. +fn scene_extra_camera_keys(scene: &Scene, clip_index: usize) -> ExtraCameraKeys { + let sources = scene.clips.get(clip_index).map(|c| c.additional_cameras.as_slice()).unwrap_or(&[]); + extra_camera_keys(&extra_cameras_for_clip(scene, clip_index), sources) +} + +/// The windowed layout regions of clip `clip_index` (what `camera_layers_at` plans from). +fn clip_layout_regions(scene: &Scene, clip_index: usize) -> Vec { + if scene.camera_layout_regions.is_empty() { + return Vec::new(); + } + scene_for_clip(scene, clip_index).camera_layout_regions +} + +fn same_extra_key(a: &Option, b: &Option) -> bool { + match (a, b) { + (None, None) => true, + (Some(a), Some(b)) => a.path == b.path && (a.offset_sec - b.offset_sec).abs() < 1e-9, + _ => false, + } +} + +fn same_extra_keys(a: &[Option], b: &[Option]) -> bool { + a.len() == b.len() && a.iter().zip(b).all(|(a, b)| same_extra_key(a, b)) +} + +/// Same spans and same cameras in each: what decides when an extra camera is active +/// (`extra_camera_active`). Rects and shapes do not matter to the decoders. +fn same_region_timing(a: &[SceneCameraLayoutRegion], b: &[SceneCameraLayoutRegion]) -> bool { + a.len() == b.len() + && a.iter().zip(b).all(|(a, b)| { + a.clip_index == b.clip_index + && a.start_sec == b.start_sec + && a.end_sec == b.end_sec + && a.layers.len() == b.layers.len() + && a.layers.iter().zip(&b.layers).all(|(x, y)| x.camera == y.camera) + }) +} + +/// Whether `set_extra_cameras` seeks the extra cameras right away. Playing on (`steady`: +/// playing, and the screen position did not jump) with the same files and the same region +/// timing, the next playback step (`advance_extras(.., false)`) moves them on: seeking every +/// camera on each scene push would stall the preview while it plays. +fn extras_need_seek(keys_changed: bool, timing_changed: bool, steady: bool) -> bool { + keys_changed || timing_changed || !steady +} + +/// Opens the extra cameras `keys` asks for, without seeking them: they are positioned when +/// they become active (`Player::advance_extras`). +unsafe fn open_extra_cameras(keys: &[Option], gpu: &Gpu) -> Vec> { + keys.iter() + .map(|key| { + let key = key.as_ref()?; + let dec = opened_or_skipped(&key.path, Decoder::open(&key.path, gpu))?; + Some(ExtraCamera { dec, offset_sec: key.offset_sec, shown: false, ended_at: None }) + }) + .collect() +} + unsafe fn open_and_seek_clip( screen_path: &str, webcam_path: &str, webcam_offset_sec: f64, + extra_keys: &[Option], source_time_sec: f64, gpu: &Gpu, ) -> Result { let source_time_sec = source_time_sec.max(0.0); let mut sdec = Decoder::open(screen_path, gpu)?; let (mut wdec, webcam_decoder_is_real) = open_webcam_or_stand_in(screen_path, webcam_path, gpu)?; - let sf = sdec.seek_to(source_time_sec)?; - let mut wf = wdec.seek_to(webcam_seek_time(source_time_sec, webcam_offset_sec))?; - if wf.is_null() { - wf = wdec.seek_to(0.0)?; + let extra = open_extra_cameras(extra_keys, gpu); + let sf = sdec.seek_to_or_last(source_time_sec)?; + if webcam_decoder_is_real && wdec.seek_to(camera_source_time(source_time_sec, webcam_offset_sec))?.is_null() { + wdec.seek_to(0.0)?; } if sf.is_null() { anyhow::bail!("clip préchargé vide au temps source {source_time_sec:.3}s (screen=\"{screen_path}\")"); } let idx = (source_time_sec * sdec.fps()).round().max(0.0) as u32; let cursor_track = CursorTrack::load(&format!("{screen_path}.cursor.json"), 0.0, 24.0 * 3600.0).ok(); - Ok(PrefetchedClip { sdec, wdec, webcam_decoder_is_real, webcam_offset_sec, idx, cursor_track }) + Ok(PrefetchedClip { + sdec, + wdec, + webcam_decoder_is_real, + webcam_offset_sec, + extra, + extra_keys: extra_keys.to_vec(), + idx, + cursor_track, + }) } /// Nombre de paires de décodeurs INACTIVES gardées ouvertes en plus de la paire active. @@ -174,11 +297,29 @@ unsafe fn open_and_seek_clip( /// les timelines 2-4 clips, à baisser si la VRAM serre. const DECODER_POOL_CAP: usize = 3; -/// Une paire de décodeurs mise de côté, prête à être réactivée sans réouverture. -struct PooledClip { +/// What a set of open decoders was opened for: the screen, camera 0 and the extra cameras. +/// A pooled set is reused only for the same key, extra cameras included — a pooled pair +/// opened without camera 2 must not stand in for a clip whose layout shows it. +#[derive(Clone)] +struct ClipKey { screen_path: String, webcam_path: String, webcam_offset_sec: f64, + extras: ExtraCameraKeys, +} + +impl ClipKey { + fn matches(&self, other: &ClipKey) -> bool { + self.screen_path == other.screen_path + && self.webcam_path == other.webcam_path + && (self.webcam_offset_sec - other.webcam_offset_sec).abs() < 1e-9 + && same_extra_keys(&self.extras, &other.extras) + } +} + +/// Une paire de décodeurs mise de côté, prête à être réactivée sans réouverture. +struct PooledClip { + key: ClipKey, clip: PrefetchedClip, } @@ -189,14 +330,19 @@ struct PooledClip { unsafe fn seek_pair( sdec: &mut Decoder, wdec: &mut Decoder, + webcam_decoder_is_real: bool, source_time_sec: f64, webcam_offset_sec: f64, ) -> Result { - let sf = sdec.seek_to(source_time_sec)?; + let sf = sdec.seek_to_or_last(source_time_sec)?; if sf.is_null() { return Ok(false); } - let mut wf = wdec.seek_to(webcam_seek_time(source_time_sec, webcam_offset_sec))?; + // Le remplaçant n'est jamais lu (`Player::webcam_frame`) : rien à positionner. + if !webcam_decoder_is_real { + return Ok(true); + } + let mut wf = wdec.seek_to(camera_source_time(source_time_sec, webcam_offset_sec))?; if wf.is_null() { wf = wdec.seek_to(0.0)?; } @@ -215,50 +361,41 @@ unsafe fn seek_pair( unsafe fn swap_clip_pooled( player: &mut Player, pool: &mut Vec, - request: &ActiveClipRequest, - active_screen: &str, - active_webcam: &str, - active_webcam_offset_sec: f64, + request: &ClipKey, + source_time_sec: f64, + active: &ClipKey, ) -> Result<()> { let timing = std::env::var("OPENSCREEN_CLIPSWITCH_TIMING").is_ok(); let t0 = std::time::Instant::now(); - let t = request.source_time_sec.max(0.0); - let matches = |p: &PooledClip| { - p.screen_path == request.screen_path - && p.webcam_path == request.webcam_path - && (p.webcam_offset_sec - request.webcam_offset_sec).abs() < 1e-9 - }; + let t = source_time_sec.max(0.0); let mut hit = false; - let incoming: PrefetchedClip = match pool.iter().position(&matches) { + let incoming: PrefetchedClip = match pool.iter().position(|p| p.key.matches(request)) { Some(i) => { let mut pooled = pool.remove(i); // Reseek les décodeurs poolés AVANT de les installer. Échec → on les jette et on // ouvre à neuf (chemin connu sûr), jamais une frame vide. - if seek_pair(&mut pooled.clip.sdec, &mut pooled.clip.wdec, t, request.webcam_offset_sec)? { + if seek_pair( + &mut pooled.clip.sdec, + &mut pooled.clip.wdec, + pooled.clip.webcam_decoder_is_real, + t, + request.webcam_offset_sec, + )? { pooled.clip.idx = (t * pooled.clip.sdec.fps()).round().max(0.0) as u32; hit = true; pooled.clip } else { drop(pooled); - player.open_clip(&request.screen_path, &request.webcam_path, request.webcam_offset_sec, t)? + player.open_clip(request, t)? } } - None => player.open_clip(&request.screen_path, &request.webcam_path, request.webcam_offset_sec, t)?, + None => player.open_clip(request, t)?, }; let outgoing = player.swap_active(incoming); // Met la paire quittée en pool : dédup par clé (jamais deux entrées d'un même média), puis // éviction LRU (le plus ancien, en tête, part en premier). - pool.retain(|p| { - !(p.screen_path == active_screen - && p.webcam_path == active_webcam - && (p.webcam_offset_sec - active_webcam_offset_sec).abs() < 1e-9) - }); - pool.push(PooledClip { - screen_path: active_screen.to_string(), - webcam_path: active_webcam.to_string(), - webcam_offset_sec: active_webcam_offset_sec, - clip: outgoing, - }); + pool.retain(|p| !p.key.matches(active)); + pool.push(PooledClip { key: active.clone(), clip: outgoing }); while pool.len() > DECODER_POOL_CAP { pool.remove(0); } @@ -285,6 +422,12 @@ pub struct Player { /// (`swap_active`), lu par la boucle de rendu pour décider de dessiner la vignette. webcam_decoder_is_real: bool, webcam_offset_sec: f64, + /// The active clip's extra cameras (index k-1 = camera k), what they were opened for, and + /// the clip's windowed layout regions that decide when each one is decoded. All empty + /// without layout regions: no decoder, nothing to advance. + extra: Vec>, + extra_keys: ExtraCameraKeys, + camera_regions: Vec, has_current_frame: bool, use_current_on_next_step: bool, idx: u32, @@ -307,6 +450,9 @@ impl Player { }, webcam_decoder_is_real, webcam_offset_sec: 0.0, + extra: Vec::new(), + extra_keys: Vec::new(), + camera_regions: Vec::new(), has_current_frame: false, use_current_on_next_step: false, idx: 0, @@ -325,10 +471,17 @@ impl Player { screen_path: &str, webcam_path: &str, webcam_offset_sec: f64, + extra_keys: &[Option], source_time_sec: f64, ) -> Result<()> { - let prefetched = - open_and_seek_clip(screen_path, webcam_path, webcam_offset_sec, source_time_sec, &self.gpu)?; + let prefetched = open_and_seek_clip( + screen_path, + webcam_path, + webcam_offset_sec, + extra_keys, + source_time_sec, + &self.gpu, + )?; self.apply_prefetched(prefetched); Ok(()) } @@ -359,7 +512,13 @@ impl Player { // Les DEUX flux doivent avoir une frame : `compose_frame` les échantillonne tous les // deux sans condition, un seul manquant suffit à le faire échouer (d'où le `false` que // `seek_pair` peut rendre → l'appelant retombe sur l'ouverture complète). - if !seek_pair(&mut self.sdec, &mut self.wdec, source_time_sec, self.webcam_offset_sec)? { + if !seek_pair( + &mut self.sdec, + &mut self.wdec, + self.webcam_decoder_is_real, + source_time_sec, + self.webcam_offset_sec, + )? { return Ok(false); } self.idx = (source_time_sec * self.sdec.fps()).round().max(0.0) as u32; @@ -389,6 +548,8 @@ impl Player { // recalculer), l'entrante impose la sienne au player. webcam_decoder_is_real: self.webcam_decoder_is_real, webcam_offset_sec: self.webcam_offset_sec, + extra: std::mem::replace(&mut self.extra, incoming.extra), + extra_keys: std::mem::replace(&mut self.extra_keys, incoming.extra_keys), idx: self.idx, // Le curseur est re-dérivé du chemin à la réactivation ; inutile de le trimballer. cursor_track: None, @@ -403,14 +564,107 @@ impl Player { /// Ouvre une nouvelle paire de décodeurs positionnée à `source_time_sec`, SANS l'installer /// (l'appelant l'échange via `swap_active`). Réutilise le device D3D11 du player. - unsafe fn open_clip( - &self, - screen: &str, - webcam: &str, - webcam_offset_sec: f64, - source_time_sec: f64, - ) -> Result { - open_and_seek_clip(screen, webcam, webcam_offset_sec, source_time_sec, &self.gpu) + unsafe fn open_clip(&self, key: &ClipKey, source_time_sec: f64) -> Result { + open_and_seek_clip( + &key.screen_path, + &key.webcam_path, + key.webcam_offset_sec, + &key.extras, + source_time_sec, + &self.gpu, + ) + } + + /// What the active extra cameras were opened for (the extra half of the pool key). + fn extra_keys(&self) -> &[Option] { + &self.extra_keys + } + + /// Installs the active clip's windowed layout regions and the extra cameras they need + /// (`keys`): slots whose key is unchanged keep their decoder, the others are closed and + /// (re)opened — a file that will not open leaves its slot empty, once. Then positions every + /// camera that is near one of its regions at the current screen time, so a paused + /// recompose shows it at once — unless nothing that matters to the decoders changed while + /// playback goes on (`steady`, see `extras_need_seek`). `true` when a decoder was closed: + /// the caller must then clear the compositor's SRV cache (which also forgets the extra + /// frame pointers). + pub(crate) unsafe fn set_extra_cameras( + &mut self, + regions: Vec, + keys: ExtraCameraKeys, + steady: bool, + ) -> bool { + let timing_changed = !same_region_timing(&self.camera_regions, ®ions); + let keys_changed = !same_extra_keys(&self.extra_keys, &keys); + self.camera_regions = regions; + let mut closed = false; + if keys_changed { + let mut old = std::mem::take(&mut self.extra); + let mut extra = Vec::with_capacity(keys.len()); + for (k, key) in keys.iter().enumerate() { + let old_cam = old.get_mut(k).and_then(Option::take); + let old_key = self.extra_keys.get(k).cloned().flatten(); + if same_extra_key(&old_key, key) { + extra.push(old_cam); + continue; + } + closed |= old_cam.is_some(); + drop(old_cam); + extra.extend(open_extra_cameras(std::slice::from_ref(key), &self.gpu)); + } + closed |= old.iter().any(Option::is_some); + self.extra = extra; + self.extra_keys = keys; + } + if !extras_need_seek(keys_changed, timing_changed, steady) { + return closed; + } + let t = self.sdec.cur_time_sec(); + closed | self.advance_extras(t, true).1 + } + + /// The extra cameras' frames at screen source time `screen_t`, each camera stepped + /// (`seek == false`) or sought toward its own source time only while it is near one of + /// its regions; an idle camera is left alone and gives null. A camera whose decoder fails + /// is closed with one warning — its layer disappears, the preview goes on. The `bool` says + /// a decoder was closed (see `set_extra_cameras`). + unsafe fn advance_extras(&mut self, screen_t: f64, seek: bool) -> (Vec<*const AVFrame>, bool) { + let mut closed = false; + let mut frames = Vec::with_capacity(self.extra.len()); + for (k, slot) in self.extra.iter_mut().enumerate() { + let camera = k + 1; + let frame = match slot { + Some(cam) if extra_camera_active(&self.camera_regions, camera, screen_t) => { + let target = camera_source_time(screen_t, cam.offset_sec); + let result = if seek { cam.seek(target) } else { cam.step_to(target) }; + match result { + Ok(frame) => frame, + Err(e) => { + eprintln!("WARNING: extra camera {camera} stopped decoding: {e:#}. Its layer will not be drawn."); + *slot = None; + closed = true; + std::ptr::null() + } + } + } + Some(cam) => { + cam.shown = false; + std::ptr::null() + } + None => std::ptr::null(), + }; + frames.push(frame); + } + (frames, closed) + } + + /// Hands the extra cameras' frames to `comp` for the next `compose_frame`. Called before + /// every compose: the compositor keeps the pointers until the next call. + unsafe fn hand_extra_frames(comp: &Compositor, frames: &[*const AVFrame], closed: bool) { + if closed { + comp.clear_srv_cache(); + } + comp.set_extra_camera_frames(frames); } /// Le décodeur webcam ACTIF est-il la vraie caméra ? `false` quand c'est le remplaçant @@ -421,6 +675,17 @@ impl Player { self.webcam_decoder_is_real } + /// La frame webcam à composer avec la frame écran `screen` : celle de la caméra, ou sans + /// caméra la frame écran elle-même, que le remplaçant aurait décodée à l'identique et que + /// la composition ne dessine pas (`should_draw_webcam`). + unsafe fn webcam_frame(&self, screen: *mut crate::ffi::AVFrame) -> *mut crate::ffi::AVFrame { + if self.webcam_decoder_is_real { + self.wdec.cur_frame() + } else { + screen + } + } + /// Temps source courant du décodeur écran — utilisé par `render_thread` pour détecter le /// franchissement de la fin de fenêtre du clip actif pendant la lecture libre, et pour /// calculer la cible de `step` en lecture libre. `pub` (pas `pub(crate)`) : le harnais @@ -429,6 +694,12 @@ impl Player { self.sdec.cur_time_sec() } + /// Temps source courant du décodeur webcam, le remplaçant compris : sans caméra il ne doit + /// jamais bouger (`tests/no_camera_stand_in.rs`). + pub unsafe fn webcam_time_sec(&self) -> f64 { + self.wdec.cur_time_sec() + } + /// Recalcule l'horloge programme pour le clip `clip_index` de `scene` (la scène COMPLÈTE, /// pas la fenêtre d'un clip : il faut la durée de sortie des clips précédents). /// @@ -509,7 +780,10 @@ impl Player { } let target_webcam_t = (self.sdec.cur_time_sec() - self.webcam_offset_sec).max(0.0); - let wf = if use_current { + let wf = if !self.webcam_decoder_is_real { + // Pas de caméra : le remplaçant n'est pas décodé (cf. `open_webcam_or_stand_in`). + sf + } else if use_current { self.wdec.cur_frame() } else { let cur = self.wdec.cur_frame(); @@ -557,6 +831,8 @@ impl Player { self.has_current_frame = true; self.sync_time(comp); + let (extra, closed) = self.advance_extras(self.sdec.cur_time_sec(), false); + Self::hand_extra_frames(comp, &extra, closed); comp.compose_frame(sf, wf, self.idx as f32, cfg)?; self.idx = self.idx.wrapping_add(1); Ok(true) @@ -582,11 +858,15 @@ impl Player { return Ok(false); } let sf = self.sdec.cur_frame(); - let wf = self.wdec.cur_frame(); + let wf = self.webcam_frame(sf); if sf.is_null() || wf.is_null() { return Ok(false); } self.sync_time(comp); + let extra = extra_frame_list(&self.extra, |cam| { + if cam.shown { cam.dec.cur_frame() as *const AVFrame } else { std::ptr::null() } + }); + comp.set_extra_camera_frames(&extra); let f = self.idx.saturating_sub(1); comp.compose_frame(sf, wf, f as f32, cfg)?; Ok(true) @@ -598,10 +878,15 @@ impl Player { /// aucun raccourci keyframe pour les seeks avant lointains (lent ET, combiné au bug de /// `set_time`, incorrect au-delà de 6s sur un enregistrement réel). pub unsafe fn present_frame(&mut self, comp: &Compositor, cfg: &Cfg, target_sec: f64) -> Result { - let sf = self.sdec.seek_to(target_sec)?; - let wf = self - .wdec - .seek_to(webcam_seek_time(target_sec, self.webcam_offset_sec))?; + let sf = self.sdec.seek_to_or_last(target_sec)?; + // La caméra aussi, au-delà de sa dernière image comprise : sans frame webcam, la frame + // ne serait pas composée. Sans caméra, la frame écran tient sa place. + let wf = if self.webcam_decoder_is_real { + self.wdec + .seek_to_or_last(camera_source_time(target_sec, self.webcam_offset_sec))? + } else { + sf + }; if sf.is_null() || wf.is_null() { self.has_current_frame = false; return Ok(false); @@ -612,6 +897,8 @@ impl Player { // "idx" ne sert plus qu'au fallback fixture (jamais lu si une scène est posée) — dérivé // du temps réel pour rester cohérent si jamais consulté. self.idx = (target_sec * self.sdec.fps()).round().max(0.0) as u32; + let (extra, closed) = self.advance_extras(target_sec, true); + Self::hand_extra_frames(comp, &extra, closed); comp.compose_frame(sf, wf, self.idx as f32, cfg)?; Ok(true) } @@ -690,6 +977,9 @@ struct ActiveClipRequest { screen_path: String, webcam_path: String, webcam_offset_sec: f64, + /// Cameras 2-4 of the clip (index k-1 = camera k); which of them open is decided by the + /// clip's layout regions (`extra_cameras_for_clip`). + additional_cameras: Vec, /// Identité dans le flux `Scene.clips` trié (les chemins ne suffisent pas pour un asset partagé). clip_index: usize, /// Playhead exprimé sur l'horloge source écran du nouveau clip. @@ -756,15 +1046,34 @@ fn scene_for_clip(scene: &Scene, clip_index: usize) -> Scene { /// Dernière frame readback vers CPU, prête pour le napi `read_frame`. /// -/// `(gen, w, h, vec)` où `vec.len() == w*h*4` octets RGBA8 tightly-packed (R, G, B, A -/// en mémoire — cf. `Compositor::readback_resized`). `gen` est une génération monotone +/// `(gen, w, h, vec, métrage, position)` où `vec.len() == w*h*4` octets RGBA8 tightly-packed +/// (R, G, B, A en mémoire — cf. `Compositor::readback_resized`). `gen` est une génération monotone /// (≥ 1, `0` réservé à « le consommateur n'a encore rien vu ») incrémentée à CHAQUE /// publication, càd uniquement quand une nouvelle frame a réellement été composée (le /// thread de rendu ne republie pas une frame identique — cf. `stepped || first`). Elle /// est l'IDENTITÉ de la frame : le consommateur (`read_frame`) ne repaie le clone + l'IPC /// que lorsqu'elle change. `None` = "aucune frame composée pour l'instant" (toutes les /// lectures avant la 1re frame composée retournent `None` côté napi, jamais un buffer vide). -type LatestFrame = (u64, u32, u32, Vec); +pub type LatestFrame = ( + u64, + u32, + u32, + Vec, + Option, + FramePosition, +); + +/// Où en est la vue quand elle compose une frame : le clip actif de la scène et le temps +/// source de la frame écran. Voyage avec chaque frame publiée, quel que soit le transport, +/// pour que l'app compare la position RÉELLE de la vue à sa propre tête de lecture au lieu +/// de la deviner à partir de l'horloge murale. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FramePosition { + /// Index du clip actif dans `scene.clips`. + pub clip_index: u32, + /// pts de la frame écran composée, en secondes du fichier source. + pub source_time_sec: f64, +} /// État partagé thread appelant → thread de rendu (commandes sans blocage). struct Shared { @@ -800,6 +1109,15 @@ struct Shared { /// vide la ferait repartir à 1 — donc rejouer des générations déjà peintes. Monotone, /// jamais remise à zéro. frame_gen: AtomicU64, + /// La vue livre ses frames dans l'anneau de textures partagées (`take_shared_frame`) au + /// lieu de les relire en RAM (`latest_frame_since`). Posé par JS ; le thread de rendu le + /// rabat à `false` si l'anneau ne peut pas servir, et les frames repassent par la RAM. + shared_frames: AtomicBool, + /// Tenue des cases de l'anneau, entre le thread de rendu et le thread Node. + slot_book: Mutex, + /// Republier la frame courante au prochain tour, même en pause : posé quand le transport + /// change, pour que le consommateur ait une image sans attendre que quelque chose bouge. + republish: AtomicBool, /// Erreur fatale du thread de rendu (device D3D11 introuvable, décodeur qui refuse /// le fichier…). Le thread meurt sur la première erreur ; sans ce champ, elle /// finissait dans un `eprintln!` que personne ne lit et l'utilisateur n'avait @@ -853,6 +1171,9 @@ impl LiveView { stop: AtomicBool::new(false), latest_frame: Mutex::new(None), frame_gen: AtomicU64::new(0), + shared_frames: AtomicBool::new(false), + slot_book: Mutex::new(SlotBook::default()), + republish: AtomicBool::new(false), fatal: Mutex::new(None), }); let sh = shared.clone(); @@ -894,7 +1215,7 @@ impl LiveView { /// FFI vers le Buffer napi). Le `Vec` retourné a `len() == w*h*4`. /// Préférer `latest_frame_since` sur le chemin chaud : il évite ce clone quand /// le consommateur possède déjà la génération courante. - pub fn latest_frame(&self) -> Option<(u64, u32, u32, Vec)> { + pub fn latest_frame(&self) -> Option { self.shared .latest_frame .lock() @@ -910,7 +1231,7 @@ impl LiveView { /// frame figée — on n'exécute PAS le clone `O(w·h)` : c'est tout l'intérêt du /// compteur. Le consommateur passe la dernière génération qu'il a peinte (`0` au /// départ) ; `None` ⇒ il ne fait rien, `Some` ⇒ il peint et retient `gen`. - pub fn latest_frame_since(&self, since_gen: u64) -> Option<(u64, u32, u32, Vec)> { + pub fn latest_frame_since(&self, since_gen: u64) -> Option { let mut guard = self.shared.latest_frame.lock().ok()?; match guard.as_ref() { // Le buffer est EMPORTÉ, pas copié. Le thread de rendu le remplace à chaque @@ -929,6 +1250,35 @@ impl LiveView { } } + /// Livrer les frames par textures partagées plutôt que par readback. Seul le backend + /// matériel de Windows sait le faire : ailleurs la demande est refusée et la vue continue + /// de relire en RAM. Rend l'état obtenu. + pub fn set_shared_frames(&self, enabled: bool) -> bool { + let on = enabled && cfg!(windows) && Gpu::probe() == Some(Backend::Hardware); + if self.shared.shared_frames.swap(on, Ordering::Relaxed) != on { + if !on { + if let Ok(mut book) = self.shared.slot_book.lock() { + book.clear_ready(); + } + } + self.shared.republish.store(true, Ordering::Relaxed); + } + on + } + + /// La dernière frame posée dans l'anneau, si elle est plus récente que `since_gen`. Sa + /// case reste tenue jusqu'à `release_shared_frame`. `None` aussi quand la vue relit en RAM. + pub fn take_shared_frame(&self, since_gen: u64) -> Option { + self.shared.slot_book.lock().ok()?.take(since_gen, Instant::now()) + } + + /// Chromium a relâché la frame `gen` de la case `slot` : le thread de rendu peut y réécrire. + pub fn release_shared_frame(&self, slot: u32, gen: u64) { + if let Ok(mut book) = self.shared.slot_book.lock() { + book.release(slot, gen); + } + } + /// Switch inspector (booléen). pub fn set_param_bool(&self, key: &str, value: bool) { if let Ok(mut p) = self.shared.inspector.lock() { @@ -1026,6 +1376,7 @@ impl LiveView { screen_path: &str, webcam_path: &str, webcam_offset_sec: f64, + additional_cameras: Vec, clip_index: usize, source_time_sec: f64, ) { @@ -1034,6 +1385,7 @@ impl LiveView { screen_path: screen_path.to_string(), webcam_path: webcam_path.to_string(), webcam_offset_sec, + additional_cameras, clip_index, source_time_sec: source_time_sec.max(0.0), }); @@ -1132,7 +1484,7 @@ type PendingPrefetch = (usize, std::sync::mpsc::Receiver> /// tâche de fond. Assez large pour couvrir un `Decoder::open` typique (ouverture fichier + /// `avformat_find_stream_info` + init D3D11VA), assez court pour ne pas garder deux paires de /// décodeurs ouvertes plus longtemps que nécessaire. -const PREFETCH_LEAD_SEC: f64 = 0.75; +pub(crate) const PREFETCH_LEAD_SEC: f64 = 0.75; /// Durée pendant laquelle la boucle continue de recomposer après un changement en pause, le /// temps qu'un effet asynchrone (segmentation webcam) livre son résultat. Généreuse : à @@ -1186,6 +1538,7 @@ unsafe fn maybe_start_prefetch( 0 }; let next_clip = scene.clips[next_index].clone(); + let extra_keys = scene_extra_camera_keys(scene, next_index); // Copie légère (COM refcount, pas de nouveau device) — même motif que `Player::open`. let gpu_clone = Gpu { device: gpu.device.clone(), @@ -1200,6 +1553,7 @@ unsafe fn maybe_start_prefetch( &next_clip.screen_path, &next_clip.webcam_path, next_clip.webcam_offset_sec, + &extra_keys, next_clip.source_start_sec, &gpu_clone, ) @@ -1237,6 +1591,7 @@ unsafe fn advance_to_next_scene_clip( active_screen_path: &mut String, active_webcam_path: &mut String, active_webcam_offset_sec: &mut f64, + active_additional_cameras: &mut Vec, active_clip_index: &mut usize, raw_cursor: &mut Option, loaded_cursor_path: &mut String, @@ -1251,6 +1606,7 @@ unsafe fn advance_to_next_scene_clip( 0 }; let next_clip = &scene.clips[next_index]; + let extra_keys = scene_extra_camera_keys(scene, next_index); // N'importe quel préchargement en cours ne concerne plus que CETTE frontière (on vient // de la franchir, bien ou mal ciblée) — on le consomme s'il correspond, on l'abandonne @@ -1278,6 +1634,7 @@ unsafe fn advance_to_next_scene_clip( &next_clip.screen_path, &next_clip.webcam_path, next_clip.webcam_offset_sec, + &extra_keys, next_clip.source_start_sec, ) } @@ -1285,6 +1642,7 @@ unsafe fn advance_to_next_scene_clip( &next_clip.screen_path, &next_clip.webcam_path, next_clip.webcam_offset_sec, + &extra_keys, next_clip.source_start_sec, ), }; @@ -1296,12 +1654,18 @@ unsafe fn advance_to_next_scene_clip( // sur l'adresse de la texture, et garder des entrées d'un décodeur fermé fait // fuir de la VRAM puis, en cas de réutilisation d'adresse, rendre l'image du clip // précédent. + let windowed = scene_for_clip(scene, next_index); + // The prefetch opened exactly these keys, so this only installs the regions and + // positions the cameras; the cache is cleared right below in any case. + // The screen position jumped to the next clip: always position the cameras. + player.set_extra_cameras(windowed.camera_layout_regions.clone(), extra_keys, false); comp.clear_srv_cache(); *active_screen_path = next_clip.screen_path.clone(); *active_webcam_path = next_clip.webcam_path.clone(); *active_webcam_offset_sec = next_clip.webcam_offset_sec; + *active_additional_cameras = next_clip.additional_cameras.clone(); *active_clip_index = next_index; - comp.set_scene(Some(scene_for_clip(scene, *active_clip_index))); + comp.set_scene(Some(windowed)); player.set_programme_clock(Some(scene), *active_clip_index); // Réutilise le curseur préchargé s'il est disponible (voir plus haut) — sinon // (préchargement pas encore prêt / raté) on retombe sur la lecture synchrone @@ -1379,6 +1743,9 @@ unsafe fn render_thread( let mut active_screen_path = screen.to_string(); let mut active_webcam_path = webcam.to_string(); let mut active_webcam_offset_sec = 0.0f64; + // Cameras 2-4 of the active clip, as the app (`set_active_clip`) or the scene (auto-advance) + // gave them. The view starts without: `create_view` carries camera 0 only. + let mut active_additional_cameras: Vec = Vec::new(); let mut active_clip_index = 0usize; // Copie de la Scene complète (tous les clips), tenue à jour à chaque push de l'app — // permet à la boucle de lecture libre de connaître la fenêtre source @@ -1426,8 +1793,19 @@ unsafe fn render_thread( // appliquée, on refuse de jouer le layout fixture (POC) : un fallback fixture ne ferait que // MASQUER un scene-push cassé. On attend la scène avant de produire le 1er frame. let mut scene_applied = false; + // Textures partagées de la preview, créées au premier besoin (voir `publish_shared`). + let mut ring = Ring::default(); + // Une frame composée qui n'a trouvé aucune case libre dans l'anneau : le RT la garde, elle + // repart dès qu'une case se libère. Sans ça, une frame composée en pause (seek, réglage) + // restait invisible jusqu'au changement suivant — en lecture, la suivante la remplace. + let mut publish_pending = false; while !shared.stop.load(Ordering::SeqCst) { + // Le transport vient de changer : le consommateur doit recevoir la frame courante + // sans attendre qu'une autre soit composée (en pause, il n'y en aurait pas). + if shared.republish.swap(false, Ordering::Relaxed) { + first = true; + } // params inspector : booléens/taps → cfg ; valeurs continues → live_params let ip = *shared.inspector.lock().unwrap(); let mut clip_changed = false; @@ -1448,6 +1826,26 @@ unsafe fn render_thread( // JAMAIS fatal : on retombe sur l'ouverture complète, chemin connu comme sûr. // L'optimisation ne s'applique donc que là où elle fonctionne démontrablement. let repositioned = same_media && matches!(player.seek_active(request.source_time_sec), Ok(true)); + // The extra cameras this clip's layout regions show, from the files the request + // carries. Resolved against the scene before the switch so that the pool and the + // fresh open both see the full key. + let request_scene = shared.scene.lock().unwrap().clone(); + let request_clip_index = request_scene.as_ref().and_then(|scene| { + resolve_scene_clip_index( + scene, + request.clip_index, + &request.screen_path, + &request.webcam_path, + request.webcam_offset_sec, + ) + }); + let request_extras = match (&request_scene, request_clip_index) { + (Some(scene), Some(index)) => extra_camera_keys( + &extra_cameras_for_clip(scene, index), + &request.additional_cameras, + ), + _ => Vec::new(), + }; let switch_result = if repositioned { Ok(()) } else { @@ -1455,13 +1853,24 @@ unsafe fn render_thread( // ouverte si possible (reseek au lieu de rouvrir) et met en pool celle qu'on // quitte, au lieu du couple ouvrir-puis-fermer. Voir `swap_clip_pooled` et la // mesure de ~120 ms/franchissement qui l'a motivé. + let incoming = ClipKey { + screen_path: request.screen_path.clone(), + webcam_path: request.webcam_path.clone(), + webcam_offset_sec: request.webcam_offset_sec, + extras: request_extras.clone(), + }; + let active = ClipKey { + screen_path: active_screen_path.clone(), + webcam_path: active_webcam_path.clone(), + webcam_offset_sec: active_webcam_offset_sec, + extras: player.extra_keys().to_vec(), + }; swap_clip_pooled( &mut player, &mut decoder_pool, - &request, - &active_screen_path, - &active_webcam_path, - active_webcam_offset_sec, + &incoming, + request.source_time_sec, + &active, ) }; match switch_result { @@ -1483,8 +1892,18 @@ unsafe fn render_thread( active_screen_path = request.screen_path; active_webcam_path = request.webcam_path; active_webcam_offset_sec = request.webcam_offset_sec; - let scene = shared.scene.lock().unwrap().clone(); + active_additional_cameras = request.additional_cameras; + let scene = request_scene; full_scene = scene.clone(); + // No scene clip for the request → no layout regions (and no extras). + let regions = match (&scene, request_clip_index) { + (Some(s), Some(index)) => clip_layout_regions(s, index), + _ => Vec::new(), + }; + // A clip request moves the screen position: position the cameras. + if player.set_extra_cameras(regions, request_extras, false) { + comp.clear_srv_cache(); + } if let Some(base_scene) = scene { if let Some(index) = resolve_scene_clip_index( &base_scene, @@ -1590,6 +2009,7 @@ unsafe fn render_thread( prefetch = None; let scene = shared.scene.lock().unwrap().clone(); full_scene = scene.clone(); + let mut clip_resolved = false; let scene = scene.map(|base_scene| { scene_applied = true; if let Some(index) = resolve_scene_clip_index( @@ -1600,9 +2020,35 @@ unsafe fn render_thread( active_webcam_offset_sec, ) { active_clip_index = index; + clip_resolved = true; } scene_for_clip(&base_scene, active_clip_index) }); + // The resolved scene clip's own camera files win over the last clip request's: an + // offset or file edited mid-clip arrives with the scene, not with a new request. + if clip_resolved { + if let Some(clip) = full_scene.as_ref().and_then(|s| s.clips.get(active_clip_index)) { + active_additional_cameras = clip.additional_cameras.clone(); + } + } + // A new scene can add, move or drop the layout regions that show an extra camera: + // open what the active clip now shows, close what it no longer does. + let (regions, keys) = match (&full_scene, &scene) { + (Some(full), Some(windowed)) => ( + windowed.camera_layout_regions.clone(), + extra_camera_keys( + &extra_cameras_for_clip(full, active_clip_index), + &active_additional_cameras, + ), + ), + _ => (Vec::new(), Vec::new()), + }; + // A scene push leaves the screen position where it is: while playing, unchanged + // cameras are stepped by playback instead of sought again. + let steady = shared.playing.load(Ordering::Relaxed); + if player.set_extra_cameras(regions, keys, steady) { + comp.clear_srv_cache(); + } comp.set_scene(scene); } // Le temps programme dépend du clip actif ET de la scène entière (durées des clips @@ -1717,6 +2163,7 @@ unsafe fn render_thread( &mut active_screen_path, &mut active_webcam_path, &mut active_webcam_offset_sec, + &mut active_additional_cameras, &mut active_clip_index, &mut raw_cursor, &mut loaded_cursor_path, @@ -1745,6 +2192,7 @@ unsafe fn render_thread( &mut active_screen_path, &mut active_webcam_path, &mut active_webcam_offset_sec, + &mut active_additional_cameras, &mut active_clip_index, &mut raw_cursor, &mut loaded_cursor_path, @@ -1783,39 +2231,56 @@ unsafe fn render_thread( stepped = true; } - if stepped || first { + if stepped || first || publish_pending { if pw > 0 && ph > 0 { - // Step complet : `compose_frame` (déjà appelé par `step`/`present_frame`/ - // `recompose`) a rastérisé le RT à la géométrie de sortie ramenée au panneau. - // On lit ce RT DIRECTEMENT à sa résolution de rendu (`readback_direct` : copy - // rt → staging → Map/Unmap), sans le resize `blit_resized` qui, depuis la - // refonte ratio, n'était plus qu'une copie identité + une alloc NV12 inutile. - match comp.readback_direct() { - Ok((rw, rh, rgba)) => { - // Publie dans `latest_frame` : on remplace le buffer précédent - // (le canvas ne montre que la dernière frame, peu importe combien - // le renderer en a raté entre deux lectures napi). On incrémente - // la génération sous le MÊME lock que l'écriture du buffer, pour - // qu'un lecteur ne puisse jamais voir un `gen` neuf appairé à un - // buffer périmé (ou l'inverse). `+ 1` depuis la précédente, `1` au - // premier publish. Les dims publiées sont celles du RENDU (`rw`×`rh`) : - // le canvas JS s'y dimensionne (packet auto-descriptif) puis CSS met à - // l'échelle vers la boîte du panneau — plus de resize GPU intermédiaire. - // La génération vient d'un compteur atomique et non du slot : la - // livraison sans copie VIDE le slot en le lisant, et un - // `unwrap_or(1)` repartirait alors de 1 — le consommateur recevrait - // des générations déjà peintes et boucherait. Séquence identique à - // l'ancienne dérivation tant que le slot n'est pas vidé. - let next_gen = shared.frame_gen.fetch_add(1, Ordering::Relaxed) + 1; - if let Ok(mut slot) = shared.latest_frame.lock() { - *slot = Some((next_gen, rw, rh, rgba)); - } + let position = FramePosition { + clip_index: active_clip_index as u32, + source_time_sec: player.screen_time_sec(), + }; + match publish_shared(&shared, &gpu, &comp, &mut ring, position) { + SharedPublish::Published => { first = false; + publish_pending = false; } - Err(e) => { - eprintln!("[live] readback_direct: {e:#}"); - std::thread::sleep(Duration::from_millis(8)); + // Chromium tient toutes les cases : la frame attend dans le RT, et la boucle + // ne tourne pas à vide le temps qu'il en relâche une. + SharedPublish::NoFreeSlot => { + publish_pending = true; + std::thread::sleep(Duration::from_millis(2)); } + // Step complet : `compose_frame` (déjà appelé par `step`/`present_frame`/ + // `recompose`) a rastérisé le RT à la géométrie de sortie ramenée au panneau. + // On lit ce RT DIRECTEMENT à sa résolution de rendu (`readback_direct` : copy + // rt → staging → Map/Unmap), sans le resize `blit_resized` qui, depuis la + // refonte ratio, n'était plus qu'une copie identité + une alloc NV12 inutile. + SharedPublish::Off => match comp.readback_direct() { + Ok((rw, rh, rgba)) => { + // Publie dans `latest_frame` : on remplace le buffer précédent + // (le canvas ne montre que la dernière frame, peu importe combien + // le renderer en a raté entre deux lectures napi). On incrémente + // la génération sous le MÊME lock que l'écriture du buffer, pour + // qu'un lecteur ne puisse jamais voir un `gen` neuf appairé à un + // buffer périmé (ou l'inverse). `+ 1` depuis la précédente, `1` au + // premier publish. Les dims publiées sont celles du RENDU (`rw`×`rh`) : + // le canvas JS s'y dimensionne (packet auto-descriptif) puis CSS met à + // l'échelle vers la boîte du panneau — plus de resize GPU intermédiaire. + // La génération vient d'un compteur atomique et non du slot : la + // livraison sans copie VIDE le slot en le lisant, et un + // `unwrap_or(1)` repartirait alors de 1 — le consommateur recevrait + // des générations déjà peintes et boucherait. Séquence identique à + // l'ancienne dérivation tant que le slot n'est pas vidé. + let next_gen = shared.frame_gen.fetch_add(1, Ordering::Relaxed) + 1; + if let Ok(mut slot) = shared.latest_frame.lock() { + *slot = Some((next_gen, rw, rh, rgba, comp.footage_quad(), position)); + } + first = false; + publish_pending = false; + } + Err(e) => { + eprintln!("[live] readback_direct: {e:#}"); + std::thread::sleep(Duration::from_millis(8)); + } + }, } } } else { @@ -1825,6 +2290,101 @@ unsafe fn render_thread( Ok(()) } +/// Ce que la publication dans l'anneau partagé a fait de la frame composée. +enum SharedPublish { + /// Posée dans une case de l'anneau. + Published, + /// Chromium tient toutes les cases : la frame attend dans le RT qu'il en relâche une. + NoFreeSlot, + /// Transport partagé coupé ou indisponible : la frame repasse par le readback. + Off, +} + +#[cfg(windows)] +type Ring = Option; +#[cfg(not(windows))] +type Ring = (); + +/// Pose la frame composée dans l'anneau de textures partagées, si la vue livre ainsi. +/// +/// Toute panne coupe le transport (`shared_frames` à `false`) et rend `Off` : la frame part +/// par le readback, et le service, qui lit les deux, n'a rien à décider. +#[cfg(windows)] +unsafe fn publish_shared( + shared: &Shared, + gpu: &Gpu, + comp: &Compositor, + ring: &mut Ring, + position: FramePosition, +) -> SharedPublish { + if !shared.shared_frames.load(Ordering::Relaxed) { + return SharedPublish::Off; + } + let turn_off = |why: String| { + eprintln!("[live] textures partagées coupées ({why}) — retour au readback"); + shared.shared_frames.store(false, Ordering::Relaxed); + SharedPublish::Off + }; + // WARP ne partage rien avec le device de Chromium. + if gpu.backend != Backend::Hardware { + return turn_off("backend logiciel".into()); + } + if ring.is_none() { + match SharedRing::new(gpu) { + Ok(created) => *ring = Some(created), + Err(e) => return turn_off(format!("{e:#}")), + } + } + let Some(ring) = ring.as_mut() else { + return SharedPublish::Off; + }; + let (claimed, lost) = match shared.slot_book.lock() { + Ok(mut book) => (book.claim(crate::shared_frames::RING_SLOTS), book.lost(Instant::now())), + Err(_) => (None, false), + }; + let Some(slot) = claimed else { + // Chromium relâche d'ordinaire une case dans la milliseconde. Une case perdue ne + // reviendra plus : la réécrire déchirerait peut-être une image qu'il lit encore, et + // l'attendre figerait la preview. + if lost { + return turn_off("une case jamais relâchée par Chromium".into()); + } + return SharedPublish::NoFreeSlot; + }; + let (width, height) = comp.render_size(); + match ring.write(slot, comp.render_target(), width, height) { + Ok(handle) => { + if let Ok(mut book) = shared.slot_book.lock() { + // Le compteur du readback : une seule suite de générations quel que soit le + // transport, incrémentée sous le lock de la publication comme là-bas. + let gen = shared.frame_gen.fetch_add(1, Ordering::Relaxed) + 1; + book.publish(SharedFrame { + gen, + slot, + handle, + width, + height, + footage: comp.footage_quad(), + position, + }); + } + SharedPublish::Published + } + Err(e) => turn_off(format!("{e:#}")), + } +} + +#[cfg(not(windows))] +unsafe fn publish_shared( + _: &Shared, + _: &Gpu, + _: &Compositor, + _: &mut Ring, + _: FramePosition, +) -> SharedPublish { + SharedPublish::Off +} + // ---------- harnais standalone (poc-d3d.exe --live) ---------- // // Le harnais crée une fenêtre Win32 simple qui héberge la preview live ; c'est un @@ -1958,7 +2518,7 @@ pub fn run_standalone(screen: &str, webcam: &str, cursor_json: &str) -> Result<( // standalone n'affiche pas réellement les pixels ici (l'embed Electron est // le consumer réel). On imprime juste une frame de temps en temps pour // confirmer que la chaîne fonctionne. - if let Some((_gen, fw, fh, _pixels)) = view.latest_frame() { + if let Some((_gen, fw, fh, _pixels, _, _)) = view.latest_frame() { if (fw, fh) != (w, h) { // garde-fou : la staging de readback suit `set_rect` côté thread // de rendu, donc ce serait une désynchro transitoire — acceptable. @@ -2092,8 +2652,147 @@ mod tests { #[test] fn webcam_seek_uses_screen_source_time_and_offset() { - assert_eq!(webcam_seek_time(22.5, 1.25), 21.25); - assert_eq!(webcam_seek_time(0.5, 1.25), 0.0); + assert_eq!(camera_source_time(22.5, 1.25), 21.25); + assert_eq!(camera_source_time(0.5, 1.25), 0.0); + } + + // --- extra cameras (2-4) ---------------------------------------------------- + + /// Two clips: clip 0 has files for cameras 1 and 3 (camera 2's slot is empty), clip 1 for + /// camera 1 only. Layout regions: clip 0 shows cameras 1 and 2 on 2-5 s and camera 3 only + /// in an empty region; clip 1 shows camera 1 on 21-22 s. + fn layout_scene() -> Scene { + let layer = |camera: usize| { + format!(r#"{{"camera":{camera},"rect":{{"x":0,"y":0,"width":0.5,"height":0.5}},"radiusFrac":0,"shape":"rectangle","fillsFrame":false}}"#) + }; + let json = format!( + r##"{{ + "clips": [ + {{"screenPath":"/s0.mp4","webcamPath":"/w0.mp4","sourceStartSec":0,"sourceEndSec":10,"webcamOffsetSec":0,"hasAudio":true, + "additionalCameras":[{{"path":"/c1.mp4","offsetSec":0.5}},{{"path":"","offsetSec":0}},{{"path":"/c3.mp4","offsetSec":0}}]}}, + {{"screenPath":"/s1.mp4","webcamPath":"/w1.mp4","sourceStartSec":20,"sourceEndSec":30,"webcamOffsetSec":0,"hasAudio":true, + "additionalCameras":[{{"path":"/d1.mp4","offsetSec":0}}]}} + ], + "layout":{{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, + "effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}}, + "background":{{"kind":"color","color":"#000000"}}, + "zoomRegions":[], + "cameraLayoutRegions":[ + {{"clipIndex":0,"startSec":2,"endSec":5,"layers":[{l0},{l1},{l2}]}}, + {{"clipIndex":0,"startSec":7,"endSec":7,"layers":[{l3}]}}, + {{"clipIndex":1,"startSec":21,"endSec":22,"layers":[{l1}]}} + ], + "cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}}, + "cropByClip":[null,null], + "output":{{"width":1920,"height":1080,"fps":30}} + }}"##, + l0 = layer(0), + l1 = layer(1), + l2 = layer(2), + l3 = layer(3), + ); + Scene::from_json(&json).expect("layout scene") + } + + fn camera(path: &str, offset_sec: f64) -> SceneClipCamera { + SceneClipCamera { path: path.to_string(), offset_sec } + } + + #[test] + fn extra_cameras_to_open_are_those_the_clips_regions_reference() { + let scene = layout_scene(); + // Camera 2 is shown but clip 0 has no file for it; camera 3 has a file but only an + // empty region; camera 0 is not an extra camera. + assert_eq!(extra_cameras_for_clip(&scene, 0), vec![1]); + // Clip 1 only sees its own region. + assert_eq!(extra_cameras_for_clip(&scene, 1), vec![1]); + assert!(extra_cameras_for_clip(&scene, 9).is_empty()); + // No layout region at all: nothing to open, whatever files the clip has. + assert!(extra_cameras_for_clip(&multiclip_scene(), 0).is_empty()); + + let keys = scene_extra_camera_keys(&scene, 0); + assert_eq!(keys.len(), 1, "trailing empty slots are dropped"); + let first = keys[0].as_ref().expect("camera 1"); + assert_eq!((first.path.as_str(), first.offset_sec), ("/c1.mp4", 0.5)); + // A shown camera after an unshown one keeps its index; nothing shown is an empty list. + let sources = [camera("/a.mp4", 0.0), camera("/b.mp4", 0.0), camera("/c.mp4", 0.0)]; + let keys = extra_camera_keys(&[3], &sources); + assert_eq!(keys.len(), 3); + assert!(keys[0].is_none() && keys[1].is_none()); + assert_eq!(keys[2].as_ref().map(|k| k.path.as_str()), Some("/c.mp4")); + assert!(extra_camera_keys(&[], &sources).is_empty()); + assert!(extra_camera_keys(&[2], &[camera("/a.mp4", 0.0)]).is_empty()); + } + + #[test] + fn an_extra_camera_is_decoded_only_near_its_regions() { + let regions = scene_for_clip(&layout_scene(), 0).camera_layout_regions; + // From PREFETCH_LEAD_SEC before the region to its end. + assert!(!extra_camera_active(®ions, 1, 2.0 - PREFETCH_LEAD_SEC - 0.01)); + assert!(extra_camera_active(®ions, 1, 2.0 - PREFETCH_LEAD_SEC)); + assert!(extra_camera_active(®ions, 1, 3.0)); + assert!(extra_camera_active(®ions, 1, 5.0)); + assert!(!extra_camera_active(®ions, 1, 5.01)); + // A camera the region does not show, and one only an empty region names. + assert!(!extra_camera_active(®ions, 3, 7.0)); + assert!(!extra_camera_active(&[], 1, 3.0)); + } + + /// Playing on with the same files and region timing, a scene push does not seek the extra + /// cameras again; any change that matters, a jump, or a pause does. + #[test] + fn extra_cameras_are_sought_again_only_when_it_matters() { + assert!(!extras_need_seek(false, false, true)); + assert!(extras_need_seek(true, false, true), "new files"); + assert!(extras_need_seek(false, true, true), "regions moved"); + assert!(extras_need_seek(false, false, false), "paused, or the position jumped"); + + let regions = scene_for_clip(&layout_scene(), 0).camera_layout_regions; + assert!(same_region_timing(®ions, ®ions.clone())); + // A moved rect is the same timing; a moved bound or another camera is not. + let mut moved_rect = regions.clone(); + moved_rect[0].layers[0].rect.x += 0.1; + assert!(same_region_timing(®ions, &moved_rect)); + let mut moved_end = regions.clone(); + moved_end[0].end_sec += 0.5; + assert!(!same_region_timing(®ions, &moved_end)); + let mut other_camera = regions.clone(); + other_camera[0].layers[0].camera = 3; + assert!(!same_region_timing(®ions, &other_camera)); + assert!(!same_region_timing(®ions, ®ions[..1])); + } + + #[test] + fn a_missing_extra_camera_skips_its_layer() { + let failed: Result = Err(anyhow::anyhow!("0-byte file")); + assert_eq!(opened_or_skipped("/c1.mp4", failed), None); + assert_eq!(opened_or_skipped("/c1.mp4", Ok(7u8)), Some(7)); + + // Slot 0 failed to open, slot 1 is open with a frame, slot 2 is open with nothing to + // show (idle or past its end): only slot 1 hands a frame to the compositor. + let frame = 0x10usize as *const AVFrame; + let slots = [None, Some(frame), Some(std::ptr::null())]; + let list = extra_frame_list(&slots, |f| *f); + assert_eq!(list, vec![std::ptr::null(), frame, std::ptr::null()]); + assert!(extra_frame_list::<*const AVFrame>(&[], |f| *f).is_empty()); + } + + #[test] + fn the_pool_key_includes_the_extra_cameras() { + let key = |extras: ExtraCameraKeys| ClipKey { + screen_path: "/s0.mp4".into(), + webcam_path: "/w0.mp4".into(), + webcam_offset_sec: 0.0, + extras, + }; + let with_camera_1 = key(vec![Some(camera("/c1.mp4", 0.5))]); + assert!(with_camera_1.matches(&key(vec![Some(camera("/c1.mp4", 0.5))]))); + // Same screen and camera 0, but camera 1 missing, another file or another offset. + assert!(!with_camera_1.matches(&key(Vec::new()))); + assert!(!with_camera_1.matches(&key(vec![Some(camera("/other.mp4", 0.5))]))); + assert!(!with_camera_1.matches(&key(vec![Some(camera("/c1.mp4", 0.75))]))); + assert!(!with_camera_1.matches(&key(vec![None, Some(camera("/c1.mp4", 0.5))]))); + assert!(key(Vec::new()).matches(&key(Vec::new()))); } #[test] diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs index b118befdd..c31eb494f 100644 --- a/crates/compositor/src/pipeline_linux.rs +++ b/crates/compositor/src/pipeline_linux.rs @@ -54,8 +54,13 @@ pub struct ClipSource { pub source_end_sec: f64, pub webcam_offset_sec: f64, pub has_audio: bool, + /// Cameras 2-4 of the clip (index k-1 = camera k, an empty `path` = none). Only those a + /// layout region of the clip shows are decoded. + pub additional_cameras: Vec, } +pub use crate::extra_cameras::ClipCamera; + /// Codec cible. Memes variantes que `pipeline_macos::ExportCodec`. #[derive(Clone, Copy, Debug)] pub enum ExportCodec { @@ -127,6 +132,12 @@ impl Decoder { self.decode_present(idx) } + /// Contrat de `pipeline_windows::Decoder::seek_to_or_last`, que `seek_to` remplit déjà ici : + /// au-delà de la fin, `pump_to_target` tient la dernière image décodée. + pub unsafe fn seek_to_or_last(&mut self, seconds: f64) -> Result<*mut AVFrame> { + self.seek_to(seconds) + } + /// Decode la frame SEQUENTIELLE suivante — pompage `next_frame`, PAS de seek. /// La frame rendue appartient au decodeur (valide jusqu'au prochain appel), /// donc elle ne se libere pas ici, contrairement au chemin `decode_at`. @@ -939,6 +950,7 @@ pub fn run_composited_multi( let mut screen_decs: HashMap = HashMap::new(); let mut webcam_decs: HashMap = HashMap::new(); + let mut extra_decs: HashMap = HashMap::new(); // ---- muxer MP4 (flux video + flux AAC) ---- let outc = CString::new(out)?; @@ -1032,6 +1044,7 @@ pub fn run_composited_multi( &scene, &mut screen_decs, &mut webcam_decs, + &mut extra_decs, &mut |n| { // Soumet la copie de la frame n SANS l'attendre et recolte la // precedente : c'est tout le pipelining GPU. L'encodage, lui, diff --git a/crates/compositor/src/pipeline_macos.rs b/crates/compositor/src/pipeline_macos.rs index 93827b00c..6fc75f01a 100644 --- a/crates/compositor/src/pipeline_macos.rs +++ b/crates/compositor/src/pipeline_macos.rs @@ -410,6 +410,25 @@ impl Decoder { } } + /// `seek_to`, mais une cible au-delà de la dernière image se pose sur cette dernière image au + /// lieu de ne rien rendre. L'audio d'un enregistrement dure souvent un peu plus que sa vidéo + /// (de 12 ms à près d'une demi-seconde mesurés) : son clip finit alors après la dernière + /// image, et y entrer par la fin, en scrubant de droite à gauche, demandait une image qui + /// n'existe pas. Le changement de clip échouait sans bruit, et la vue gardait la scène du + /// clip qu'on quittait. + pub unsafe fn seek_to_or_last(&mut self, seconds: f64) -> Result<*mut crate::ffi::AVFrame> { + let frame = self.seek_to(seconds)?; + if !frame.is_null() { + return Ok(frame); + } + // Le seek a décodé jusqu'à l'EOF : `cur_pts` est celui de la dernière image. L'EOF a vidé + // la frame courante, donc il faut le seek complet : `take` écarte le chemin rapide. + match self.cur_pts.take() { + Some(last) => self.seek_to(last as f64 * self.tb_sec()), + None => Ok(frame), + } + } + /// Déroule le décodeur en avant jusqu'à la première frame à `seconds` ou après, SANS /// jeter son état. Symétrique de `pipeline_windows::Decoder::decode_forward_to`. unsafe fn decode_forward_to(&mut self, seconds: f64, tb_sec: f64) -> Result<*mut crate::ffi::AVFrame> { @@ -644,8 +663,13 @@ pub struct ClipSource { pub source_end_sec: f64, pub webcam_offset_sec: f64, pub has_audio: bool, + /// Cameras 2-4 of the clip (index k-1 = camera k, an empty `path` = none). Only those a + /// layout region of the clip shows are decoded. + pub additional_cameras: Vec, } +pub use crate::extra_cameras::ClipCamera; + /// Codec cible pour l'export. Identique à `pipeline_windows::ExportCodec`. pub enum ExportCodec { H264, @@ -1141,6 +1165,8 @@ pub fn run_composited_multi( std::collections::HashMap::new(); let mut webcam_decs: std::collections::HashMap = std::collections::HashMap::new(); + let mut extra_decs: std::collections::HashMap = + std::collections::HashMap::new(); // ---- encodeur (candidat VT ou software) ---- let mut enc = @@ -1222,6 +1248,7 @@ pub fn run_composited_multi( &scene, &mut screen_decs, &mut webcam_decs, + &mut extra_decs, &mut |n| { enc.send_composited(comp, out_w, out_h, n as i64)?; { diff --git a/crates/compositor/src/pipeline_windows.rs b/crates/compositor/src/pipeline_windows.rs index fabe31207..8094fd44b 100644 --- a/crates/compositor/src/pipeline_windows.rs +++ b/crates/compositor/src/pipeline_windows.rs @@ -968,6 +968,25 @@ impl Decoder { } } + /// `seek_to`, mais une cible au-delà de la dernière image se pose sur cette dernière image au + /// lieu de ne rien rendre. L'audio d'un enregistrement dure souvent un peu plus que sa vidéo + /// (de 12 ms à près d'une demi-seconde mesurés) : son clip finit alors après la dernière + /// image, et y entrer par la fin, en scrubant de droite à gauche, demandait une image qui + /// n'existe pas. Le changement de clip échouait sans bruit, et la vue gardait la scène du + /// clip qu'on quittait. + pub(crate) unsafe fn seek_to_or_last(&mut self, seconds: f64) -> Result<*mut AVFrame> { + let frame = self.seek_to(seconds)?; + if !frame.is_null() { + return Ok(frame); + } + // Le seek a décodé jusqu'à l'EOF : `cur_pts` est celui de la dernière image. L'EOF a vidé + // la frame courante, donc il faut le seek complet : `take` écarte le chemin rapide. + match self.cur_pts.take() { + Some(last) => self.seek_to(last as f64 * self.tb_sec()), + None => Ok(frame), + } + } + /// Déroule le décodeur en avant jusqu'à la première frame à `seconds` ou après, SANS /// jeter son état. Critère d'arrêt identique à celui du seek complet — c'est ce qui /// garantit que les deux chemins rendent exactement la même frame. @@ -1265,8 +1284,13 @@ pub struct ClipSource { pub source_end_sec: f64, pub webcam_offset_sec: f64, pub has_audio: bool, + /// Cameras 2-4 of the clip (index k-1 = camera k, an empty `path` = none). Only those a + /// layout region of the clip shows are decoded. + pub additional_cameras: Vec, } +pub use crate::extra_cameras::ClipCamera; + /// Export **multiclip** : rend la timeline (clips ordonnés, avec trims) en un seul MP4. /// Perf (contrainte §multiclip) : décodeurs ouverts une fois par source (cache) et réutilisés /// entre clips du même asset ; **un seul seek keyframe par frontière de clip** ; décodage @@ -1631,6 +1655,7 @@ unsafe fn run_multi_inner( // pour deux &mut indépendants). let mut screen_decs: HashMap = HashMap::new(); let mut webcam_decs: HashMap = HashMap::new(); + let mut extra_decs: HashMap = HashMap::new(); // fps de sortie : choix explicite de l'app si fourni, sinon dérivé du 1er clip (recordings // uniformes) — comportement historique. @@ -1711,6 +1736,7 @@ unsafe fn run_multi_inner( &scene, &mut screen_decs, &mut webcam_decs, + &mut extra_decs, &mut |frame_index| { // Backend CPU (WARP) : la frame composée descend en mémoire système via // `send_composited` (le compositeur relit son NV12 interne vers un AVFrame @@ -2836,6 +2862,29 @@ mod tests { ); } + #[test] + fn a_seek_past_the_last_frame_can_land_on_it() { + let Some(gpu) = strict_hardware_gpu("a_seek_past_the_last_frame_can_land_on_it") else { + return; + }; + let path = encode_color(&["-c:v", "libopenh264", "-b:v", "200k"], "last-frame.mp4"); + let mut dec = unsafe { Decoder::open(path.to_str().expect("utf8 path"), &gpu) } + .unwrap_or_else(|e| panic!("H.264 Decoder::open: {e:#}")); + + let frame = unsafe { dec.seek_to_or_last(10.0) }.expect("seek past the end"); + assert!(!frame.is_null(), "a target past the end must land on the last frame"); + assert!(!dec.cur_frame().is_null(), "the last frame must be presentable"); + assert!( + unsafe { dec.next() }.expect("decode after the last frame").is_null(), + "the frame it landed on must be the last one" + ); + assert!( + unsafe { dec.seek_to(10.0) }.expect("plain seek past the end").is_null(), + "seek_to itself still reports no frame past the end" + ); + println!("CORE_ASSERTIONS_COMPLETED:a_seek_past_the_last_frame_can_land_on_it"); + } + /// Playhead crossing clips is `Decoder::open` of the next source on the /// same `Gpu` (#554). H.264 must stay on D3D11VA after an AV1 software /// decoder has been opened and dropped. diff --git a/crates/compositor/src/prism_mesh.rs b/crates/compositor/src/prism_mesh.rs new file mode 100644 index 000000000..c045462b0 --- /dev/null +++ b/crates/compositor/src/prism_mesh.rs @@ -0,0 +1,876 @@ +//! Les deux curseurs de Prism Glow en MAILLAGE : les facettes tracées sur l'art 2D +//! (`design/cursors/prism-glow/model`), taillées en cristal. Fichier GÉNÉRÉ par +//! `design/cursors/prism-glow/model/export_compositor.py` : ne pas l'éditer à la main. +//! +//! Repère du modèle du mode 15 : unité = côté du sprite 128 px du thème, origine au hotspot, +//! x à droite, y vers le bas, z vers la caméra, dessus du serti en z = 0. Chaque triangle du +//! cristal tient en quatre float4 : (v0, drapeaux), (e1, r), (e2, g), (normale, b) ; drapeaux = +//! paroi (1) + 2 × arêtes réelles (bit k : l'arête opposée au sommet k). Le fond plat, en +//! z = -épaisseur, est laissé au shader. `POLYS` tient les silhouettes (bord extérieur du +//! serti) puis les contours du cristal (son bord intérieur). `BOXES` range les triangles +//! par voisinage : deux float4 par boîte, (coin bas, premier triangle), (coin haut, nombre). + +/// Un des deux modèles : ses triangles et sa silhouette dans les tables, sa boîte de dessin. +pub struct PrismModel { + pub tri_start: usize, + pub tri_count: usize, + pub sil_start: usize, + pub sil_count: usize, + pub out_start: usize, + pub out_count: usize, + pub box_start: usize, + pub box_count: usize, + /// Largeur de la boîte, hauteur 1 (le sprite), en unités du modèle. + pub size: [f32; 2], + /// Hotspot, fraction de la boîte. + pub hotspot: [f32; 2], + /// Haut de la silhouette, fraction de la hauteur. + pub top: f32, + /// Sommet du cristal au-dessus de z = 0, et épaisseur sous z = 0. + pub height: f32, + pub thick: f32, +} + +/// Arrondi des arêtes du serti, en unités du modèle (`PRISM_BEVEL` des shaders). +pub const BEVEL: f32 = 0.012883; + +/// Flèche puis main, dans l'ordre des identifiants du shader. +pub const MODELS: [PrismModel; 2] = [ + PrismModel { + tri_start: 0, + tri_count: 31, + sil_start: 0, + sil_count: 25, + out_start: 67, + out_count: 9, + box_start: 0, + box_count: 4, + size: [0.63089, 1.0], + hotspot: [0.022658, 0.064647], + top: 0.0625, + height: 0.115951, + thick: 0.042945, + }, + PrismModel { + tri_start: 31, + tri_count: 130, + sil_start: 25, + sil_count: 42, + out_start: 76, + out_count: 34, + box_start: 4, + box_count: 18, + size: [0.797658, 1.0], + hotspot: [0.343388, 0.063582], + top: 0.0625, + height: 0.0822, + thick: 0.043263, + }, +]; + +pub const TRIS: [[f32; 4]; 644] = [ + [0.252301, 0.266258, -0.042945, 11.0], + [-0.221166, -0.203988, 0.0, 0.01033], + [-0.221166, -0.203988, 0.042945, 0.045182], + [0.677984, -0.735077, 0.0, 0.577584], + [0.252301, 0.266258, -0.042945, 3.0], + [-0.221166, -0.203988, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.677984, -0.735077, 0.0, 0.577584], + [0.031135, 0.06227, -0.042945, 7.0], + [0.004294, 0.419785, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [-0.999948, 0.01023, 0.0, 0.577584], + [0.169632, 0.356442, 0.115951, 14.0], + [0.082669, -0.090184, -0.115951, 0.590616], + [-0.138497, -0.294172, -0.115951, 0.737911], + [0.466385, -0.505659, 0.725806, 0.973449], + [0.035429, 0.482055, 0.0, 14.0], + [0.134202, -0.125614, 0.115951, 0.000305], + [-0.004294, -0.419785, 0.0, 0.938689], + [-0.650191, 0.006652, 0.759742, 0.854994], + [0.031135, 0.06227, -0.042945, 9.0], + [0.004294, 0.419785, 0.0, 0.01033], + [0.004294, 0.419785, 0.042945, 0.045182], + [-0.999948, 0.01023, 0.0, 0.577584], + [0.169632, 0.356442, 0.115951, 14.0], + [0.159969, 0.080521, -0.045092, 0.381323], + [0.082669, -0.090184, -0.115951, 0.056123], + [0.461407, -0.510214, 0.725799, 0.973449], + [0.541104, 0.544325, -0.042945, 9.0], + [-0.288804, -0.278068, 0.0, 0.01033], + [-0.288804, -0.278068, 0.042945, 0.045182], + [0.69359, -0.72037, 0.0, 0.577584], + [0.329601, 0.436963, 0.070859, 14.0], + [0.211503, 0.107362, -0.070859, 0.545724], + [-0.077301, -0.170706, -0.070859, 0.064805], + [0.49567, -0.514808, 0.699488, 0.973449], + [0.268405, 0.533589, 0.081595, 14.0], + [0.061196, -0.096626, -0.010736, 0.000916], + [-0.098773, -0.177147, 0.034356, 0.046662], + [0.247834, 0.049458, 0.967539, 0.904662], + [0.541104, 0.544325, -0.042945, 7.0], + [-0.288804, -0.278068, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.69359, -0.72037, 0.0, 0.577584], + [0.199693, 0.595859, 0.0, 14.0], + [0.068712, -0.06227, 0.081595, 0.0], + [-0.030061, -0.239417, 0.115951, 0.01445], + [-0.504461, 0.426837, 0.750553, 0.254154], + [0.268405, 0.533589, 0.081595, 14.0], + [0.272699, 0.010736, -0.081595, 0.0], + [0.061196, -0.096626, -0.010736, 0.025193], + [0.283244, 0.073137, 0.956255, 0.527108], + [0.035429, 0.482055, 0.0, 14.0], + [0.0, 0.258742, 0.0, 0.0], + [0.134202, -0.125614, 0.115951, 0.473533], + [-0.653777, 0.0, 0.756687, 0.964691], + [0.359663, 0.559356, 0.0, 14.0], + [0.181442, -0.015031, 0.0, 0.006043], + [-0.091258, -0.025767, 0.081595, 0.076188], + [0.076465, 0.923048, 0.37701, 0.964691], + [0.035429, 0.482055, -0.042945, 3.0], + [0.0, 0.258742, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [-1.0, 0.0, 0.0, 0.577584], + [0.035429, 0.482055, -0.042945, 11.0], + [0.0, 0.258742, 0.0, 0.01033], + [0.0, 0.258742, 0.042945, 0.045182], + [-1.0, 0.0, 0.0, 0.577584], + [0.035429, 0.740798, -0.042945, 7.0], + [0.164264, -0.144939, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.661622, 0.749838, 0.0, 0.577584], + [0.035429, 0.740798, 0.0, 14.0], + [0.164264, -0.144939, 0.0, 0.0], + [0.134202, -0.384356, 0.115951, 0.010956], + [0.332572, 0.376915, 0.864483, 0.215854], + [0.035429, 0.740798, -0.042945, 11.0], + [0.164264, -0.144939, 0.0, 0.01033], + [0.164264, -0.144939, 0.042945, 0.045182], + [0.661622, 0.749838, 0.0, 0.577584], + [0.199693, 0.595859, -0.042945, 7.0], + [0.105215, 0.22546, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [-0.906183, 0.422886, 0.0, 0.577584], + [0.199693, 0.595859, 0.0, 14.0], + [0.105215, 0.22546, 0.0, 0.0], + [0.068712, -0.06227, 0.081595, 0.015991], + [-0.613882, 0.286478, 0.735581, 0.473533], + [0.199693, 0.595859, -0.042945, 11.0], + [0.105215, 0.22546, 0.0, 0.01033], + [0.105215, 0.22546, 0.042945, 0.045182], + [-0.906183, 0.422886, 0.0, 0.577584], + [0.304908, 0.821319, 0.0, 14.0], + [0.128834, -0.067638, 0.0, 0.0], + [-0.036503, -0.28773, 0.081595, 0.254154], + [0.133686, 0.254641, 0.957751, 0.973449], + [0.304908, 0.821319, -0.042945, 7.0], + [0.128834, -0.067638, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.464834, 0.885398, 0.0, 0.577584], + [0.433742, 0.753681, 0.0, 14.0], + [-0.07408, -0.194325, 0.0, 0.0], + [-0.165337, -0.220092, 0.081595, 0.964691], + [0.683362, -0.260508, 0.682021, 0.964691], + [0.433742, 0.753681, -0.042945, 11.0], + [-0.07408, -0.194325, 0.0, 0.01033], + [-0.07408, -0.194325, 0.042945, 0.045182], + [0.934406, -0.35621, 0.0, 0.577584], + [0.304908, 0.821319, -0.042945, 11.0], + [0.128834, -0.067638, 0.0, 0.01033], + [0.128834, -0.067638, 0.042945, 0.045182], + [0.464834, 0.885398, 0.0, 0.577584], + [0.433742, 0.753681, -0.042945, 7.0], + [-0.07408, -0.194325, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.934406, -0.35621, 0.0, 0.577584], + [0.359663, 0.559356, -0.042945, 7.0], + [0.181442, -0.015031, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.082557, 0.996586, 0.0, 0.577584], + [0.359663, 0.559356, -0.042945, 11.0], + [0.181442, -0.015031, 0.0, 0.01033], + [0.181442, -0.015031, 0.042945, 0.045182], + [0.082557, 0.996586, 0.0, 0.577584], + [0.044345, 0.043263, -0.043263, 11.0], + [-0.043263, -0.005408, 0.0, 0.01033], + [-0.043263, -0.005408, 0.043263, 0.045182], + [0.124035, -0.992278, 0.0, 0.577584], + [0.044345, 0.043263, -0.043263, 7.0], + [-0.043263, -0.005408, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.124035, -0.992278, 0.0, 0.577584], + [0.001082, 0.037855, -0.043263, 7.0], + [-0.044345, 0.040019, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.669964, -0.742393, 0.0, 0.577584], + [0.044345, 0.043263, 0.0, 10.0], + [-0.043263, -0.005408, 0.0, 0.000305], + [-0.040019, 0.030284, 0.049753, 0.964691], + [0.101434, -0.811469, 0.575525, 0.973449], + [0.004326, 0.073548, 0.049753, 14.0], + [-0.003245, -0.035692, -0.049753, 0.0], + [-0.04759, 0.004326, -0.049753, 0.955978], + [-0.580475, -0.643229, 0.499304, 0.896269], + [0.001082, 0.037855, -0.043263, 11.0], + [-0.044345, 0.040019, 0.0, 0.01033], + [-0.044345, 0.040019, 0.043263, 0.045182], + [-0.669964, -0.742393, 0.0, 0.577584], + [-0.031366, 0.229295, 0.058405, 14.0], + [0.035692, -0.155748, -0.008653, 0.000305], + [-0.011897, -0.151422, -0.058405, 0.955978], + [-0.716533, -0.20131, 0.667874, 0.904662], + [-0.043263, 0.077874, -0.043263, 7.0], + [-0.001082, 0.154666, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999976, -0.006993, 0.0, 0.577584], + [-0.031366, 0.229295, 0.058405, 14.0], + [0.089771, -0.137361, -0.02055, 0.445197], + [0.035692, -0.155748, -0.008653, 0.973449], + [0.216369, -0.004654, 0.976301, 0.973449], + [-0.031366, 0.229295, 0.058405, 14.0], + [-0.011897, -0.151422, -0.058405, 0.000305], + [-0.012979, 0.003245, -0.058405, 0.955978], + [-0.976245, -0.006827, 0.216564, 0.904662], + [-0.043263, 0.077874, -0.043263, 9.0], + [-0.001082, 0.154666, 0.0, 0.01033], + [-0.001082, 0.154666, 0.043263, 0.045182], + [-0.999976, -0.006993, 0.0, 0.577584], + [-0.045426, 0.564586, 0.052998, 14.0], + [0.014061, -0.335291, 0.005408, 0.520989], + [0.001082, -0.332046, -0.052998, 0.088655], + [-0.975941, -0.037462, 0.214794, 0.973449], + [-0.044345, 0.23254, 0.0, 3.0], + [0.0, 0.0, -0.043263, 0.01033], + [-0.001082, 0.332046, -0.043263, 0.045182], + [-0.999995, -0.003257, 0.0, 0.577584], + [-0.163319, 0.380717, -0.043263, 7.0], + [-0.060569, 0.02704, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.407651, -0.913138, 0.0, 0.577584], + [-0.094098, 0.431551, -0.043263, 11.0], + [-0.069221, -0.050834, 0.0, 0.01033], + [-0.069221, -0.050834, 0.043263, 0.045182], + [0.591909, -0.806004, 0.0, 0.577584], + [-0.163319, 0.380717, -0.043263, 11.0], + [-0.060569, 0.02704, 0.0, 0.01033], + [-0.060569, 0.02704, 0.043263, 0.045182], + [-0.407651, -0.913138, 0.0, 0.577584], + [0.075711, 0.08869, -0.043263, 11.0], + [-0.031366, -0.045426, 0.0, 0.01033], + [-0.031366, -0.045426, 0.043263, 0.045182], + [0.822897, -0.568191, 0.0, 0.577584], + [0.044345, 0.043263, 0.0, 6.0], + [-0.040019, 0.030284, 0.049753, 0.000305], + [0.014061, 0.048671, 0.037855, 0.964691], + [0.365607, -0.634949, 0.680566, 0.973449], + [0.075711, 0.08869, -0.043263, 7.0], + [-0.031366, -0.045426, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.822897, -0.568191, 0.0, 0.577584], + [0.058405, 0.091934, 0.037855, 14.0], + [0.017305, -0.003245, -0.037855, 0.0], + [-0.014061, -0.048671, -0.037855, 0.473533], + [0.757369, -0.522945, 0.39105, 0.904662], + [-0.031366, 0.229295, 0.058405, 14.0], + [0.107077, -0.140606, -0.058405, 0.0], + [0.089771, -0.137361, -0.02055, 0.376257], + [0.812052, 0.481346, 0.329966, 0.830777], + [0.077874, 0.407756, -0.043263, 11.0], + [-0.002163, -0.319067, 0.0, 0.01033], + [-0.002163, -0.319067, 0.043263, 0.045182], + [0.999977, -0.006779, 0.0, 0.577584], + [-0.031366, 0.229295, 0.058405, 14.0], + [0.10924, 0.178461, -0.058405, 0.0], + [0.107077, -0.140606, -0.058405, 0.376257], + [0.475583, -0.003224, 0.879665, 0.830777], + [0.147095, 0.272559, -0.043263, 7.0], + [-0.035692, 0.034611, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.696146, -0.7179, 0.0, 0.577584], + [0.147095, 0.272559, -0.043263, 11.0], + [-0.035692, 0.034611, 0.0, 0.01033], + [-0.035692, 0.034611, 0.043263, 0.045182], + [-0.696146, -0.7179, 0.0, 0.577584], + [0.144932, 0.307169, 0.049753, 14.0], + [0.002163, -0.034611, -0.049753, 0.0], + [-0.033529, 0.0, -0.049753, 0.9131], + [-0.630237, -0.649931, 0.424725, 0.904662], + [0.183869, 0.317985, 0.037855, 6.0], + [-0.036774, -0.045426, -0.037855, 0.0], + [-0.038937, -0.010816, 0.011897, 0.806958], + [0.374433, -0.753477, 0.540438, 0.854994], + [0.077874, 0.407756, -0.043263, 7.0], + [-0.002163, -0.319067, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.999977, -0.006779, 0.0, 0.577584], + [0.111403, 0.307169, -0.043263, 7.0], + [-0.002163, 0.131953, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999866, -0.016391, 0.0, 0.577584], + [0.10924, 0.439122, 0.0, 6.0], + [0.035692, -0.131953, 0.049753, 0.18117], + [0.002163, -0.131953, 0.0, 0.955978], + [-0.82919, -0.013593, 0.558802, 0.964691], + [0.128708, 0.463999, 0.063813, 14.0], + [0.055161, -0.146014, -0.025958, 0.246204], + [0.016224, -0.156829, -0.014061, 0.964691], + [0.305397, -0.053644, 0.950713, 0.973449], + [0.111403, 0.307169, -0.043263, 11.0], + [-0.002163, 0.131953, 0.0, 0.01033], + [-0.002163, 0.131953, 0.043263, 0.045182], + [-0.999866, -0.016391, 0.0, 0.577584], + [0.189277, 0.28013, -0.043263, 11.0], + [-0.042182, -0.007571, 0.0, 0.01033], + [-0.042182, -0.007571, 0.043263, 0.045182], + [0.176664, -0.984271, 0.0, 0.577584], + [0.183869, 0.317985, 0.037855, 10.0], + [0.005408, -0.037855, -0.037855, 0.0], + [-0.036774, -0.045426, -0.037855, 0.806958], + [0.124328, -0.692684, 0.710445, 0.854994], + [0.189277, 0.28013, -0.043263, 7.0], + [-0.042182, -0.007571, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.176664, -0.984271, 0.0, 0.577584], + [0.183869, 0.317985, 0.037855, 14.0], + [0.02704, -0.004326, -0.037855, 0.0], + [0.005408, -0.037855, -0.037855, 0.514916], + [0.70062, -0.452013, 0.552101, 0.846876], + [0.210908, 0.313659, -0.043263, 11.0], + [-0.021632, -0.033529, 0.0, 0.01033], + [-0.021632, -0.033529, 0.043263, 0.045182], + [0.840297, -0.542127, 0.0, 0.577584], + [0.210908, 0.445612, 0.0, 10.0], + [0.0, -0.131953, 0.0, 0.0], + [-0.02704, -0.127627, 0.037855, 0.439658], + [0.813734, 0.0, 0.581238, 0.871366], + [0.210908, 0.313659, -0.043263, 7.0], + [-0.021632, -0.033529, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.840297, -0.542127, 0.0, 0.577584], + [0.210908, 0.445612, -0.043263, 11.0], + [0.0, -0.131953, 0.0, 0.01033], + [0.0, -0.131953, 0.043263, 0.045182], + [1.0, 0.0, 0.0, 0.577584], + [0.241193, 0.343943, -0.043263, 7.0], + [-0.002163, 0.107077, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999796, -0.020198, 0.0, 0.577584], + [0.276885, 0.311496, -0.043263, 11.0], + [-0.035692, 0.032447, 0.0, 0.01033], + [-0.035692, 0.032447, 0.043263, 0.045182], + [-0.672673, -0.73994, 0.0, 0.577584], + [0.23903, 0.45102, 0.0, 14.0], + [0.042182, -0.108158, 0.046508, 0.000305], + [0.002163, -0.107077, 0.0, 0.846876], + [-0.758098, -0.015315, 0.651961, 0.896269], + [0.276885, 0.311496, -0.043263, 7.0], + [-0.035692, 0.032447, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.672673, -0.73994, 0.0, 0.577584], + [0.281211, 0.342862, 0.046508, 14.0], + [-0.004326, -0.031366, -0.046508, 0.0], + [-0.040019, 0.001082, -0.046508, 0.752941], + [-0.58651, -0.645161, 0.489667, 0.846876], + [0.246601, 0.459672, 0.06165, 14.0], + [0.070303, -0.101669, -0.024876, 0.262257], + [0.034611, -0.116811, -0.015142, 0.973449], + [0.27928, -0.041605, 0.959308, 0.973449], + [0.319067, 0.319067, -0.043263, 11.0], + [-0.042182, -0.007571, 0.0, 0.01033], + [-0.042182, -0.007571, 0.043263, 0.045182], + [0.176664, -0.984271, 0.0, 0.577584], + [0.319067, 0.319067, 0.0, 10.0], + [-0.042182, -0.007571, 0.0, 0.0], + [-0.037855, 0.023795, 0.046508, 0.896269], + [0.1483, -0.826245, 0.54344, 0.896269], + [0.319067, 0.319067, -0.043263, 7.0], + [-0.042182, -0.007571, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.176664, -0.984271, 0.0, 0.577584], + [0.319067, 0.319067, 0.0, 6.0], + [-0.037855, 0.023795, 0.046508, 0.0], + [-0.002163, 0.038937, 0.036774, 0.896269], + [0.437906, -0.604311, 0.665618, 0.896269], + [0.316904, 0.358004, 0.036774, 14.0], + [0.024876, -0.004326, -0.036774, 0.0], + [0.002163, -0.038937, -0.036774, 0.617212], + [0.707338, -0.464191, 0.533104, 0.921584], + [0.34178, 0.353677, -0.043263, 11.0], + [-0.022713, -0.034611, 0.0, 0.01033], + [-0.022713, -0.034611, 0.043263, 0.045182], + [0.836048, -0.548657, 0.0, 0.577584], + [0.340698, 0.474815, 0.0, 14.0], + [0.001082, -0.121137, 0.0, 0.0], + [-0.023795, -0.116811, 0.036774, 0.473533], + [0.828664, 0.007399, 0.559697, 0.930114], + [0.34178, 0.353677, -0.043263, 7.0], + [-0.022713, -0.034611, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.836048, -0.548657, 0.0, 0.577584], + [0.340698, 0.474815, -0.043263, 11.0], + [0.001082, -0.121137, 0.0, 0.01033], + [0.001082, -0.121137, 0.043263, 0.045182], + [0.99996, 0.008928, 0.0, 0.577584], + [0.411001, 0.351514, -0.043263, 11.0], + [-0.034611, 0.030284, 0.0, 0.01033], + [-0.034611, 0.030284, 0.043263, 0.045182], + [-0.658505, -0.752577, 0.0, 0.577584], + [0.411001, 0.351514, -0.043263, 7.0], + [-0.034611, 0.030284, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.658505, -0.752577, 0.0, 0.577584], + [0.412083, 0.38288, 0.043263, 14.0], + [-0.001082, -0.031366, -0.043263, 0.0], + [-0.035692, -0.001082, -0.043263, 0.745403], + [-0.574039, -0.656045, 0.489984, 0.854994], + [0.45102, 0.359085, -0.043263, 11.0], + [-0.040019, -0.007571, 0.0, 0.01033], + [-0.040019, -0.007571, 0.043263, 0.045182], + [0.185892, -0.98257, 0.0, 0.577584], + [0.45102, 0.359085, 0.0, 10.0], + [-0.040019, -0.007571, 0.0, 0.000305], + [-0.038937, 0.023795, 0.043263, 0.879622], + [0.151736, -0.802035, 0.577682, 0.9131], + [0.45102, 0.359085, -0.043263, 7.0], + [-0.040019, -0.007571, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.185892, -0.98257, 0.0, 0.577584], + [0.45102, 0.359085, 0.0, 6.0], + [-0.038937, 0.023795, 0.043263, 0.000305], + [0.003245, 0.0411, 0.032447, 0.879622], + [0.417852, -0.583049, 0.696744, 0.9131], + [0.473733, 0.391533, -0.043263, 11.0], + [-0.022713, -0.032447, 0.0, 0.01033], + [-0.022713, -0.032447, 0.043263, 0.045182], + [0.819232, -0.573462, 0.0, 0.577584], + [0.454264, 0.400185, 0.032447, 14.0], + [0.019468, -0.008653, -0.032447, 0.0], + [-0.003245, -0.0411, -0.032447, 0.527108], + [0.688617, -0.482032, 0.541712, 0.887922], + [0.473733, 0.391533, -0.043263, 7.0], + [-0.022713, -0.032447, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.819232, -0.573462, 0.0, 0.577584], + [-0.223888, 0.407756, -0.043263, 7.0], + [0.002163, 0.06814, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999496, 0.03173, 0.0, 0.577584], + [-0.223888, 0.407756, -0.043263, 11.0], + [0.002163, 0.06814, 0.0, 0.01033], + [0.002163, 0.06814, 0.043263, 0.045182], + [-0.999496, 0.03173, 0.0, 0.577584], + [-0.167645, 0.416409, 0.046508, 14.0], + [-0.056242, -0.008653, -0.046508, 0.000305], + [-0.054079, 0.059487, -0.046508, 0.955978], + [-0.638985, 0.020285, 0.768952, 0.973449], + [-0.167645, 0.416409, 0.046508, 14.0], + [0.004326, -0.035692, -0.046508, 0.000305], + [-0.056242, -0.008653, -0.046508, 0.955978], + [-0.339782, -0.761112, 0.552501, 0.973449], + [-0.221724, 0.475896, 0.0, 14.0], + [0.139524, 0.055161, 0.043263, 0.520989], + [0.054079, -0.059487, 0.046508, 0.973449], + [-0.39307, 0.317373, 0.863001, 0.973449], + [-0.167645, 0.416409, 0.046508, 6.0], + [0.073548, 0.015142, -0.046508, 0.0], + [0.004326, -0.035692, -0.046508, 0.768154], + [0.490917, -0.668482, 0.558688, 0.871366], + [-0.094098, 0.431551, -0.043263, 7.0], + [-0.069221, -0.050834, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.591909, -0.806004, 0.0, 0.577584], + [-0.167645, 0.416409, 0.046508, 10.0], + [0.085445, 0.114648, -0.003245, 0.0], + [0.073548, 0.015142, -0.046508, 0.768154], + [0.548356, -0.38771, 0.740936, 0.871366], + [-0.0822, 0.531057, -0.043263, 11.0], + [-0.011897, -0.099506, 0.0, 0.01033], + [-0.011897, -0.099506, 0.043263, 0.045182], + [0.992928, -0.11872, 0.0, 0.577584], + [-0.044345, 0.23254, 0.0, 7.0], + [-0.001082, 0.332046, -0.043263, 0.01033], + [-0.001082, 0.332046, 0.052998, 0.045182], + [-0.999995, -0.003257, 0.0, 0.577584], + [-0.045426, 0.564586, 0.052998, 14.0], + [0.1233, -0.156829, -0.052998, 0.520989], + [0.014061, -0.335291, 0.005408, 0.088655], + [0.429355, 0.032562, 0.902549, 0.973449], + [0.10924, 0.439122, 0.0, 14.0], + [-0.031366, -0.031366, 0.0, 0.132876], + [-0.154666, 0.125464, 0.052998, 0.028428], + [0.182761, -0.182761, 0.966021, 0.955978], + [0.10924, 0.439122, -0.043263, 11.0], + [-0.031366, -0.031366, 0.0, 0.01033], + [-0.031366, -0.031366, 0.043263, 0.045182], + [0.707107, -0.707107, 0.0, 0.577584], + [0.10924, 0.439122, -0.043263, 7.0], + [-0.031366, -0.031366, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.707107, -0.707107, 0.0, 0.577584], + [0.10924, 0.439122, 0.0, 10.0], + [0.019468, 0.024876, 0.063813, 0.18117], + [0.035692, -0.131953, 0.049753, 0.955978], + [-0.933936, -0.126584, 0.334276, 0.964691], + [0.128708, 0.463999, 0.063813, 14.0], + [0.050834, 0.002163, -0.005408, 0.0], + [0.0822, -0.018387, -0.063813, 0.439658], + [0.07859, -0.92643, 0.368172, 0.871366], + [-0.221724, 0.475896, -0.043263, 7.0], + [0.122219, 0.207664, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.861819, 0.507216, 0.0, 0.577584], + [-0.221724, 0.475896, -0.043263, 9.0], + [0.122219, 0.207664, 0.0, 0.01033], + [0.122219, 0.207664, 0.043263, 0.045182], + [-0.861819, 0.507216, 0.0, 0.577584], + [-0.099506, 0.68356, 0.0, 14.0], + [0.017305, -0.152503, 0.043263, 0.0], + [-0.122219, -0.207664, 0.0, 0.723049], + [-0.365879, 0.215335, 0.905408, 0.854994], + [-0.0822, 0.531057, -0.043263, 7.0], + [-0.011897, -0.099506, 0.043263, 0.01033], + [0.0, 0.0, 0.086527, 0.045182], + [0.992928, -0.11872, 0.0, 0.577584], + [-0.099506, 0.68356, 0.0, 14.0], + [0.054079, -0.118974, 0.052998, 0.0], + [0.017305, -0.152503, 0.043263, 0.514916], + [-0.41958, 0.203352, 0.884647, 0.863157], + [-0.099506, 0.68356, -0.043263, 3.0], + [0.086527, 0.149258, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.86514, 0.501531, 0.0, 0.577584], + [-0.045426, 0.564586, -0.043263, 11.0], + [-0.036774, -0.033529, 0.0, 0.01033], + [-0.036774, -0.033529, 0.086527, 0.045182], + [0.673754, -0.738956, 0.0, 0.577584], + [-0.045426, 0.564586, -0.043263, 7.0], + [-0.036774, -0.033529, 0.086527, 0.01033], + [0.0, 0.0, 0.096261, 0.045182], + [0.673754, -0.738956, 0.0, 0.577584], + [-0.099506, 0.68356, 0.0, 14.0], + [0.086527, 0.149258, 0.0, 0.0], + [0.054079, -0.118974, 0.052998, 0.262257], + [-0.385563, 0.223515, 0.8952, 0.9131], + [-0.099506, 0.68356, -0.043263, 11.0], + [0.086527, 0.149258, 0.0, 0.01033], + [0.086527, 0.149258, 0.043263, 0.045182], + [-0.86514, 0.501531, 0.0, 0.577584], + [-0.012979, 0.832818, 0.0, 14.0], + [0.214153, -0.194685, 0.0822, 0.000305], + [-0.032447, -0.268232, 0.052998, 0.930114], + [-0.176863, 0.211325, 0.961281, 0.973449], + [0.128708, 0.463999, 0.063813, 14.0], + [-0.019468, -0.024876, -0.063813, 0.168276], + [-0.174135, 0.100587, -0.010816, 0.038209], + [-0.469238, -0.764881, 0.441331, 0.955978], + [0.201174, 0.638133, 0.0822, 14.0], + [-0.072466, -0.174135, -0.018387, 0.18117], + [-0.246601, -0.073548, -0.029203, 0.082277], + [-0.098561, -0.063844, 0.993081, 0.973449], + [-0.012979, 0.832818, -0.043263, 7.0], + [0.339617, -0.002163, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.006369, 0.99998, 0.0, 0.577584], + [0.179543, 0.466162, 0.058405, 10.0], + [-0.050834, -0.002163, 0.005408, 0.000305], + [0.021632, 0.171972, 0.023795, 0.022171], + [0.110896, -0.149888, 0.982464, 0.381323], + [-0.012979, 0.832818, 0.0, 14.0], + [0.339617, -0.002163, 0.0, 0.0], + [0.214153, -0.194685, 0.0822, 0.027314], + [0.002492, 0.391296, 0.920261, 0.49102], + [0.183869, 0.317985, 0.037855, 10.0], + [-0.055161, 0.146014, 0.025958, 0.0], + [0.02704, 0.127627, -0.037855, 0.439658], + [0.623841, 0.097823, 0.775405, 0.871366], + [0.210908, 0.445612, -0.043263, 7.0], + [0.0, -0.131953, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [1.0, 0.0, 0.0, 0.577584], + [0.210908, 0.445612, 0.0, 10.0], + [-0.031366, 0.02055, 0.058405, 0.000305], + [0.035692, 0.014061, 0.06165, 0.022171], + [-0.105864, -0.954447, 0.278971, 0.381323], + [0.23903, 0.45102, -0.043263, 11.0], + [-0.028121, -0.005408, 0.0, 0.01033], + [-0.028121, -0.005408, 0.043263, 0.045182], + [0.188847, -0.982006, 0.0, 0.577584], + [0.23903, 0.45102, -0.043263, 7.0], + [-0.028121, -0.005408, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.188847, -0.982006, 0.0, 0.577584], + [0.246601, 0.459672, 0.06165, 11.0], + [-0.007571, -0.008653, -0.06165, 0.000305], + [-0.035692, -0.014061, -0.06165, 0.022171], + [0.187618, -0.975617, 0.113888, 0.381323], + [0.241193, 0.343943, -0.043263, 11.0], + [-0.002163, 0.107077, 0.0, 0.01033], + [-0.002163, 0.107077, 0.043263, 0.045182], + [-0.999796, -0.020198, 0.0, 0.577584], + [0.23903, 0.45102, 0.0, 15.0], + [0.007571, 0.008653, 0.06165, 0.000305], + [0.042182, -0.108158, 0.046508, 0.846876], + [-0.94107, -0.299262, 0.157572, 0.896269], + [0.316904, 0.358004, 0.036774, 14.0], + [-0.070303, 0.101669, 0.024876, 0.0], + [0.023795, 0.116811, -0.036774, 0.473533], + [0.523424, 0.157027, 0.837478, 0.930114], + [0.340698, 0.474815, -0.043263, 7.0], + [0.001082, -0.121137, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.99996, 0.008928, 0.0, 0.577584], + [0.376391, 0.381799, -0.043263, 11.0], + [-0.001082, 0.107077, 0.0, 0.01033], + [-0.001082, 0.107077, 0.043263, 0.045182], + [-0.999949, -0.0101, 0.0, 0.577584], + [0.376391, 0.381799, -0.043263, 7.0], + [-0.001082, 0.107077, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999949, -0.0101, 0.0, 0.577584], + [0.375309, 0.488875, 0.0, 14.0], + [0.036774, -0.105995, 0.043263, 0.000305], + [0.001082, -0.107077, 0.0, 0.879622], + [-0.771254, -0.00779, 0.63648, 0.896269], + [0.375309, 0.488875, 0.0, 14.0], + [0.078955, -0.08869, 0.032447, 0.223224], + [0.036774, -0.105995, 0.043263, 0.964691], + [0.071225, 0.39802, 0.914608, 0.973449], + [0.473733, 0.603523, 0.0, 14.0], + [0.0, -0.21199, 0.0, 0.158953], + [-0.019468, -0.203337, 0.032447, 0.030716], + [0.857493, 0.0, 0.514496, 0.964691], + [0.473733, 0.603523, -0.043263, 11.0], + [0.0, -0.21199, 0.0, 0.01033], + [0.0, -0.21199, 0.043263, 0.045182], + [1.0, 0.0, 0.0, 0.577584], + [0.276885, 0.484549, 0.058405, 10.0], + [-0.030284, -0.024876, 0.003245, 0.000305], + [-0.097342, -0.018387, 0.0, 0.022171], + [-0.03153, 0.166923, 0.985466, 0.381323], + [0.246601, 0.459672, 0.06165, 14.0], + [0.030284, 0.024876, -0.003245, 0.0], + [0.094098, 0.015142, -0.06165, 0.473533], + [0.518869, -0.545855, 0.657888, 0.930114], + [0.375309, 0.488875, -0.043263, 11.0], + [-0.034611, -0.014061, 0.0, 0.01033], + [-0.034611, -0.014061, 0.043263, 0.045182], + [0.376377, -0.926467, 0.0, 0.577584], + [0.340698, 0.474815, 0.0, 14.0], + [-0.063813, 0.009734, 0.058405, 0.000305], + [0.034611, 0.014061, 0.0, 0.022171], + [0.3276, -0.806402, 0.492334, 0.381323], + [0.375309, 0.488875, -0.043263, 7.0], + [-0.034611, -0.014061, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.376377, -0.926467, 0.0, 0.577584], + [0.454264, 0.400185, 0.032447, 12.0], + [-0.078955, 0.08869, -0.032447, 0.158953], + [0.019468, 0.203337, -0.032447, 0.030716], + [-0.201684, 0.173144, 0.964025, 0.964691], + [0.201174, 0.638133, 0.0822, 8.0], + [0.075711, -0.153585, -0.023795, 0.000305], + [-0.021632, -0.171972, -0.023795, 0.022171], + [0.026497, -0.140281, 0.989757, 0.381323], + [0.473733, 0.603523, -0.043263, 7.0], + [0.0, -0.21199, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [1.0, 0.0, 0.0, 0.577584], + [0.276885, 0.484549, 0.058405, 14.0], + [-0.075711, 0.153585, 0.023795, 0.000305], + [0.098424, 0.004326, -0.058405, 0.022171], + [0.50316, 0.115345, 0.856461, 0.381323], + [0.375309, 0.488875, 0.0, 10.0], + [0.090853, 0.12979, 0.0, 0.158953], + [0.098424, 0.114648, 0.0, 0.030716], + [0.0, 0.0, 1.0, 0.964691], + [0.466162, 0.618665, -0.043263, 11.0], + [0.007571, -0.015142, 0.0, 0.01033], + [0.007571, -0.015142, 0.043263, 0.045182], + [0.894427, 0.447214, 0.0, 0.577584], + [0.466162, 0.618665, -0.043263, 3.0], + [0.007571, -0.015142, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.894427, 0.447214, 0.0, 0.577584], + [0.326638, 0.830655, 0.0, 14.0], + [0.139524, -0.21199, 0.0, 0.0], + [0.048671, -0.34178, 0.0, 0.015213], + [0.0, 0.0, 1.0, 0.428687], + [0.326638, 0.830655, 0.0, 14.0], + [0.048671, -0.34178, 0.0, 0.007492], + [-0.125464, -0.192522, 0.0822, 0.054475], + [0.472493, 0.067285, 0.878762, 0.964691], + [0.326638, 0.830655, -0.043263, 9.0], + [0.139524, -0.21199, 0.0, 0.01033], + [0.139524, -0.21199, 0.043263, 0.045182], + [0.835314, 0.549773, 0.0, 0.577584], + [0.326638, 0.830655, -0.043263, 7.0], + [0.139524, -0.21199, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.835314, 0.549773, 0.0, 0.577584], + [-0.012979, 0.832818, -0.043263, 11.0], + [0.339617, -0.002163, 0.0, 0.01033], + [0.339617, -0.002163, 0.043263, 0.045182], + [0.006369, 0.99998, 0.0, 0.577584], +]; + +pub const POLYS: [[f32; 2]; 110] = [ + [0.006442, -0.002147], + [-0.001074, 0.001074], + [-0.004294, 0.007515], + [-0.004294, 0.806288], + [-0.001074, 0.809509], + [0.004294, 0.809509], + [0.022546, 0.796626], + [0.185736, 0.655982], + [0.281288, 0.86319], + [0.286656, 0.870706], + [0.293098, 0.871779], + [0.465951, 0.781595], + [0.478834, 0.773006], + [0.478834, 0.763344], + [0.407975, 0.592638], + [0.45092, 0.586196], + [0.601227, 0.573313], + [0.606595, 0.569018], + [0.606595, 0.564724], + [0.601227, 0.556135], + [0.50138, 0.453067], + [0.425153, 0.37684], + [0.289877, 0.246933], + [0.081595, 0.057975], + [0.013957, 0.0], + [-0.011897, 0.0], + [-0.055161, 0.037855], + [-0.081119, 0.063813], + [-0.083282, 0.06814], + [-0.084363, 0.38288], + [-0.089771, 0.38288], + [-0.158993, 0.33529], + [-0.166564, 0.336372], + [-0.250927, 0.379635], + [-0.261743, 0.388288], + [-0.263906, 0.461836], + [-0.262824, 0.486712], + [-0.173053, 0.638133], + [-0.044345, 0.860939], + [-0.036774, 0.867429], + [-0.024876, 0.871755], + [0.33529, 0.871755], + [0.349351, 0.867429], + [0.363412, 0.853368], + [0.513752, 0.619747], + [0.513752, 0.379635], + [0.466162, 0.322311], + [0.461836, 0.320148], + [0.399104, 0.311496], + [0.393696, 0.313659], + [0.374227, 0.332046], + [0.343943, 0.288782], + [0.336372, 0.282293], + [0.269314, 0.271477], + [0.263906, 0.27364], + [0.243356, 0.292027], + [0.21199, 0.244438], + [0.207664, 0.241193], + [0.149258, 0.231459], + [0.138443, 0.231459], + [0.118974, 0.248764], + [0.116811, 0.222806], + [0.117892, 0.162237], + [0.115729, 0.076792], + [0.06814, 0.011897], + [0.062732, 0.007571], + [0.005408, 0.0], + [0.031135, 0.06227], + [0.252301, 0.266258], + [0.541104, 0.544325], + [0.359663, 0.559356], + [0.433742, 0.753681], + [0.304908, 0.821319], + [0.199693, 0.595859], + [0.035429, 0.740798], + [0.035429, 0.482055], + [-0.043263, 0.077874], + [0.001082, 0.037855], + [0.044345, 0.043263], + [0.075711, 0.08869], + [0.077874, 0.407756], + [0.10924, 0.439122], + [0.111403, 0.307169], + [0.147095, 0.272559], + [0.189277, 0.28013], + [0.210909, 0.313659], + [0.210909, 0.445612], + [0.23903, 0.45102], + [0.241193, 0.343943], + [0.276885, 0.311496], + [0.319067, 0.319067], + [0.34178, 0.353677], + [0.340698, 0.474815], + [0.375309, 0.488875], + [0.376391, 0.381799], + [0.411001, 0.351514], + [0.45102, 0.359085], + [0.473733, 0.391533], + [0.473733, 0.603523], + [0.466162, 0.618665], + [0.326638, 0.830655], + [-0.012979, 0.832818], + [-0.099506, 0.68356], + [-0.221724, 0.475896], + [-0.223888, 0.407756], + [-0.163319, 0.380717], + [-0.094098, 0.431551], + [-0.0822, 0.531057], + [-0.045426, 0.564586], + [-0.044345, 0.23254], +]; + +pub const BOXES: [[f32; 4]; 44] = [ + [0.030135, 0.06127, -0.043945, 0.0], + [0.330601, 0.483055, 0.116951, 7.0], + [0.034429, 0.265258, -0.043945, 7.0], + [0.542104, 0.741798, 0.116951, 8.0], + [0.034429, 0.355442, -0.043945, 15.0], + [0.305908, 0.822319, 0.116951, 8.0], + [0.267405, 0.532589, -0.043945, 23.0], + [0.542104, 0.822319, 0.082595, 8.0], + [-0.045345, 0.036855, -0.044263, 31.0], + [0.045345, 0.23354, 0.059405, 8.0], + [-0.224888, 0.072548, -0.044263, 39.0], + [0.059405, 0.565586, 0.059405, 8.0], + [-0.032366, 0.042263, -0.044263, 47.0], + [0.148095, 0.408756, 0.059405, 8.0], + [0.074711, 0.08769, -0.044263, 55.0], + [0.184869, 0.464999, 0.064813, 8.0], + [0.146095, 0.271559, -0.044263, 63.0], + [0.211908, 0.446612, 0.038855, 8.0], + [0.23803, 0.310496, -0.044263, 71.0], + [0.320067, 0.460672, 0.06265, 8.0], + [0.275885, 0.310496, -0.044263, 79.0], + [0.412001, 0.475815, 0.047508, 8.0], + [0.375391, 0.350514, -0.044263, 87.0], + [0.45202, 0.38388, 0.044263, 4.0], + [0.410001, 0.350514, -0.044263, 91.0], + [0.474733, 0.401185, 0.044263, 5.0], + [-0.224888, 0.379717, -0.044263, 96.0], + [-0.0812, 0.532057, 0.047508, 8.0], + [-0.095098, 0.228295, -0.044263, 104.0], + [0.211908, 0.565586, 0.064813, 8.0], + [-0.222724, 0.430551, -0.044263, 112.0], + [-0.011979, 0.833818, 0.053998, 8.0], + [-0.100506, 0.438122, -0.044263, 120.0], + [0.327638, 0.833818, 0.0832, 8.0], + [0.127708, 0.312659, -0.044263, 128.0], + [0.282211, 0.467162, 0.064813, 8.0], + [0.245601, 0.352677, -0.044263, 136.0], + [0.474733, 0.604523, 0.06265, 8.0], + [0.178543, 0.390533, -0.044263, 144.0], + [0.474733, 0.639133, 0.0832, 8.0], + [0.200174, 0.483549, -0.044263, 152.0], + [0.474733, 0.639133, 0.0832, 4.0], + [-0.013979, 0.487875, -0.044263, 156.0], + [0.467162, 0.833818, 0.0832, 5.0], +]; diff --git a/crates/compositor/src/regions.rs b/crates/compositor/src/regions.rs index 097863e29..971525b53 100644 --- a/crates/compositor/src/regions.rs +++ b/crates/compositor/src/regions.rs @@ -8,7 +8,9 @@ use crate::camera::Follow; use crate::cursor::CursorTrack; -use crate::scene::{SceneCameraFullscreenRegion, SceneSpeedRegion, SceneZoomRegion}; +use crate::scene::{ + SceneCameraFullscreenRegion, SceneCameraLayoutRegion, SceneSpeedRegion, SceneZoomRegion, +}; /// Quantification commune vidéo/audio : le web retranche exactement 1 ms avant `ceil`. pub const SPEED_FRAME_EPSILON_SEC: f64 = 0.001; @@ -49,18 +51,22 @@ pub fn speed_segments_for_window( for region in overlapping { let start = region.start_sec.max(source_start_sec).max(cursor); let end = region.end_sec.min(source_end_sec); + // Projection can leave a zero-width region at a trim boundary. It may still pass + // the overlap filter when rounding puts its start just inside the clip window. Skip + // it before emitting the 1x gap, or the unchanged cursor makes that gap render twice. + if end <= start { + continue; + } if start > cursor { push_speed_segment(&mut spans, cursor, start, 1.0, fps); } - if end > start { - let speed = if region.speed.is_finite() && region.speed > 0.0 { - region.speed - } else { - 1.0 - }; - push_speed_segment(&mut spans, start, end, speed, fps); - cursor = end; - } + let speed = if region.speed.is_finite() && region.speed > 0.0 { + region.speed + } else { + 1.0 + }; + push_speed_segment(&mut spans, start, end, speed, fps); + cursor = end; } if cursor < source_end_sec { push_speed_segment(&mut spans, cursor, source_end_sec, 1.0, fps); @@ -267,8 +273,11 @@ fn push_speed_segment( } // mêmes fenêtres de transition que le web (TRANSITION_WINDOW_MS etc., converties en secondes). -const TRANSITION_WINDOW_S: f32 = 1.01505; -const FULLSCREEN_LEAD_OUT_WINDOW_S: f32 = TRANSITION_WINDOW_S * 1.5; +pub(crate) const TRANSITION_WINDOW_S: f32 = 1.01505; +pub(crate) const FULLSCREEN_LEAD_OUT_WINDOW_S: f32 = TRANSITION_WINDOW_S * 1.5; +/// Two camera regions (layout or Full Camera) closer than this (source seconds) meet at a +/// seam: they hand over directly, without the default PiP in between. +pub(crate) const REGION_SEAM_S: f64 = 0.001; // Durée d'un zoom à l'écran selon l'échelle visée, cf. `zoom_transition_s` ; miroir de // `ZOOM_TRANSITION_BASE_MS` / `ZOOM_TRANSITION_PER_LN_MS` (TS). const ZOOM_TRANSITION_BASE_S: f32 = 0.6; @@ -324,7 +333,7 @@ fn cubic_bezier(x1: f32, y1: f32, x2: f32, y2: f32, t: f32) -> f32 { } /// Port de `easeOutScreenStudio` (TS) : cubic-bezier(0.16, 1, 0.3, 1). -fn ease_out_screen_studio(t: f32) -> f32 { +pub(crate) fn ease_out_screen_studio(t: f32) -> f32 { cubic_bezier(0.16, 1.0, 0.3, 1.0, t) } @@ -462,6 +471,10 @@ pub fn zoom_cursor_alpha(regions: &[SceneZoomRegion], t: f32, clock: &ScreenCloc /// pixel shader par `compositor.rs`, ce module ne fait que le calcul temporel). pub struct ZoomState { pub scale: f32, + /// Force de la région (0..1) : plein régime sur son span et sur les transitions/paliers + /// chaînés, 0 hors zoom. C'est l'enveloppe du zoom — le rétrécissement réactif de la webcam + /// (`reactive_scale`) s'y accroche pour animer sa taille, au lieu de suivre la profondeur. + pub strength: f32, pub focus: [f32; 2], pub rotation: [f32; 3], /// À quel point un préset 3D est installé (0..1) : la force de la région quand elle en porte @@ -479,6 +492,7 @@ pub struct ZoomState { const IDENTITY_ZOOM: ZoomState = ZoomState { scale: 1.0, + strength: 0.0, focus: [0.5, 0.5], rotation: [0.0, 0.0, 0.0], tilt: 0.0, @@ -865,6 +879,7 @@ pub fn zoom_state_in( |p: [f32; 2], q: [f32; 2]| [lerp(p[0], q[0], progress), lerp(p[1], q[1], progress)]; return ZoomState { scale: scale_lerp(cur.scale, next.scale, progress), + strength: 1.0, focus: [ lerp(cur_focus[0], next_focus[0], progress), lerp(cur_focus[1], next_focus[1], progress), @@ -886,6 +901,7 @@ pub fn zoom_state_in( let seen = orbit_at(next, t, frame); return ZoomState { scale: next.scale, + strength: 1.0, focus: resolve_focus(next, t, cursor, clock), rotation: fixed_rotation(next), tilt: fixed_flag(next), @@ -939,6 +955,7 @@ pub fn zoom_state_in( let seen = orbit_at(r, t, frame); ZoomState { scale, + strength, focus: [ease(focus[0]), ease(focus[1])], rotation: lerp_rotation3d([0.0, 0.0, 0.0], fixed_rotation(r), strength), tilt: fixed_flag(r) * strength, @@ -961,16 +978,20 @@ fn camera_fullscreen_region_phase( region: &SceneCameraFullscreenRegion, t: f32, clock: &ScreenClock, + layouts: &[SceneCameraLayoutRegion], ) -> f32 { let start = region.start_sec as f32; let end = region.end_sec as f32; if t <= start || t >= end { return 0.0; } + // At a seam with a layout region the camera is already frame-filling on that side: the + // layout region's own glide (`camera_layers_at`) moves it, so the phase holds 1 there. + let seams = full_camera_seams(region, layouts); let (start, end, t) = (clock.at(start), clock.at(end), clock.at(t)); let half = (end - start) * 0.5; - let lead_in = TRANSITION_WINDOW_S.min(half); - let lead_out = FULLSCREEN_LEAD_OUT_WINDOW_S.min(half); + let lead_in = if seams.start { 0.0 } else { TRANSITION_WINDOW_S.min(half) }; + let lead_out = if seams.end { 0.0 } else { FULLSCREEN_LEAD_OUT_WINDOW_S.min(half) }; let lead_in_end = start + lead_in; let lead_out_start = end - lead_out; if t < lead_in_end { @@ -990,16 +1011,38 @@ fn camera_fullscreen_region_phase( } } +/// Which ends of a Full Camera region meet a (non-empty) camera layout region at a seam +/// (`REGION_SEAM_S`): `start` = a layout region ends where it starts, `end` = one starts where +/// it ends. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct FullCameraSeams { + pub start: bool, + pub end: bool, +} + +pub(crate) fn full_camera_seams( + region: &SceneCameraFullscreenRegion, + layouts: &[SceneCameraLayoutRegion], +) -> FullCameraSeams { + let mut seams = FullCameraSeams::default(); + for l in layouts.iter().filter(|l| l.end_sec > l.start_sec) { + seams.start |= (l.end_sec - region.start_sec).abs() <= REGION_SEAM_S; + seams.end |= (l.start_sec - region.end_sec).abs() <= REGION_SEAM_S; + } + seams +} + /// Phase Full Camera (0..1) au temps `t`. Régions superposées (ne devrait pas arriver, gardé /// défensif comme le web) → la plus forte gagne. fn camera_fullscreen_phase_at( regions: &[SceneCameraFullscreenRegion], t: f32, clock: &ScreenClock, + layouts: &[SceneCameraLayoutRegion], ) -> f32 { let mut strongest = 0.0f32; for r in regions { - let s = camera_fullscreen_region_phase(r, t, clock); + let s = camera_fullscreen_region_phase(r, t, clock, layouts); if s > strongest { strongest = s; } @@ -1012,8 +1055,9 @@ pub fn camera_fullscreen_progress_at( regions: &[SceneCameraFullscreenRegion], t: f32, clock: &ScreenClock, + layouts: &[SceneCameraLayoutRegion], ) -> f32 { - ease_out_screen_studio(camera_fullscreen_phase_at(regions, t, clock)) + ease_out_screen_studio(camera_fullscreen_phase_at(regions, t, clock, layouts)) } /// Ce qui reste de la bulle PiP (coins arrondis, ombre) au temps `t` : 1 = bulle entière, @@ -1026,8 +1070,72 @@ pub fn camera_fullscreen_shape_at( regions: &[SceneCameraFullscreenRegion], t: f32, clock: &ScreenClock, + layouts: &[SceneCameraLayoutRegion], +) -> f32 { + 1.0 - smoothstep(0.5, 1.0, camera_fullscreen_phase_at(regions, t, clock, layouts)) +} + +/// The Full Camera region in effect at `t`: the one with the strongest phase, so a region's +/// orientation and its grow/shrink always refer to the same region. `None` wherever the +/// envelope is 0 — outside every region and on their bounds. +pub fn camera_fullscreen_region_at<'a>( + regions: &'a [SceneCameraFullscreenRegion], + t: f32, + clock: &ScreenClock, + layouts: &[SceneCameraLayoutRegion], +) -> Option<&'a SceneCameraFullscreenRegion> { + let mut best: Option<(&'a SceneCameraFullscreenRegion, f32)> = None; + for r in regions { + let phase = camera_fullscreen_region_phase(r, t, clock, layouts); + if phase > 0.0 && best.map_or(true, |(_, b)| phase > b) { + best = Some((r, phase)); + } + } + best.map(|(r, _)| r) +} + +/// How long a turned section's camera takes to sharpen after the hold, and to blur again +/// before the shrink. See `camera_fullscreen_cover_at`. +pub const DESK_COVER_FADE_S: f32 = 0.5; + +/// Cover strength of the webcam at `t` (0 = sharp, 1 = fully blurred and dimmed). Only a +/// turned section (a camera tilted onto the desk) is covered: the camera is moving at both +/// ends of such a section, so the picture is hidden for exactly the grow and the shrink, plus a +/// short fade into and out of the steady part. Measured on screen time like the grow, and 0 +/// on and outside the bounds like the grow. At a seam with a layout region (`layouts`) that +/// side has no grow or shrink to hold over: the cover is only its own fade there. +pub fn camera_fullscreen_cover_at( + regions: &[SceneCameraFullscreenRegion], + t: f32, + clock: &ScreenClock, + layouts: &[SceneCameraLayoutRegion], ) -> f32 { - 1.0 - smoothstep(0.5, 1.0, camera_fullscreen_phase_at(regions, t, clock)) + let Some(r) = camera_fullscreen_region_at(regions, t, clock, layouts) else { + return 0.0; + }; + if r.rotation != 180 { + return 0.0; + } + let seams = full_camera_seams(r, layouts); + let (start, end, t) = (clock.at(r.start_sec as f32), clock.at(r.end_sec as f32), clock.at(t)); + // Each hold gets at most half the section; the two fades share what is left, so the start + // fade ends at or before the point where the end fade begins and the cover never jumps. + let len = end - start; + let half = len * 0.5; + let hold_in = if seams.start { 0.0 } else { TRANSITION_WINDOW_S.min(half) }; + let hold_out = if seams.end { 0.0 } else { FULLSCREEN_LEAD_OUT_WINDOW_S.min(half) }; + let steady = (len - hold_in - hold_out).max(0.0); + let fade = DESK_COVER_FADE_S.min(steady * 0.5); + let side = |since: f32, hold: f32, fade: f32| -> f32 { + if since <= hold { + 1.0 + } else if fade > 0.0 && since < hold + fade { + 1.0 - smoothstep(0.0, fade, since - hold) + } else { + 0.0 + } + }; + side(t - start, hold_in, fade).max(side(end - t, hold_out, fade)) } // ============ Rotation 3D (tilt perspective, présets iso/left/right) ================ @@ -1965,6 +2073,9 @@ mod zoom_focus_tests { clip_index: None, start_sec: 18.0 * k, end_sec: 22.0 * k, + rotation: 0, + mirror: None, + full_frame: false, }] }; let (sped, plain) = (zooms(4.0), zooms(1.0)); @@ -1984,21 +2095,223 @@ mod zoom_focus_tests { b.scale ); assert!((a.focus[0] - b.focus[0]).abs() < 1e-4, "t={t}"); - let a = camera_fullscreen_progress_at(&full_camera(4.0), t, &clock); + let a = camera_fullscreen_progress_at(&full_camera(4.0), t, &clock, &[]); let b = - camera_fullscreen_progress_at(&full_camera(1.0), t / 4.0, &ScreenClock::default()); + camera_fullscreen_progress_at(&full_camera(1.0), t / 4.0, &ScreenClock::default(), &[]); assert!((a - b).abs() < 1e-4, "Full Camera t={t} : {a} vs {b}"); } } + fn cam(start: f64, end: f64, rotation: u16) -> SceneCameraFullscreenRegion { + SceneCameraFullscreenRegion { + clip_index: None, + start_sec: start, + end_sec: end, + rotation, + mirror: None, + full_frame: rotation != 0, + } + } + + /// Orientation and transition must name the same region: inside it the region, at its + /// bounds and outside nothing, exactly where the progress envelope is 0. + #[test] + fn the_full_camera_region_at_t_is_the_one_the_envelope_uses() { + let r = [cam(10.0, 20.0, 180), cam(30.0, 40.0, 0)]; + let clock = ScreenClock::default(); + assert_eq!(camera_fullscreen_region_at(&r, 15.0, &clock, &[]).map(|c| c.rotation), Some(180)); + assert_eq!(camera_fullscreen_region_at(&r, 35.0, &clock, &[]).map(|c| c.rotation), Some(0)); + for t in [5.0, 10.0, 20.0, 25.0] { + assert!(camera_fullscreen_region_at(&r, t, &clock, &[]).is_none(), "t={t}"); + assert_eq!(camera_fullscreen_progress_at(&r, t, &clock, &[]), 0.0, "t={t}"); + } + } + + + /// The hold covers exactly the grow; then the picture sharpens over the fade. + #[test] + fn a_turned_section_is_covered_while_the_camera_moves() { + let r = [cam(10.0, 30.0, 180)]; + let clock = ScreenClock::default(); + let c = |t: f32| camera_fullscreen_cover_at(&r, t, &clock, &[]); + assert_eq!(c(10.0 + 0.01), 1.0, "start of the hold"); + assert_eq!(c(10.0 + TRANSITION_WINDOW_S - 0.01), 1.0, "end of the hold"); + let mid_fade = c(10.0 + TRANSITION_WINDOW_S + DESK_COVER_FADE_S / 2.0); + assert!((mid_fade - 0.5).abs() < 1e-3, "half way through the fade: {mid_fade}"); + assert_eq!(c(20.0), 0.0, "steady part is sharp"); + // Mirrored at the end: fade up, then hold for the whole shrink. + let mid_rise = c(30.0 - FULLSCREEN_LEAD_OUT_WINDOW_S - DESK_COVER_FADE_S / 2.0); + assert!((mid_rise - 0.5).abs() < 1e-3, "half way up: {mid_rise}"); + assert_eq!(c(30.0 - FULLSCREEN_LEAD_OUT_WINDOW_S + 0.01), 1.0); + assert_eq!(c(30.0 - 0.01), 1.0); + } + + fn layout_region(start: f64, end: f64) -> SceneCameraLayoutRegion { + SceneCameraLayoutRegion { clip_index: None, start_sec: start, end_sec: end, layers: Vec::new() } + } + + /// A layout region meeting a Full Camera region at a seam: the phase holds 1 on that side + /// (no lead-out before it, no lead-in after it); the other side is unchanged. + #[test] + fn the_full_camera_phase_holds_at_a_seam_with_a_layout_region() { + let clock = ScreenClock::default(); + let r = [cam(10.0, 20.0, 0)]; + let p = |t: f32, layouts: &[SceneCameraLayoutRegion]| { + camera_fullscreen_progress_at(&r, t, &clock, layouts) + }; + let after = [layout_region(20.0, 25.0)]; + let before = [layout_region(5.0, 10.0)]; + for t in [19.0, 19.5, 19.99] { + assert_eq!(p(t, &after), 1.0, "t={t}"); + assert!(p(t, &[]) < 1.0, "a lone region leads out (t={t})"); + } + assert!(p(10.3, &after) < 1.0, "the far side keeps its lead-in"); + for t in [10.01, 10.3, 10.9] { + assert_eq!(p(t, &before), 1.0, "t={t}"); + } + assert!(p(19.5, &before) < 1.0, "the far side keeps its lead-out"); + // On the bounds the phase is 0 as always: the layout region draws the seam. + assert_eq!(p(20.0, &after), 0.0); + assert_eq!(p(10.0, &before), 0.0); + // Off the seam (10 ms), or an empty layout region, changes nothing. + for layouts in [[layout_region(20.01, 25.0)], [layout_region(20.0, 20.0)]] { + assert_eq!(p(19.5, &layouts), p(19.5, &[])); + } + } + + /// A turned section at a seam: no hold on that side, the cover is its own fade there. + #[test] + fn at_a_seam_the_cover_is_only_its_fade() { + let clock = ScreenClock::default(); + let r = [cam(10.0, 30.0, 180)]; + let layouts = [layout_region(5.0, 10.0), layout_region(30.0, 35.0)]; + let c = |t: f32| camera_fullscreen_cover_at(&r, t, &clock, &layouts); + assert!(c(10.001) > 0.99, "covered at the seam: {}", c(10.001)); + let mid_fade = c(10.0 + DESK_COVER_FADE_S / 2.0); + assert!((mid_fade - 0.5).abs() < 1e-3, "half way through the fade: {mid_fade}"); + assert_eq!(c(10.0 + DESK_COVER_FADE_S + 0.01), 0.0, "sharp after the fade"); + let mid_rise = c(30.0 - DESK_COVER_FADE_S / 2.0); + assert!((mid_rise - 0.5).abs() < 1e-3, "half way up: {mid_rise}"); + assert_eq!(c(30.0 - DESK_COVER_FADE_S - 0.01), 0.0); + assert!(c(29.999) > 0.99); + // Without the seams the holds are back. + let lone = |t: f32| camera_fullscreen_cover_at(&r, t, &clock, &[]); + assert_eq!(lone(10.0 + DESK_COVER_FADE_S + 0.01), 1.0); + } + + #[test] + fn the_cover_is_zero_on_and_outside_the_bounds() { + let r = [cam(10.0, 30.0, 180)]; + let clock = ScreenClock::default(); + for t in [0.0, 9.99, 10.0, 30.0, 30.01, 40.0] { + assert_eq!(camera_fullscreen_cover_at(&r, t, &clock, &[]), 0.0, "t={t}"); + } + } + + #[test] + fn a_plain_section_is_never_covered() { + let r = [cam(10.0, 30.0, 0)]; + let clock = ScreenClock::default(); + for step in 0..=400 { + let t = step as f32 * 0.1; + assert_eq!(camera_fullscreen_cover_at(&r, t, &clock, &[]), 0.0, "t={t}"); + } + } + + /// Each hold gets at most half the section and the fades share what is left, so a section + /// shorter than both holds is covered throughout and a longer one keeps its full fades. + #[test] + fn short_sections_shrink_the_fades_first() { + let clock = ScreenClock::default(); + let short = [cam(10.0, 11.5, 180)]; // half = 0.75 < hold-in: no fade at all + for step in 1..150 { + let t = 10.0 + step as f32 * 0.01; + let c = camera_fullscreen_cover_at(&short, t, &clock, &[]); + assert!((0.0..=1.0).contains(&c), "t={t}: {c}"); + assert_eq!(c, 1.0, "a section shorter than both holds is covered throughout (t={t})"); + } + // 3.6 s: steady = 3.6 - 1.015 - 1.523 = 1.06, so both ends keep their full 0.5 s fade. + let medium = [cam(10.0, 13.6, 180)]; + let c = |t: f32| camera_fullscreen_cover_at(&medium, t, &clock, &[]); + let mid_fade = c(10.0 + TRANSITION_WINDOW_S + DESK_COVER_FADE_S / 2.0); + assert!((mid_fade - 0.5).abs() < 1e-3, "start fade half way: {mid_fade}"); + let mid_rise = c(13.6 - FULLSCREEN_LEAD_OUT_WINDOW_S - DESK_COVER_FADE_S / 2.0); + assert!((mid_rise - 0.5).abs() < 1e-3, "end fade half way: {mid_rise}"); + } + + /// 2.6 s: hold-in 1.015, hold-out 1.3 (half), steady 0.285 — the two fades share it, + /// 0.1425 s each, so the start fade ends exactly where the end fade begins. + #[test] + fn medium_sections_share_the_steady_part_between_the_fades() { + let clock = ScreenClock::default(); + let r = [cam(10.0, 12.6, 180)]; + let c = |t: f32| camera_fullscreen_cover_at(&r, t, &clock, &[]); + let fade = (2.6 - TRANSITION_WINDOW_S - 1.3) / 2.0; + let meet = 10.0 + TRANSITION_WINDOW_S + fade; + let mid_fade = c(10.0 + TRANSITION_WINDOW_S + fade / 2.0); + assert!((mid_fade - 0.5).abs() < 1e-2, "start fade half way: {mid_fade}"); + assert!(c(meet) < 1e-2, "both fades are near zero where they meet: {}", c(meet)); + let mid_rise = c(12.6 - 1.3 - fade / 2.0); + assert!((mid_rise - 0.5).abs() < 1e-2, "end fade half way: {mid_rise}"); + assert_eq!(c(12.6 - 1.3 + 0.01), 1.0, "the end hold is half the section"); + } + + /// No section length makes the cover jump: sampled every 1 ms, no step exceeds 0.1. (At + /// 10 ms the shortest legitimate fade — 42 ms in a 2.2 s section — already steps ~0.3 per + /// sample, so the finer grid is what tells a ramp from a jump.) + #[test] + fn the_cover_is_continuous_for_every_section_length() { + let clock = ScreenClock::default(); + for len in [2.0_f32, 2.2, 2.6, 3.0, 3.5, 4.0, 4.5] { + let r = [cam(10.0, 10.0 + len as f64, 180)]; + let steps = (len * 1000.0).round() as i32; + let mut prev = camera_fullscreen_cover_at(&r, 10.0 + 0.0005, &clock, &[]); + for step in 1..steps { + let t = 10.0 + step as f32 * 0.001; + let c = camera_fullscreen_cover_at(&r, t, &clock, &[]); + assert!((0.0..=1.0).contains(&c), "len={len} t={t}: {c}"); + assert!((c - prev).abs() <= 0.1, "len={len} t={t}: {prev} -> {c}"); + prev = c; + } + } + } + + /// Under a speed change the windows are measured on screen time, like the grow. + #[test] + fn the_cover_runs_on_screen_time() { + let clock = ScreenClock::new(&[speed(None, 0.0, 1000.0, 2.0)], 0); + let r = [cam(10.0, 40.0, 180)]; + // At 2x, the 1.015 s hold spans 2.03 s of source time. + assert_eq!(camera_fullscreen_cover_at(&r, 10.0 + 1.9, &clock, &[]), 1.0); + assert!(camera_fullscreen_cover_at(&r, 10.0 + 3.6, &clock, &[]) < 1.0); + } + #[test] + fn the_region_fields_parse_and_default() { + let plain: SceneCameraFullscreenRegion = + serde_json::from_str(r#"{"startSec":1,"endSec":2}"#).unwrap(); + assert_eq!((plain.rotation, plain.mirror, plain.full_frame), (0, None, false)); + let desk: SceneCameraFullscreenRegion = serde_json::from_str( + r#"{"startSec":1,"endSec":2,"rotation":180,"mirror":false,"fullFrame":true}"#, + ) + .unwrap(); + assert_eq!((desk.rotation, desk.mirror, desk.full_frame), (180, Some(false), true)); + } + /// Les coins suivent la phase, pas le rect. Avec `1 - progrès`, ils étaient carrés presque /// tout du long de la montée et ne revenaient qu'à la toute fin du retour. #[test] fn full_camera_corners_dissolve_late_and_come_back_early() { - let r = [SceneCameraFullscreenRegion { clip_index: None, start_sec: 10.0, end_sec: 20.0 }]; + let r = [SceneCameraFullscreenRegion { + clip_index: None, + start_sec: 10.0, + end_sec: 20.0, + rotation: 0, + mirror: None, + full_frame: false, + }]; let clock = ScreenClock::default(); let at = |t: f32| { - (camera_fullscreen_progress_at(&r, t, &clock), camera_fullscreen_shape_at(&r, t, &clock)) + (camera_fullscreen_progress_at(&r, t, &clock, &[]), camera_fullscreen_shape_at(&r, t, &clock, &[])) }; // Premier tiers de la montée : la caméra a fait l'essentiel du chemin, coins intacts. let (progress, shape) = at(10.0 + TRANSITION_WINDOW_S / 3.0); @@ -2178,6 +2491,32 @@ mod arrow_tests { let (_, half) = arrow_local_geometry("right", -5.0, [100.0, 100.0]); assert_eq!(half, 0.0); } + + /// L'éditeur encadre chaque flèche par l'enveloppe de ses traits (`ARROW_EXTENTS`, + /// `src/lib/ai-edition/annotations/arrowBounds.ts`) : elle doit être celle que dessine ce + /// module, sinon le gimbal ne tombe plus sur la flèche. + #[test] + fn the_editor_frames_each_arrow_by_its_strokes() { + let ts = include_str!("../../../src/lib/ai-edition/annotations/arrowBounds.ts"); + let table = &ts[ts.find("ARROW_EXTENTS").expect("la table")..]; + let table = &table[..table.find("};").expect("sa fin")]; + for dir in ["up", "down", "left", "right", "up-right", "up-left", "down-right", "down-left"] { + let key = if dir.contains('-') { format!("\"{dir}\":") } else { format!("\t{dir}:") }; + let row = &table[table.find(&key).unwrap_or_else(|| panic!("{dir} absente"))..]; + let row = &row[row.find('[').expect("[") + 1..row.find(']').expect("]")]; + let got: Vec = row.split(',').map(|v| v.trim().parse().expect("nombre")).collect(); + let segs = arrow_segments_viewbox(dir); + let xs = segs.iter().flat_map(|s| [s[0], s[2]]); + let ys = segs.iter().flat_map(|s| [s[1], s[3]]); + let want = [ + xs.clone().fold(f32::MAX, f32::min), + ys.clone().fold(f32::MAX, f32::min), + xs.fold(f32::MIN, f32::max), + ys.fold(f32::MIN, f32::max), + ]; + assert_eq!(got, want, "{dir}"); + } + } } #[cfg(test)] @@ -3125,6 +3464,16 @@ mod exporter_frame_totals { 60, "région débordant la fenêtre gardée" ); + let issue_876_fps = 60.0; + let first_clip = frames( + 0.0, + 44.81830642526596, + &[region(44.818, 44.818, 1.5)], + issue_876_fps, + ); + let second_clip = frames(0.0, 47.833333, &[region(0.0, 1.87, 1.5)], issue_876_fps); + assert_eq!(first_clip, 2690, "une région vide ne double pas le premier clip"); + assert_eq!(first_clip + second_clip, 5523, "total corrigé du repro #876"); assert_eq!( frames( 0.0, diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs index de24278fd..98a5957c4 100644 --- a/crates/compositor/src/scene.rs +++ b/crates/compositor/src/scene.rs @@ -19,6 +19,67 @@ pub struct SceneClip { /// Une source sans piste audio décodable garde sa durée via du silence natif. #[serde(default)] pub has_audio: bool, + /// Cameras 2-4: index k-1 = camera k. An empty `path` means "no camera in this slot". + #[serde(default)] + pub additional_cameras: Vec, +} + +/// An additional camera (2-4) of a clip. = `CompositorClipCamera` (TS). +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneClipCamera { + pub path: String, + /// Camera source time = screen source time - this. + pub offset_sec: f64, +} + +/// Settings of one camera (index 0 = camera 1). With a `homography` the camera's rotation, +/// mirror and crop are not sent (the corner order already defines the picture). +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneCamera { + pub index: usize, + /// 0 or 180. + #[serde(default)] + pub rotation: u16, + #[serde(default)] + pub mirror: Option, + #[serde(default)] + pub crop: Option, + /// Row-major 3x3, target uv -> camera uv. + #[serde(default)] + pub homography: Option<[f32; 9]>, + /// Width/height of the corrected picture. + #[serde(default)] + pub aspect: Option, +} + +/// One camera placed by a layout region, in draw order. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneCameraLayer { + pub camera: usize, + pub rect: SceneRect, + #[serde(default)] + pub radius_frac: f32, + /// Same vocabulary as `SceneLayout::webcam_shape` (`webcam_shape_code`). + #[serde(default)] + pub shape: String, + /// Covers the screen: drawn without a shadow. + #[serde(default)] + pub fills_frame: bool, +} + +/// A camera layout (anything but a plain Full Camera) on a span of a clip's source time. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneCameraLayoutRegion { + #[serde(default)] + pub clip_index: Option, + pub start_sec: f64, + pub end_sec: f64, + #[serde(default)] + pub layers: Vec, } #[derive(Debug, Clone, Copy, Deserialize)] @@ -291,7 +352,7 @@ impl SceneFrame { } /// Fond derrière l'écran (parsé depuis `settings.wallpaper`). -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(tag = "kind", rename_all = "lowercase")] pub enum SceneBackground { Color { color: String }, @@ -528,7 +589,8 @@ pub struct SceneSpeedRegion { /// Une zone "Full Camera" de la timeline (temps en secondes) : la caméra PREND tout le cadre /// pendant cette fenêtre (plein écran net — ni marge, ni arrondi, ni masque, ni fond derrière). -/// Pas de champs au-delà des bornes temporelles (miroir de `CameraFullscreenRegion`, TS). +/// The orientation fields are resolved by the app (`sceneDescription.ts`): a desk shot +/// arrives as `rotation: 180`, `mirror: false`, `fullFrame: true`. #[derive(Debug, Clone, Copy, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SceneCameraFullscreenRegion { @@ -537,6 +599,15 @@ pub struct SceneCameraFullscreenRegion { pub clip_index: Option, pub start_sec: f64, pub end_sec: f64, + /// 0 or 180. Anything else is treated as 0 by `frame_geometry::webcam_orientation`. + #[serde(default)] + pub rotation: u16, + /// Mirror inside this region; `None` = the layout's `webcam_mirror`. + #[serde(default)] + pub mirror: Option, + /// Ignore `layout.webcam_crop` inside this region. + #[serde(default)] + pub full_frame: bool, } /// Rendu du curseur. @@ -730,6 +801,12 @@ pub struct Scene { /// `#[serde(default)]` : champ ajouté après coup, absent des JSON de test existants. #[serde(default)] pub camera_fullscreen_regions: Vec, + /// Settings of the cameras that have any (see `Scene::camera`). + #[serde(default)] + pub cameras: Vec, + /// Layouts that put several cameras on screen. Absent = none. + #[serde(default)] + pub camera_layout_regions: Vec, pub cursor: SceneCursor, /// Global audio finishing. Default keeps old scene payloads bit-for-bit compatible. #[serde(default)] @@ -751,6 +828,11 @@ pub struct Scene { } impl Scene { + /// The settings of camera `index` (0 = camera 1), if the scene carries any. + pub fn camera(&self, index: usize) -> Option<&SceneCamera> { + self.cameras.iter().find(|c| c.index == index) + } + /// Parse le JSON produit par `buildSceneDescription` (TS). pub fn from_json(json: &str) -> anyhow::Result { Ok(serde_json::from_str(json)?) @@ -788,6 +870,9 @@ impl Scene { scene.camera_fullscreen_regions.retain(|region| { belongs(region.clip_index, region.start_sec, region.end_sec) }); + scene.camera_layout_regions.retain(|region| { + belongs(region.clip_index, region.start_sec, region.end_sec) + }); scene.annotations.retain(|annotation| { belongs(annotation.clip_index, annotation.start_sec, annotation.end_sec) }); @@ -943,6 +1028,83 @@ mod tests { } } + /// A scene JSON with one clip; the arguments are spliced in as extra clip / top-level fields. + fn scene_json_with(clip_extra: &str, top_extra: &str) -> String { + format!( + r##"{{"clips":[{{"screenPath":"/s.mp4","webcamPath":"/w.mp4","sourceStartSec":0,"sourceEndSec":10,"webcamOffsetSec":0{clip_extra}}}],"layout":{{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}},"background":{{"kind":"color","color":"#000000"}},"zoomRegions":[],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":null}}{top_extra}}}"## + ) + } + + #[test] + fn parses_extra_cameras_settings_and_layout_regions() { + let json = scene_json_with( + r#","additionalCameras":[{"path":"/w-2.mp4","offsetSec":0.12},{"path":"","offsetSec":0}]"#, + r#","cameras":[{"index":1,"rotation":180,"mirror":true,"crop":{"x":0,"y":0.1,"width":0.5,"height":0.8}},{"index":2,"homography":[1,0,0,0,1,0,0,0,1],"aspect":1.5}], + "cameraLayoutRegions":[{"clipIndex":0,"startSec":1,"endSec":4,"underTrim":true,"layers":[ + {"camera":1,"rect":{"x":0,"y":0,"width":1,"height":1},"radiusFrac":0,"shape":"rectangle","fillsFrame":true}, + {"camera":0,"rect":{"x":0.7,"y":0.7,"width":0.22,"height":0.2},"radiusFrac":0.12,"shape":"rounded","fillsFrame":false}]}]"#, + ); + let s = Scene::from_json(&json).expect("parse"); + assert_eq!(s.clips[0].additional_cameras.len(), 2); + assert_eq!(s.clips[0].additional_cameras[0].path, "/w-2.mp4"); + assert!((s.clips[0].additional_cameras[0].offset_sec - 0.12).abs() < 1e-9); + assert_eq!(s.clips[0].additional_cameras[1].path, ""); + assert!(s.camera(0).is_none()); + let c1 = s.camera(1).expect("camera 1"); + assert_eq!(c1.rotation, 180); + assert_eq!(c1.mirror, Some(true)); + assert_eq!(c1.crop.expect("crop").height, 0.8); + assert!(c1.homography.is_none() && c1.aspect.is_none()); + let c2 = s.camera(2).expect("camera 2"); + assert_eq!(c2.rotation, 0); + assert_eq!(c2.homography, Some([1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0])); + assert_eq!(c2.aspect, Some(1.5)); + assert_eq!(s.camera_layout_regions.len(), 1); + let r = &s.camera_layout_regions[0]; + assert_eq!((r.clip_index, r.start_sec, r.end_sec), (Some(0), 1.0, 4.0)); + assert_eq!(r.layers.len(), 2); + assert_eq!(r.layers[0].camera, 1); + assert!(r.layers[0].fills_frame); + assert_eq!(r.layers[1].shape, "rounded"); + assert!((r.layers[1].radius_frac - 0.12).abs() < 1e-6); + assert!((r.layers[1].rect.width - 0.22).abs() < 1e-6); + } + + #[test] + fn old_scene_json_parses_without_the_new_fields() { + let s = Scene::from_json(&scene_json_with("", "")).expect("parse"); + assert!(s.clips[0].additional_cameras.is_empty()); + assert!(s.cameras.is_empty()); + assert!(s.camera_layout_regions.is_empty()); + assert!(s.camera(0).is_none()); + } + + #[test] + fn for_clip_window_keeps_only_this_clips_layout_regions() { + let layers = r#""layers":[{"camera":0,"rect":{"x":0,"y":0,"width":1,"height":1},"radiusFrac":0,"shape":"rectangle","fillsFrame":true}]"#; + let json = scene_json_with( + "", + &format!( + r#","cameraLayoutRegions":[ + {{"clipIndex":0,"startSec":1,"endSec":2,{layers}}}, + {{"clipIndex":1,"startSec":1,"endSec":2,{layers}}}, + {{"startSec":8,"endSec":9,{layers}}}, + {{"startSec":20,"endSec":30,{layers}}}]"# + ), + ); + let s = Scene::from_json(&json).expect("parse"); + let w = s.for_clip_window(0, 0.0, 10.0); + // Clip 0's own region and the unaddressed one that overlaps the window; not clip 1's, + // not the unaddressed one outside the window. + let kept: Vec<(Option, f64)> = w + .camera_layout_regions + .iter() + .map(|r| (r.clip_index, r.start_sec)) + .collect(); + assert_eq!(kept, vec![(Some(0), 1.0), (None, 8.0)]); + assert_eq!(s.for_clip_window(1, 0.0, 10.0).camera_layout_regions.len(), 2); + } + #[test] fn parses_webcam_rect_payload() { // webcamRect est une fraction 0..1 du cadre de sortie ; sa présence doit désactiver diff --git a/crates/compositor/src/sculpt.rs b/crates/compositor/src/sculpt.rs index de9c4e65a..89fa9a277 100644 --- a/crates/compositor/src/sculpt.rs +++ b/crates/compositor/src/sculpt.rs @@ -1,17 +1,18 @@ -//! Curseurs sculptés : la flèche et la main des cinq thèmes d'origine, modelées en volumes dans les -//! shaders (mode 15, `sculpt_proto`) au lieu d'extruder leur PNG — capsules et unions lissées, -//! extrusions arrondies, voxels, polyèdres taillés. Ce module en tient ce que la géométrie doit -//! savoir côté CPU : l'identifiant que lit le shader, et la boîte du modèle (`SpriteShape`) qui pose -//! le hotspot, règle la garde au sol et borne la boîte de dessin. +//! Curseurs sculptés : la flèche et la main des thèmes d'origine, modelées en volumes dans les +//! shaders (mode 15, `sculpt_proto`) au lieu d'extruder leur PNG — voxels (Pixel Candy), pièces +//! cerclées du trait de leur dessin (Studio Ink, Pop Coral, Star Sprout). Prism Glow est à part : +//! un MAILLAGE tracé sur son dessin à facettes, un cristal lancé de rayons boîte par boîte +//! dans son serti marine (`prism_mesh`). Ce module en tient ce que la géométrie doit savoir côté +//! CPU : l'identifiant que lit le shader, et la boîte du modèle (`SpriteShape`) qui pose le +//! hotspot, règle la garde au sol et borne la boîte de dessin. //! //! Les formes sont écrites dans le repère du PROTOTYPE où elles ont été dessinées : hauteur du //! curseur 1, x à droite, y VERS LE HAUT, z vers la caméra, l'écran en z = 0. Les constantes //! ci-dessous sont le miroir des `SCULPT_*` des trois shaders. //! -//! Les voxels de la flèche de Pixel Candy (`SCULPT_ARROWPIX`, `SCULPT_ARROWBACK`) sont son polygone -//! échantillonné : une cellule de 1/16 porte un voxel quand son centre est à plus de 0,01 dedans, un -//! bloc de la couche violette quand il est à moins de 0,9 cellule. Changer le polygone, c'est -//! régénérer ces deux tables. +//! Les grilles de voxels de Pixel Candy sont des tables que +//! `scripts/generate-pixel-candy-voxels.mjs` écrit dans les trois shaders : on change la grille +//! dans le script, on le relance, et on ajuste ici la boîte si la forme a bougé. use crate::frame_geometry::SpriteShape; @@ -24,29 +25,48 @@ const ZREF_ARROW: f32 = 0.2; const ZREF_HAND: f32 = 0.185; /// Le dessous de tous les modèles, dans le prototype. const Z_LOW: f32 = 0.05; -/// Le plus haut : la table de la flèche taillée, l'étoile de Star Sprout sur la manchette. -const Z_HIGH_ARROW: f32 = 0.29; +/// Le côté d'un cube de Pixel Candy : seize lignes font un curseur haut de 1. +const VOX: f32 = 0.0625; +/// Le plus haut : les yeux de l'étoile de Star Sprout, sur la flèche comme sur la manchette. +const Z_HIGH_ARROW: f32 = 0.31; const Z_HIGH_HAND: f32 = 0.4; -/// Boîtes des deux formes dans le prototype (x0, x1, haut), qui contiennent leurs cinq variantes : -/// couche violette des voxels, étoile, cristal compris. Leur hauteur est celle du sprite, 1 / SCULPT_SCALE : -/// le mode 15 tient le plus grand côté de la boîte pour 1 (`sprite_size`). -const BOX_ARROW: [f32; 3] = [-0.1, 0.75, 0.09]; -const BOX_HAND: [f32; 3] = [-0.27, 0.73, 0.07]; +/// Boîtes des modèles dans le prototype (x0, x1, haut). Leur hauteur est celle du sprite, +/// 1 / SCULPT_SCALE : le mode 15 tient le plus grand côté de la boîte pour 1 (`sprite_size`). +/// Celles des cubes de Pixel Candy ; puis des thèmes cerclés, que leur trait élargit et que +/// débordent l'étoile et les feuilles (Star Sprout), les tirets du clic et les calques (Pop +/// Coral), le pouce. +const BOX_PIXEL_ARROW: [f32; 3] = [-0.03, 0.72, 0.03]; +const BOX_PIXEL_HAND: [f32; 3] = [-0.345, 0.72, 0.03]; +const BOX_INK_ARROW: [f32; 3] = [-0.08, 0.6, 0.03]; +const BOX_INK_HAND: [f32; 3] = [-0.36, 0.6, 0.03]; +const BOX_CORAL_ARROW: [f32; 3] = [-0.35, 0.6, 0.06]; +const BOX_CORAL_HAND: [f32; 3] = [-0.38, 0.62, 0.04]; +const BOX_SPROUT_ARROW: [f32; 3] = [-0.08, 0.87, 0.03]; +const BOX_SPROUT_HAND: [f32; 3] = [-0.33, 0.56, 0.03]; /// Dans l'ordre des identifiants du shader. const THEMES: [&str; 5] = ["studio-ink", "prism-glow", "pop-coral", "pixel-candy", "star-sprout"]; -/// Le haut de la silhouette (y du prototype) : la couche violette des voxels dépasse la pointe -/// d'une cellule, la pointe arrondie de la flèche et le sommet de sa table taillée dépassent le -/// hotspot de 0,03, le bout de l'index y est. -fn silhouette_top(theme: usize, arrow: bool) -> f32 { +/// La boîte du modèle (x0, x1, haut) dans le prototype. +fn model_box(theme: usize, arrow: bool) -> [f32; 3] { match (THEMES[theme], arrow) { - ("pixel-candy", _) => 0.0625, - (_, true) => 0.03, - (_, false) => 0.0, + ("studio-ink", true) => BOX_INK_ARROW, + ("studio-ink", false) => BOX_INK_HAND, + ("pop-coral", true) => BOX_CORAL_ARROW, + ("pop-coral", false) => BOX_CORAL_HAND, + ("pixel-candy", true) => BOX_PIXEL_ARROW, + ("pixel-candy", false) => BOX_PIXEL_HAND, + ("star-sprout", true) => BOX_SPROUT_ARROW, + ("star-sprout", false) => BOX_SPROUT_HAND, + (other, _) => unreachable!("thème sculpté inconnu : {other}"), } } +/// Le curseur sculpté que nomme la scène est de verre (Prism Glow) : il réfracte ce qui est dessous. +pub fn refracts(name: &str) -> bool { + sculpted_shape(name).is_some_and(|s| THEMES[((s.sculpt - 1) / 2) as usize] == "prism-glow") +} + /// Le curseur sculpté que nomme la scène (`"/<état>"`, cf. `resolveCursorSprites`), sous /// la forme que le mode 15 attend. `None` pour un nom inconnu : l'appelant extrude le sprite. pub fn sculpted_shape(name: &str) -> Option { @@ -57,7 +77,20 @@ pub fn sculpted_shape(name: &str) -> Option { "pointer" => false, _ => return None, }; - let [x0, x1, y1] = if arrow { BOX_ARROW } else { BOX_HAND }; + let sculpt = 1 + 2 * theme as u32 + u32::from(!arrow); + if THEMES[theme] == "prism-glow" { + // Le maillage est déjà dans le repère du modèle : sa boîte, son dessus et son dessous. + let m = &crate::prism_mesh::MODELS[usize::from(!arrow)]; + return Some(SpriteShape { + size: m.size, + hotspot: m.hotspot, + top: m.top, + max_height: m.height, + thick: m.thick, + sculpt, + }); + } + let [x0, x1, y1] = model_box(theme, arrow); let (zref, z_high) = if arrow { (ZREF_ARROW, Z_HIGH_ARROW) } else { (ZREF_HAND, Z_HIGH_HAND) }; let s = SCULPT_SCALE; // Repère du modèle : y vers le bas, le coin haut-gauche de la boîte est donc (x0, y1). @@ -65,10 +98,10 @@ pub fn sculpted_shape(name: &str) -> Option { Some(SpriteShape { size, hotspot: [-x0 * s / size[0], y1 * s], - top: (y1 - silhouette_top(theme, arrow)) * s, + top: y1 * s, max_height: (z_high - zref) * s, thick: (zref - Z_LOW) * s, - sculpt: 1 + 2 * theme as u32 + u32::from(!arrow), + sculpt, }) } @@ -99,20 +132,21 @@ mod tests { #[test] fn other_states_and_themes_keep_the_extruded_sprite() { - for name in ["default/arrow", "pop-coral/text", "pop-coral", "", "pop-coral/arrow/x"] { + for name in ["default/arrow", "pop-coral/text", "pop-coral", "", "pop-coral/arrow/x", "prism-glow/text"] { assert!(sculpted_shape(name).is_none(), "{name}"); } } /// Le hotspot est dans la boîte, sous le haut de la silhouette ; le modèle a une épaisseur et /// une hauteur ; la boîte a la hauteur du sprite, son plus grand côté, comme le veut le mode 15. + /// Le serti de Prism Glow est le plus mince (0,043) : c'est son épaisseur dans le dessin. #[test] fn the_box_holds_the_hotspot_and_the_silhouette_top() { for name in NAMES { let s = sculpted_shape(name).unwrap(); assert!((0.0..1.0).contains(&s.hotspot[0]) && (0.0..1.0).contains(&s.hotspot[1]), "{name}"); assert!(s.top <= s.hotspot[1] + 1e-6, "{name} : haut de silhouette sous le hotspot"); - assert!(s.thick > 0.05 && s.max_height > 0.0, "{name}"); + assert!(s.thick > 0.04 && s.max_height > 0.0, "{name}"); assert!(s.size[0] < 1.0 && s.size[1] == 1.0, "{name} : le plus grand côté n'est pas 1"); } } @@ -131,6 +165,8 @@ mod tests { ("SCULPT_ZREF_ARROW", ZREF_ARROW), ("SCULPT_ZREF_HAND", ZREF_HAND), ("SCULPT_HOVER", Z_LOW), + ("SCULPT_VOX", VOX), + ("PRISM_BEVEL", crate::prism_mesh::BEVEL), ] { let line = src .lines() @@ -145,4 +181,158 @@ mod tests { } } } + + /// Les nombres de la table `name` d'un shader, du `=` à la fin de la liste, constructeurs de + /// vecteurs retirés : la même table s'écrit alors pareil dans les trois langages. + fn table(src: &str, name: &str) -> Vec { + // La déclaration : `NAME =` en WGSL, `NAME[taille]` en HLSL et en Metal (pas un `NAME[i]` du code). + let decl = format!("{name}["); + let sized = |&i: &usize| src[i + decl.len()..].starts_with(|c: char| c.is_ascii_digit()); + let at = src.find(&format!("{name} =")).or_else(|| src.match_indices(&decl).map(|(i, _)| i).find(sized)); + let rest = &src[at.expect(name)..]; + let rest = &rest[rest.find('=').unwrap()..]; + let open = rest.find(['(', '{']).unwrap() + 1; + let close = [rest.find(");"), rest.find("};")].into_iter().flatten().min().unwrap(); + let mut list = rest[open..close].to_string(); + for ty in ["vec2", "vec3", "vec4", "vec4", "float2", "float3", "float4", "int4"] { + list = list.replace(ty, ""); + } + list.split(|c: char| !(c.is_ascii_digit() || c == '.' || c == '-')) + .filter(|t| !t.is_empty()) + .map(|t| t.parse().unwrap_or_else(|_| panic!("{name} : nombre illisible {t}"))) + .collect() + } + + /// Les grilles de Pixel Candy, que `scripts/generate-pixel-candy-voxels.mjs` écrit dans les + /// trois shaders, y sont les mêmes ; chaque pixel coloré est plein, et d'une seule couleur. + #[test] + fn the_voxel_tables_match_in_the_three_shaders() { + let hlsl = include_str!("shaders.hlsl"); + let metal = include_str!("shaders.metal"); + let wgsl = include_str!("vk_shaders/layer.wgsl"); + let names = ["PIX_BODY", "PIX_LINE", "PIX_HI", "PIX_SHADE", "PIX_RECT_N", "PIX_GRID", "PIX_ORIGIN", "PIX_BOX", "PIX_RECT"]; + for name in names { + let reference = table(wgsl, name); + assert!(!reference.is_empty(), "{name} vide"); + assert_eq!(table(hlsl, name), reference, "{name} : HLSL et WGSL diffèrent"); + assert_eq!(table(metal, name), reference, "{name} : Metal et WGSL diffèrent"); + } + let body = table(wgsl, "PIX_BODY"); + let classes = ["PIX_LINE", "PIX_HI", "PIX_SHADE"].map(|name| table(wgsl, name)); + for (i, &face) in body.iter().enumerate() { + let mut seen = 0u32; + for class in &classes { + let m = class[i] as u32; + assert_eq!(m & !(face as u32), 0, "couleur hors de la forme, ligne {i}"); + assert_eq!(m & seen, 0, "deux couleurs sur un pixel, ligne {i}"); + seen |= m; + } + } + } + + /// Seul Prism Glow est de verre, flèche et main. + #[test] + fn only_prism_glow_refracts() { + for name in NAMES { + assert_eq!(refracts(name), name.starts_with("prism-glow/"), "{name}"); + } + assert!(!refracts("prism-glow/text")); + } + + /// Le maillage de Prism Glow tient dans sa boîte (le shader n'y lance de rayons que là), son + /// dessous est à l'épaisseur annoncée (la garde au sol en dépend) et son sommet à la hauteur. + #[test] + fn the_prism_boxes_hold_their_mesh() { + use crate::prism_mesh::{MODELS, POLYS, TRIS}; + for (k, name) in ["prism-glow/arrow", "prism-glow/pointer"].iter().enumerate() { + let s = sculpted_shape(name).unwrap(); + let m = &MODELS[k]; + let [x0, y0] = [-s.hotspot[0] * s.size[0], -s.hotspot[1] * s.size[1]]; + let inside = |x: f32, y: f32| x >= x0 && x <= x0 + s.size[0] && y >= y0 && y <= y0 + s.size[1]; + let (mut zlo, mut zhi) = (f32::MAX, f32::MIN); + for t in TRIS[4 * m.tri_start..4 * (m.tri_start + m.tri_count)].chunks(4) { + for e in [[0.0; 3], [t[1][0], t[1][1], t[1][2]], [t[2][0], t[2][1], t[2][2]]] { + let [x, y, z] = [t[0][0] + e[0], t[0][1] + e[1], t[0][2] + e[2]]; + assert!(inside(x, y), "{name} : sommet ({x}, {y}) hors de la boîte"); + zlo = zlo.min(z); + zhi = zhi.max(z); + } + } + let sil = &POLYS[m.sil_start..m.sil_start + m.sil_count]; + let out = &POLYS[m.out_start..m.out_start + m.out_count]; + for p in sil.iter().chain(out) { + assert!(inside(p[0], p[1]), "{name} : contour ({}, {}) hors de la boîte", p[0], p[1]); + } + assert!((zlo + s.thick).abs() < 1e-5, "{name} : dessous {zlo}, épaisseur {}", s.thick); + assert!((zhi - s.max_height).abs() < 1e-5, "{name} : sommet {zhi}, hauteur {}", s.max_height); + } + } + + /// Les trois shaders portent le maillage de `prism_mesh.rs` tel quel : triangles, boîtes, + /// polygones, et les débuts et longueurs de chaque modèle dans ces tables. Le WGSL lit + /// triangles, polygones et boîtes dans un uniform rempli de `prism_mesh.rs` + /// (`compositor_linux.rs`) : il n'en déclare que la taille. + #[test] + fn the_shaders_carry_the_prism_mesh() { + use crate::prism_mesh::{PrismModel, BOXES, MODELS, POLYS, TRIS}; + let per_model = |f: fn(&PrismModel) -> usize| MODELS.iter().map(|m| f(m) as f32).collect::>(); + let want: [(&str, Vec); 11] = [ + ("PRISM_TRIS", TRIS.iter().flatten().copied().collect()), + ("PRISM_POLY", POLYS.iter().flatten().copied().collect()), + ("PRISM_BOXES", BOXES.iter().flatten().copied().collect()), + ("PRISM_BOX_START", per_model(|m| m.box_start)), + ("PRISM_BOX_COUNT", per_model(|m| m.box_count)), + ("PRISM_TRI_START", per_model(|m| m.tri_start)), + ("PRISM_TRI_COUNT", per_model(|m| m.tri_count)), + ("PRISM_SIL_START", per_model(|m| m.sil_start)), + ("PRISM_SIL_COUNT", per_model(|m| m.sil_count)), + ("PRISM_OUT_START", per_model(|m| m.out_start)), + ("PRISM_OUT_COUNT", per_model(|m| m.out_count)), + ]; + let wgsl = include_str!("vk_shaders/layer.wgsl"); + for field in [ + format!("tris: array, {}>", TRIS.len()), + format!("poly: array, {}>", POLYS.len()), + format!("boxes: array, {}>", BOXES.len()), + ] { + assert!(wgsl.contains(&field), "vk_shaders/layer.wgsl : `{field}` attendu dans PrismMesh"); + } + for (file, src, tables) in [ + ("shaders.hlsl", include_str!("shaders.hlsl"), &want[..]), + ("shaders.metal", include_str!("shaders.metal"), &want[..]), + ("vk_shaders/layer.wgsl", wgsl, &want[3..]), + ] { + for (name, values) in tables { + let got = table(src, name); + assert_eq!(got.len(), values.len(), "{file} : {name}, nombre de valeurs"); + for (i, (g, w)) in got.iter().zip(values).enumerate() { + assert!((g - w).abs() <= 1e-6 + 1e-6 * w.abs(), "{file} : {name}[{i}] = {g} au lieu de {w}"); + } + } + } + } + + /// Chaque triangle du cristal est dans une boîte et une seule, qui le contient : le shader ne + /// teste les triangles d'une boîte que si le rayon la touche, un triangle hors de sa boîte + /// disparaîtrait par endroits. + #[test] + fn each_crystal_triangle_sits_in_its_box() { + use crate::prism_mesh::{BOXES, MODELS, TRIS}; + for (k, m) in MODELS.iter().enumerate() { + let mut next = m.tri_start; + for b in m.box_start..m.box_start + m.box_count { + let (lo, hi) = (BOXES[2 * b], BOXES[2 * b + 1]); + let (first, count) = (lo[3] as usize, hi[3] as usize); + assert_eq!(first, next, "modèle {k}, boîte {b} : triangles non contigus"); + next += count; + for t in TRIS[4 * first..4 * (first + count)].chunks(4) { + for e in [[0.0; 3], [t[1][0], t[1][1], t[1][2]], [t[2][0], t[2][1], t[2][2]]] { + let v = [t[0][0] + e[0], t[0][1] + e[1], t[0][2] + e[2]]; + assert!((0..3).all(|a| v[a] >= lo[a] && v[a] <= hi[a]), "modèle {k}, boîte {b} : sommet {v:?} dehors"); + } + } + } + assert_eq!(next, m.tri_start + m.tri_count, "modèle {k} : triangles hors des boîtes"); + } + } } diff --git a/crates/compositor/src/shaders.hlsl b/crates/compositor/src/shaders.hlsl index 30fbf1bd5..e5777929e 100644 --- a/crates/compositor/src/shaders.hlsl +++ b/crates/compositor/src/shaders.hlsl @@ -1,5 +1,12 @@ // Compositeur — un draw par calque (quad). NV12->RGB maison (E1), coins arrondis SDF (E2). // Tout écrit depuis les maths (§7), rien repris de l'ancien paradigme. +// +// `ps_main` est compilé DEUX fois (`build.rs`) : sans `LAYER_MODELS` pour tous les calques, et +// avec, en `ps_main_models`, pour les seuls modèles 3D (modes 15 à 17, `LayerCB::needs_models`). +// Un pixel shader alloue à chaque draw les registres de sa branche la plus lourde, et ce sont +// ces modèles ray-tracés : le même shader les portait pour le fond, l'ombre et la vidéo, qui +// couvrent toute la sortie. Mesuré sur la version WGSL (RADV) : 168 VGPR, 6 vagues par SIMD au +// lieu de 32, et un export Linux deux fois plus lent ; sans eux, 48. cbuffer Layer : register(b0) { @@ -16,6 +23,9 @@ cbuffer Layer : register(b0) float4 trail_a; // mode 8 : coins TL, TR du plan à la frame précédente (px locaux, comme fx) ; mode 18 incliné : en fractions de sortie float4 trail_b; // mode 8 : coins BR, BL du plan à la frame précédente (comme src_prev) ; mode 18 incliné : en fractions de sortie float4 trail_mb; // mode 8 : x = taps, y = force du flou de mouvement (ceux du mode 0) ; mode 18 incliné : le `mb` du mode 8 (profondeur de champ), et `color.xy` sa lampe ; 0 ailleurs + float4 cover; // x = desk-view cover 0..1, y = blur radius (quad px), z = dim, w unused + float4 persp[3]; // mode 0: rows of the homography quad point (0..1 in dst) -> camera uv (0..1 of the valid frame), xyz; read when layer_fx.y = 1 + float4 layer_fx; // x = transparency 0..1 (0 = opaque), every mode; y = 1 when persp applies; z, w unused }; // Mode 15 (curseur modélisé) : le détail des emplacements est dans `frame_geometry.rs`, en tête // de la section « Curseur modélisé » (`cursor_model_cb`). Mode 17 (appareil modelé) : en tête de @@ -81,6 +91,7 @@ float3 sample_yuv(float2 uv) return yuv709_limited(y, cbcr); } +// La même, au niveau 0 : lisible dans une boucle ou une branche (le cristal du mode 15). // SDF segment à bouts ronds — la primitive des flèches d'annotation, dont les tracés SVG sont // trois segments `stroke-linecap="round"` (cf. ArrowSvgs.tsx). float sd_segment(float2 p, float2 a, float2 b) @@ -105,9 +116,13 @@ float sd_segment(float2 p, float2 a, float2 b) float sd_round_rect(float2 p, float2 halfsz, float r) { float hmin = min(halfsz.x, halfsz.y); - if (r <= 0.0 || hmin <= 0.0) + float2 q0 = abs(p) - halfsz; + // Hors des coins, la distance est celle du rectangle vif quel que soit l'exposant : un seul + // axe déborde de l'étendue E, qui s'y ajoute puis s'en retranche. E ne dépasse pas 1,42 r + // (n = 3) : sous 1,5 r du bord sur un axe, on rend la boîte sans calculer `n` ni E, deux + // transcendantes de moins sur presque tous les pixels d'un calque arrondi. + if (r <= 0.0 || hmin <= 0.0 || min(q0.x, q0.y) <= -min(1.5 * r, hmin)) { - float2 q0 = abs(p) - halfsz; return length(max(q0, 0.0)) + min(max(q0.x, q0.y), 0.0); } float n = 3.0 - smoothstep(0.5, 1.0, r / hmin); @@ -463,6 +478,19 @@ float band_cov(float x, float half_w) return saturate(half_w + 0.5 - abs(x)); } +// Camera homography (`persp`, read when `layer_fx.y` = 1): the camera point (0..1 of its valid +// frame) seen at `local` (0..1 in the layer's dst quad), and in z whether there is one -- 0 when +// the point is behind the projection (q.z <= 0) or outside the camera frame, where the layer is +// transparent. +float3 persp_camera(float2 local) +{ + float3 p = float3(local, 1.0); + float3 q = float3(dot(persp[0].xyz, p), dot(persp[1].xyz, p), dot(persp[2].xyz, p)); + float2 cam = q.xy / max(q.z, 1e-6); + bool inside = q.z > 0.0 && all(cam >= 0.0) && all(cam <= 1.0); + return float3(cam, inside ? 1.0 : 0.0); +} + // Fond flouté pour le mode "blur" de la webcam. // Disque de Vogel (spirale à angle d'or) à 21 échantillons avec pondération gaussienne et // rotation par pixel via Interleaved Gradient Noise (IGN) pour un bokeh photographique doux, isotrope et rapide. @@ -490,10 +518,15 @@ static const float3 VOGEL_TAPS[21] = { float3(-0.633036, -0.758588, 0.087119) }; -float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 local_px) +// Vogel-disc blur of the camera texture at a radius in quad pixels. `valid` is the part of the +// texture the picture fills (fx.xy): decoders allocate aligned textures (a 1080-line camera in a +// 1088-line texture), so each tap is clamped half a chroma texel inside it, never into padding. +float3 blur_webcam_radius(float2 uv, float max_r_px, float2 qpx, float2 local_px, float2 valid) { - float max_r_px = max(intensity, 0.0) * 22.0 + 1.5; float2 step = max_r_px / max(qpx, 1.0); + float cw, ch; + texUV.GetDimensions(cw, ch); + float2 hi = max(valid - 0.5 / max(float2(cw, ch), 1.0), 0.0); // Interleaved Gradient Noise pour rotation aléatoire par pixel float noise = frac(52.9829189 * frac(0.06711056 * local_px.x + 0.00583715 * local_px.y)); float angle = noise * 6.2831853; @@ -506,12 +539,17 @@ float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 local_px) float2 p = VOGEL_TAPS[k].xy; float w = VOGEL_TAPS[k].z; float2 rot_p = float2(p.x * c - p.y * s, p.x * s + p.y * c); - sum += sample_yuv(saturate(uv + rot_p * step)) * w; + sum += sample_yuv(clamp(uv + rot_p * step, 0.0, hi)) * w; total += w; } return sum / max(total, 1e-4); } +float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 local_px, float2 valid) +{ + return blur_webcam_radius(uv, max(intensity, 0.0) * 22.0 + 1.5, qpx, local_px, valid); +} + // ============ Curseur MODÉLISÉ (mode 15) ============ // L'état courant en objet 3D, lancé de rayons par pixel. Deux sortes d'objets : // - un curseur SCULPTÉ (`trail_a.x` > 0) : la flèche ou la main d'un des cinq thèmes d'origine, @@ -587,13 +625,12 @@ static const float MODEL_CONTACT_ALPHA = 0.5; // curseur 1, x à droite, y VERS LE HAUT, z vers la caméra, l'écran en z = 0 et le modèle posé à // SCULPT_HOVER au-dessus. `sculpt_point` y amène un point du repère du modèle. Identifiant : // 1 + 2 × thème + forme ; thèmes 0 Studio Ink, 1 Prism Glow, 2 Pop Coral, 3 Pixel Candy, -// 4 Star Sprout ; formes 0 flèche, 1 main. Id des matières : 1 corps, 2 liseré ou manchette, -// 3 étoile, 4 feuilles, 5 yeux, 6 face des voxels, 7 couche violette, 8 cristal. +// 4 Star Sprout ; formes 0 flèche, 1 main. Id des matières : 1 corps, 2 bande, calque ou +// manchette, 3 tirets, calque corail ou étoile, 4 feuilles, 5 trait des thèmes cerclés (et +// yeux), 6 cubes de Pixel Candy, 8 serti de Prism Glow, 9 son cristal (plus bas, `prism_*`). static const float SCULPT_SCALE = 0.85; static const float SCULPT_HOVER = 0.05; static const float SCULPT_VOX = 0.0625; -static const float SCULPT_AR_ROUND = 0.03; -static const float SCULPT_HAND_ZC = 0.185; // Hauteur, dans le prototype, du z = 0 du modèle : le dessus de la pointe de la flèche, l'axe du // bout de l'index. static const float SCULPT_ZREF_ARROW = 0.2; @@ -619,13 +656,6 @@ float2 s_opu(float2 a, float2 b) return a.x < b.x ? a : b; } -float s_capsule(float3 p, float3 a, float3 b, float r) -{ - float3 pa = p - a, ba = b - a; - float h = saturate(dot(pa, ba) / dot(ba, ba)); - return length(pa - ba * h) - r; -} - float s_round_box(float3 p, float3 b, float r) { float3 q = abs(p) - b + r; @@ -652,25 +682,91 @@ float2 s_rot(float2 v, float a) return float2(c * v.x - s * v.y, s * v.x + c * v.y); } -// La flèche : polygone à sept sommets, pointe à l'origine. -static const float2 SCULPT_ARROW[7] = { - float2(0.0, 0.0), float2(0.0, -0.86), float2(0.215, -0.665), float2(0.37, -1.0), - float2(0.53, -0.93), float2(0.38, -0.60), float2(0.64, -0.60) +float s_star5(float2 p, float r, float rf) +{ + const float2 k1 = float2(0.809016994375, -0.587785252292); + const float2 k2 = float2(-0.809016994375, -0.587785252292); + p.x = abs(p.x); + p -= 2.0 * max(dot(k1, p), 0.0) * k1; + p -= 2.0 * max(dot(k2, p), 0.0) * k2; + p.x = abs(p.x); + p.y -= r; + float2 ba = rf * float2(-k1.y, k1.x) - float2(0.0, 1.0); + float h = clamp(dot(p, ba) / dot(ba, ba), 0.0, r); + return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); +} + +// Lentille : deux disques de rayon r aux centres à ±d sur x, une feuille pointue à ses deux bouts +// (0, ±sqrt(r² - d²)). +float s_vesica(float2 p, float r, float d) +{ + p = abs(p); + float b = sqrt(r * r - d * d); + return (p.y - b) * d > p.x * b ? length(p - float2(0.0, b)) : length(p + float2(d, 0.0)) - r; +} + +// Les thèmes CERCLÉS (Studio Ink, Pop Coral, Star Sprout) suivent la planche 3D de référence +// (`design/cursors/3d-concept.png`) : le trait de leur dessin 2D n'est pas jeté, il devient la +// matière qui porte tout. Chaque pièce (la flèche, le gant, la manchette, les feuilles, l'étoile) +// est un plateau sous sa silhouette, un jonc sur le trait et un coussin de couleur bombé dedans. +// RIM_POLY : sept sommets par polygone (un sommet coupe un côté en deux quand il en faut six). +// D'abord la couleur de la flèche, commune aux trois thèmes (le trait l'élargit de 0,06, sa pointe +// arrondie touche le hotspot), puis les paumes de Star Sprout, de Studio Ink et de Pop Coral, +// arrondies de 0,05. +static const float2 RIM_POLY[28] = { + float2(0.0, -0.06), float2(0.0, -0.7712), float2(0.165, -0.6325), float2(0.313, -0.9318), + float2(0.4234, -0.8557), float2(0.2672, -0.5686), float2(0.456, -0.5393), + float2(-0.173, -0.5237), float2(-0.016, -0.75), float2(0.329, -0.75), float2(0.411, -0.592), + float2(0.411, -0.43), float2(0.19, -0.435), float2(-0.03, -0.44), + float2(-0.1825, -0.6215), float2(0.0037, -0.9195), float2(0.3462, -0.9195), float2(0.467, -0.6906), + float2(0.467, -0.49), float2(0.2, -0.49), float2(-0.03, -0.49), + float2(-0.1807, -0.5534), float2(0.0194, -0.872), float2(0.3485, -0.872), float2(0.47, -0.6795), + float2(0.47, -0.49), float2(0.2, -0.49), float2(-0.03, -0.49) }; -float s_arrow2(float2 p) +// Les trois gants, mesurés sur leur dessin (g : 0 Star Sprout, 1 Studio Ink, 2 Pop Coral), neuf +// lignes chacun : les segments de l'index, des trois doigts repliés et du pouce, ceux des trois +// fentes entre les doigts, puis le V entre le pouce et l'index (un point du bord du pouce et la +// normale qui entre dans le V). RIM_GLOVE_R, trois lignes par gant : les rayons des quatre doigts ; +// ceux du pouce et des trois fentes ; le bord de l'index que longe le V, le fond du V, le trait. +static const float4 RIM_GLOVE[27] = { + float4(0.0, -0.1077, 0.0, -0.6), float4(0.1493, -0.3279, 0.1493, -0.6), + float4(0.2863, -0.3654, 0.2863, -0.6), float4(0.4193, -0.4043, 0.4193, -0.6), + float4(-0.2, -0.478, -0.075, -0.625), float4(0.0788, -0.25, 0.0788, -0.3992), + float4(0.2188, -0.3, 0.2188, -0.4234), float4(0.3552, -0.33, 0.3552, -0.4534), + float4(-0.0845, -0.5386, 0.762, 0.648), + float4(0.0, -0.1186, 0.0, -0.65), float4(0.1614, -0.3838, 0.1614, -0.65), + float4(0.3177, -0.427, 0.3177, -0.65), float4(0.4645, -0.4848, 0.4645, -0.65), + float4(-0.215, -0.535, -0.1208, -0.6864), float4(0.0832, -0.3, 0.0832, -0.48), + float4(0.2414, -0.35, 0.2414, -0.482), float4(0.3959, -0.41, 0.3959, -0.533), + float4(-0.122, -0.5718, 0.867, 0.498), + float4(0.0, -0.122, 0.0, -0.65), float4(0.172, -0.387, 0.172, -0.65), + float4(0.3267, -0.427, 0.3267, -0.65), float4(0.4727, -0.4867, 0.4727, -0.65), + float4(-0.199, -0.483, -0.102, -0.645), float4(0.0927, -0.3, 0.0927, -0.476), + float4(0.2493, -0.35, 0.2493, -0.478), float4(0.402, -0.41, 0.402, -0.52), + float4(-0.0993, -0.5267, 0.858, 0.514) +}; +static const float4 RIM_GLOVE_R[9] = { + float4(0.0552, 0.047, 0.045, 0.0375), float4(0.052, 0.0235, 0.0225, 0.0225), float4(0.0552, -0.548, 0.0525, 0.0), + float4(0.064, 0.0585, 0.0585, 0.0515), float4(0.06, 0.0197, 0.0178, 0.0172), float4(0.0648, -0.6, 0.058, 0.0), + float4(0.066, 0.0553, 0.0553, 0.0507), float4(0.063, 0.024, 0.022, 0.02), float4(0.0647, -0.555, 0.056, 0.0) +}; + +// Distance signée au polygone de RIM_POLY qui commence en `base` : distance aux arêtes, signe par +// la parité des traversées d'une demi-droite. +float s_rim_poly(float2 p, int base) { - float d = dot(p - SCULPT_ARROW[0], p - SCULPT_ARROW[0]); + float d = dot(p - RIM_POLY[base], p - RIM_POLY[base]); float s = 1.0; - int j = 6; - [loop] for (int i = 0; i < 7; i++) + int j = base + 6; + [loop] for (int i = base; i < base + 7; i++) { - float2 e = SCULPT_ARROW[j] - SCULPT_ARROW[i]; - float2 w = p - SCULPT_ARROW[i]; + float2 e = RIM_POLY[j] - RIM_POLY[i]; + float2 w = p - RIM_POLY[i]; float2 b = w - e * saturate(dot(w, e) / dot(e, e)); d = min(d, dot(b, b)); - bool c0 = p.y >= SCULPT_ARROW[i].y; - bool c1 = p.y < SCULPT_ARROW[j].y; + bool c0 = p.y >= RIM_POLY[i].y; + bool c1 = p.y < RIM_POLY[j].y; bool c2 = e.x * w.y > e.y * w.x; if ((c0 && c1 && c2) || (!c0 && !c1 && !c2)) { @@ -681,311 +777,1262 @@ float s_arrow2(float2 p) return s * sqrt(d); } -float s_star5(float2 p, float r, float rf) +// Le gant `g` dans le plan, autour de sa paume `palm` : (silhouette, couleur). La silhouette unit +// la paume, l'index levé, trois doigts repliés et le pouce ; la couleur en est creusée des +// rainures du dessin, trois fentes entre les doigts et le V entre le pouce et l'index. +float2 s_rim_glove(float2 p, int g, float palm) { - const float2 k1 = float2(0.809016994375, -0.587785252292); - const float2 k2 = float2(-0.809016994375, -0.587785252292); - p.x = abs(p.x); - p -= 2.0 * max(dot(k1, p), 0.0) * k1; - p -= 2.0 * max(dot(k2, p), 0.0) * k2; - p.x = abs(p.x); - p.y -= r; - float2 ba = rf * float2(-k1.y, k1.x) - float2(0.0, 1.0); - float h = clamp(dot(p, ba) / dot(ba, ba), 0.0, r); - return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); + float4 r0 = RIM_GLOVE_R[3 * g]; + float4 r1 = RIM_GLOVE_R[3 * g + 1]; + float4 r2 = RIM_GLOVE_R[3 * g + 2]; + int i = 9 * g; + float f = min(sd_segment(p, RIM_GLOVE[i].xy, RIM_GLOVE[i].zw) - r0.x, + sd_segment(p, RIM_GLOVE[i + 1].xy, RIM_GLOVE[i + 1].zw) - r0.y); + f = min(f, sd_segment(p, RIM_GLOVE[i + 2].xy, RIM_GLOVE[i + 2].zw) - r0.z); + f = min(f, sd_segment(p, RIM_GLOVE[i + 3].xy, RIM_GLOVE[i + 3].zw) - r0.w); + f = min(f, sd_segment(p, RIM_GLOVE[i + 4].xy, RIM_GLOVE[i + 4].zw) - r1.x); + float grooves = min(sd_segment(p, RIM_GLOVE[i + 5].xy, RIM_GLOVE[i + 5].zw) - r1.y, + sd_segment(p, RIM_GLOVE[i + 6].xy, RIM_GLOVE[i + 6].zw) - r1.z); + grooves = min(grooves, sd_segment(p, RIM_GLOVE[i + 7].xy, RIM_GLOVE[i + 7].zw) - r1.w); + float sil = s_smin(palm, f, 0.03); + float4 v = RIM_GLOVE[i + 8]; + float wedge = max(max(-dot(p - v.xy, v.zw), p.x + r2.x), r2.y - p.y); + return float2(sil, max(sil, -min(grooves, wedge))); } -// La flèche en volume : extrusion arrondie, bombée d'un dôme qui plafonne avant l'axe médian de -// la silhouette (un dôme qui monterait encore y plierait le dessus). -float s_arrow_solid(float3 p, float h, float re, float dome) +// Une pièce cerclée : (distance, matière). `d` : le bord intérieur du trait dans le plan, `sil` : +// le même avant les rainures, `dc` : le coussin. Le plateau (matière 5) monte de SCULPT_HOVER à +// `zt` sous la silhouette élargie du trait `w`. Le jonc (matière 5), un tore de rayon w/2 à la +// hauteur `zt`, suit le milieu du trait : dans une rainure plus étroite que le trait il n'en reste +// que la crête, un muret entre les doigts. Le coussin de la matière `mat` sort du plateau, haut de +// `h` au bord, bombé sur 0,045 puis de `bump` ; sa hauteur variable penche le champ, d'où la +// distance minorée (× 0,8). +float2 s_piece(float d, float dc, float sil, float z, float w, float zt, float h, float bump, float mat) { - float d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - float hh = h + dome * smoothstep(0.0, 0.07, -d2); - return s_extrude(d2, p.z - (SCULPT_HOVER + h + dome), hh, re); + float tray = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + zt), 0.5 * (zt - SCULPT_HOVER), 0.012); + float bead = length(float2(d - 0.5 * w, z - zt)) - 0.5 * w; + float u = saturate(-dc / 0.045); + float cushion = 0.8 * s_extrude(dc, z - zt, h + 0.022 * u * (2.0 - u) + bump, 0.75 * h); + return s_opu(float2(min(tray, bead), 5.0), float2(cushion, mat)); } -// Le liseré de Studio Ink : un jonc posé sur le dessus, en retrait du bord. -float s_piping(float3 p, float ztop) +// Pop Coral est du papier découpé : des feuilles à plat, au dessus plat et au bord à peine +// cassé. La feuille marine sous la silhouette élargie du trait `w`, de SCULPT_HOVER à 0,15 ; la +// feuille de couleur (matière 1) posée dessus, 0,02 d'épaisseur, découpée des rainures entre les +// doigts qui laissent voir la marine. +float2 s_paper(float d, float sil, float z, float w) { - float d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - return length(float2(d2 + 0.075, p.z - ztop)) - 0.017; + float navy = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + 0.15), 0.5 * (0.15 - SCULPT_HOVER), 0.006); + float sheet = s_extrude(d, z - 0.16, 0.01, 0.004); + return s_opu(float2(navy, 5.0), float2(sheet, 1.0)); } -// Le gant : l'index levé, trois doigts repliés, le pouce, fondus dans la paume. -float s_glove(float3 p, float zc) +// Bosse de rayon r en c, 1 au centre et 0 au bord : un coussin bombé sans pli. Une hauteur tirée +// de la distance au bord plierait le coussin sur l'axe médian de sa silhouette. +float s_bump(float2 p, float2 c, float r) { - float index = s_capsule(p, float3(0.0, -0.10, zc), float3(0.0, -0.52, zc), 0.098); - float palm = s_round_box(p - float3(0.185, -0.70, zc), float3(0.255, 0.19, 0.105), 0.1); - float f1 = s_capsule(p, float3(0.17, -0.57, zc + 0.012), float3(0.17, -0.41, zc + 0.045), 0.086); - float f2 = s_capsule(p, float3(0.31, -0.59, zc + 0.01), float3(0.31, -0.45, zc + 0.04), 0.08); - float f3 = s_capsule(p, float3(0.435, -0.625, zc + 0.005), float3(0.435, -0.52, zc + 0.03), 0.07); - float thumb = s_capsule(p, float3(0.03, -0.77, zc + 0.03), float3(-0.165, -0.60, zc + 0.065), 0.082); - float d = s_smin(palm, min(f1, min(f2, f3)), 0.035); - d = s_smin(d, index, 0.05); - return s_smin(d, thumb, 0.05); + float k = saturate(1.0 - dot(p - c, p - c) / (r * r)); + return k * k; } -// La manchette, ronde autour du poignet (elliptique en xz) : le gant y entre. -float s_cuff(float3 p, float zc) +// Un tiret du clic de Pop Coral : capsule inégale de a (rayon ra) à b (rayon rb), exacte. +float s_dash(float2 p, float2 a, float2 b, float ra, float rb) { - float3 q = p - float3(0.185, -0.925, zc); - float2 ab = float2(0.272, 0.12); - float e = (length(q.xz / ab) - 1.0) * min(ab.x, ab.y); - return s_extrude(e, q.y, 0.07, 0.06); + p -= a; + b -= a; + float hb = dot(b, b); + float2 q = float2(abs(dot(p, float2(b.y, -b.x))), dot(p, b)) / hb; + float2 c = float2(sqrt(hb - (ra - rb) * (ra - rb)), ra - rb); + float k = c.x * q.y - c.y * q.x; + if (k < 0.0) + { + return sqrt(hb * dot(q, q)) - ra; + } + if (k > c.x) + { + return sqrt(hb * (dot(q, q) + 1.0 - 2.0 * q.y)) - rb; + } + return dot(c, q) - ra; } -// L'étoile de Star Sprout, face à la caméra, centrée en c : son visage et ses deux feuilles. -float2 s_sprout(float3 p, float3 c) +// Les thèmes cerclés. La flèche : Star Sprout menthe, Pop Coral corail, et Studio Ink noir, dont le +// trait se partage en un jonc noir au bord et une bande ivoire en relief, le champ restant le +// dessus du plateau. Le gant ivoire, jaune ou ivoire. Pop Coral est en papier découpé +// (`s_paper`), ses calques et ses tirets aussi : des feuilles au dessus plat. Puis ce que chacun ajoute devant : le calque +// jaune de la flèche, le calque corail du gant et les tirets du clic de Pop Coral ; la manchette, +// l'étoile, ses feuilles et ses yeux de Star Sprout, lus dans le repère de l'étoile (centre `c`, +// tournée, en unités de son rayon `rs`). Matières : 1 corps, 2 bande ivoire, calque jaune ou +// manchette, 3 tirets, calque corail ou étoile, 4 feuilles, 5 le trait (plateaux, joncs, yeux). +float2 s_rimmed(float3 p, int theme, int shape) { - float3 q = p - c; - float st2 = s_star5(q.xy, 0.125, 0.52) - 0.022; - float2 r = float2(s_extrude(st2, q.z, 0.038, 0.034), 3.0); - float3 e = float3(abs(q.x) - 0.032, q.y + 0.005, q.z - 0.036); - r = s_opu(r, float2(length(e) - 0.0135, 5.0)); - float3 l1 = q - float3(-0.04, 0.15, 0.0); - l1.xy = s_rot(l1.xy, -0.6); - float3 l2 = q - float3(0.045, 0.155, 0.0); - l2.xy = s_rot(l2.xy, 0.7); - float leaves = min(s_ellipsoid(l1, float3(0.03, 0.058, 0.02)), s_ellipsoid(l2, float3(0.03, 0.058, 0.02))); - return s_opu(r, float2(leaves, 4.0)); + bool arrow = shape == 0; + int g = theme == 4 ? 0 : (theme == 0 ? 1 : 2); + float poly = s_rim_poly(p.xy, arrow ? 0 : 7 * g + 7); + float2 body = float2(poly, poly); + float w = 0.06, zt = 0.17, h = 0.03; + float bump = 0.022 * s_bump(p.xy, float2(0.15, -0.45), 0.3); + float back = 1e9; + if (!arrow) + { + w = RIM_GLOVE_R[3 * g + 2].z; + // Le gant, puis pour Pop Coral son calque corail : le même gant décalé en bas à gauche. Une + // boucle et non deux appels (FXC recopierait le gant) ; le calque est sauté quand `p` est + // au-dessus du plateau du gant, qui en est alors plus près que lui. + [loop] for (int i = 0; i < 2; i++) + { + float2 q = p.xy + (i == 1 ? float2(0.03, 0.04) : float2(0.0, 0.0)); + float2 v = s_rim_glove(q, g, (i == 1 ? s_rim_poly(q, 7 * g + 7) : poly) - 0.05); + if (i == 1) + { + back = v.x - w; + } + else + { + body = v; + if (theme != 2 || v.x - w <= 0.0) + { + break; + } + } + } + zt = 0.185; + h = 0.028; + bump = 0.03 * s_bump(p.xy, float2(0.19, -0.62), 0.28); + } + float dc = body.y; + if (arrow && theme == 0) + { + // Studio Ink : pas de coussin, le jonc sur le bord extérieur du trait. + body -= 0.015; + w = 0.045; + dc = 1.0; + } + float2 r = theme == 2 ? s_paper(body.y, body.x, p.z, w) + : s_piece(body.y, dc, body.x, p.z, w, zt, h, bump, 1.0); + if (theme == 0) + { + if (arrow) + { + float band = s_extrude(abs(poly + 0.0075) - 0.0225, p.z - zt, 0.028, 0.012); + r = s_opu(r, float2(band, 2.0)); + } + return r; + } + if (theme == 2) + { + float dash; + if (arrow) + { + // Le calque jaune, décalé en bas à gauche, derrière le plateau. + float back = s_rim_poly(p.xy + float2(0.025, 0.05), 0) - 0.06; + r = s_opu(r, float2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 2.0)); + dash = min(s_dash(p.xy, float2(-0.1179, -0.1374), float2(-0.1799, -0.0443), 0.028, 0.045), + s_dash(p.xy, float2(-0.1347, -0.2493), float2(-0.248, -0.2056), 0.026, 0.042)); + } + else + { + r = s_opu(r, float2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 3.0)); + dash = min(s_dash(p.xy, float2(0.216, -0.1695), float2(0.2593, -0.0685), 0.028, 0.042), + s_dash(p.xy, float2(0.3054, -0.2339), float2(0.3949, -0.1652), 0.0275, 0.041)); + } + return s_opu(r, float2(s_extrude(dash, p.z - 0.11, 0.06, 0.006), 3.0)); + } + if (!arrow) + { + // La manchette : un rectangle arrondi, cintré en sourire comme au dessin. + float2 cq = p.xy - float2(0.1541, -0.9047); + float2 bq = abs(float2(cq.x, cq.y - 0.2066 * cq.x * cq.x)) - float2(0.214, 0.0361); + float cuff = length(max(bq, 0.0)) + min(max(bq.x, bq.y), 0.0) - 0.03; + r = s_opu(r, s_piece(cuff, cuff, cuff, p.z, 0.0477, 0.225, 0.025, 0.0, 2.0)); + } + float2 c = arrow ? float2(0.6118, -0.8079) : float2(0.15, -0.885); + float rs = arrow ? 0.1678 : 0.128; + float zs = arrow ? 0.23 : 0.285; + float2 q = s_rot(p.xy - c, arrow ? 0.2443 : 0.0) / rs; + float star = (s_star5(q, 0.82, 0.55) - 0.18) * rs; + float leaves = min(s_vesica(s_rot(q - float2(-0.33, 1.38), -0.925), 0.4296, 0.2626), + s_vesica(s_rot(q - float2(0.53, 1.37), 0.873), 0.4296, 0.2626)) * rs; + r = s_opu(r, s_piece(leaves, leaves, leaves, p.z, 0.038, zs - 0.02, 0.018, 0.0, 4.0)); + r = s_opu(r, s_piece(star, star, star, p.z, 0.038, zs, 0.022, 0.018 * s_bump(q, float2(0.0, 0.0), 1.0), 3.0)); + // Les yeux : deux ovales marine qui affleurent du coussin. + float3 e = float3(abs(q.x) - 0.25, q.y - 0.08, (p.z - zs - 0.056) / rs); + return s_opu(r, float2(s_ellipsoid(e, float3(0.075, 0.13, 0.1)) * rs, 5.0)); } -// Pixel Candy : une ligne par entier, bit c = colonne c, ligne 0 juste sous la pointe. La flèche -// est le polygone échantillonné à 16 cellules par unité (`sculpt.rs` dit comment la régénérer) ; -// la main est dessinée à la main, un gant échantillonné fondant ses doigts. Les tables *BACK -// portent la couche violette, la face dilatée d'une cellule, décalée d'une ligne et d'une colonne. -static const int SCULPT_ARROWPIX[16] = { 0, 1, 3, 7, 31, 63, 127, 255, 511, 63, 55, 115, 113, 224, 224, 64 }; -static const int SCULPT_ARROWBACK[18] = { - 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 255, 511, 511, 999, 995, 960, 192 +// Pixel Candy est un pixel art, dessiné une fois : la même grille donne ses PNG 2D et, ici, un cube +// par pixel. Une ligne par entier, bit c = colonne c, ligne 0 en haut. `PIX_BODY` : les pixels +// pleins ; `PIX_LINE`, `PIX_HI`, `PIX_SHADE` : ceux du contour prune, du reflet rose pâle et de +// l'ombre rose foncé, le reste est rose. `PIX_GRID` : colonnes, lignes et début de chaque forme +// dans les tables ; `PIX_ORIGIN` : le coin haut-gauche du pixel (0, 0) ; `PIX_BOX` (centre, +// demi-côtés) : la boîte de la forme ; `PIX_RECT`, bornés par `PIX_RECT_N` : la forme en +// rectangles, pour l'ombre. Tables générées par scripts/generate-pixel-candy-voxels.mjs, qui +// réécrit les trois shaders et les PNG : c'est là qu'on redessine une grille. +// +static const int PIX_BODY[32] = { 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 127, 247, 243, 480, 192, 48, 120, 120, 120, 504, 4088, 32760, 65534, 65535, 65535, 65534, 32766, 32764, 16380, 16376, 16376 }; +static const int PIX_LINE[32] = { 1, 3, 5, 9, 17, 33, 65, 129, 257, 513, 1985, 73, 149, 147, 288, 192, 48, 72, 72, 72, 456, 3656, 29256, 37454, 32777, 32769, 32770, 16386, 16388, 8196, 8200, 16376 }; +static const int PIX_HI[32] = { 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 18, 34, 32, 64, 0, 0, 16, 16, 16, 16, 16, 16, 16, 22, 18, 4, 4, 8, 8, 16, 0 }; +static const int PIX_SHADE[32] = { 0, 0, 0, 4, 8, 16, 32, 64, 128, 448, 40, 36, 64, 64, 128, 0, 0, 32, 32, 32, 32, 288, 2336, 18720, 16384, 16384, 16384, 8192, 8192, 4096, 8160, 0 }; +static const int PIX_RECT_N[3] = { 0, 17, 29 }; +static const int4 PIX_GRID[2] = { + int4(11, 16, 0, 0), + int4(16, 16, 16, 0) }; -static const int SCULPT_HANDPIX[15] = { 4, 14, 14, 14, 110, 878, 7022, 7022, 8190, 8191, 8191, 8191, 8190, 4092, 4092 }; -static const int SCULPT_HANDBACK[17] = { - 28, 62, 62, 62, 510, 4094, 32766, 32766, 32766, 32767, 32767, 32767, 32767, 32767, 32766, 16380, 16380 +static const float2 PIX_ORIGIN[2] = { + float2(0.0, 0.0), + float2(-0.3125, 0.0) }; -// La couche violette de la main, une plage de colonnes par ligne (toutes pleines) : la distance -// exacte à son contour, pour l'ombre portée. -static const float2 SCULPT_HANDSPAN[17] = { - float2(2.0, 4.0), float2(1.0, 5.0), float2(1.0, 5.0), float2(1.0, 5.0), float2(1.0, 8.0), - float2(1.0, 11.0), float2(1.0, 14.0), float2(1.0, 14.0), float2(1.0, 14.0), float2(0.0, 14.0), - float2(0.0, 14.0), float2(0.0, 14.0), float2(0.0, 14.0), float2(0.0, 14.0), float2(1.0, 14.0), - float2(2.0, 13.0), float2(2.0, 13.0) +static const float4 PIX_BOX[2] = { + float4(0.3438, -0.5, 0.3438, 0.5), + float4(0.1875, -0.5, 0.5, 0.5) }; +static const float4 PIX_RECT[29] = { + float4(0.0313, -0.0313, 0.0313, 0.0313), + float4(0.0625, -0.0938, 0.0625, 0.0313), + float4(0.0938, -0.1563, 0.0938, 0.0313), + float4(0.125, -0.2188, 0.125, 0.0313), + float4(0.1563, -0.2813, 0.1563, 0.0313), + float4(0.1875, -0.3438, 0.1875, 0.0313), + float4(0.2188, -0.4063, 0.2188, 0.0313), + float4(0.25, -0.4688, 0.25, 0.0313), + float4(0.2813, -0.5313, 0.2813, 0.0313), + float4(0.3125, -0.5938, 0.3125, 0.0313), + float4(0.3438, -0.6563, 0.3438, 0.0313), + float4(0.2188, -0.7188, 0.2188, 0.0313), + float4(0.0938, -0.7813, 0.0938, 0.0313), + float4(0.375, -0.8125, 0.125, 0.0625), + float4(0.0625, -0.8438, 0.0625, 0.0313), + float4(0.4375, -0.9063, 0.125, 0.0313), + float4(0.4375, -0.9688, 0.0625, 0.0313), + float4(0.0, -0.0313, 0.0625, 0.0313), + float4(0.0, -0.1563, 0.125, 0.0938), + float4(0.0625, -0.2813, 0.1875, 0.0313), + float4(0.1563, -0.3438, 0.2813, 0.0313), + float4(0.25, -0.4063, 0.375, 0.0313), + float4(0.2188, -0.4688, 0.4688, 0.0313), + float4(0.1875, -0.5625, 0.5, 0.0625), + float4(0.2188, -0.6563, 0.4688, 0.0313), + float4(0.1875, -0.7188, 0.4375, 0.0313), + float4(0.2188, -0.7813, 0.4063, 0.0313), + float4(0.1875, -0.8438, 0.375, 0.0313), + float4(0.2188, -0.9375, 0.3438, 0.0625) +}; +// -// Origine de la grille : la flèche part de sa pointe, la main de 2,5 cellules à sa gauche. -float2 s_grid_origin(int shape) -{ - return shape == 0 ? float2(0.0, 0.0) : float2(-2.5 * SCULPT_VOX, 0.0); -} - -// Le bit `c` de la ligne `r` d'une table de `rows` lignes et `cols` colonnes. FXC évalue les deux -// côtés d'un `&&` : l'index est ramené dans la table avant la lecture, le test fait le reste. -bool s_bit(int row, int r, int c, int rows, int cols) +// Le bit `c` d'une ligne. FXC évalue les deux côtés d'un `&&` : chaque index est ramené dans sa +// table avant la lecture, les tests de bornes font le reste. +bool s_pix_bit(int m, int c) { - return r >= 0 && r < rows && c >= 0 && c < cols && ((row >> clamp(c, 0, 31)) & 1) == 1; + return ((m >> clamp(c, 0, 31)) & 1) == 1; } -// La cellule `id` (colonne, ligne, y vers le haut) porte-t-elle un voxel de la face ? -bool s_occ(float2 id, int shape) +// Le pixel (c, r) de la forme `shape`, r compté vers le bas depuis la ligne 0, est-il plein ? +bool s_pix_body(int shape, int c, int r) { - int c = (int)id.x; - int r = -(int)id.y - 1; - if (shape == 0) - { - return s_bit(SCULPT_ARROWPIX[clamp(r, 0, 15)], r, c, 16, 16); - } - return s_bit(SCULPT_HANDPIX[clamp(r, 0, 14)], r, c, 15, 13); + int4 g = PIX_GRID[shape]; + return c >= 0 && c < g.x && r >= 0 && r < g.y && s_pix_bit(PIX_BODY[g.z + clamp(r, 0, g.y - 1)], c); } -// … et un bloc de la couche violette. -bool s_occ_back(float2 id, int shape) +// La cellule qui contient `p` (prototype, y vers le haut). +int2 s_pix_cell(float2 p, int shape) { - int c = (int)id.x + 1; - int r = -(int)id.y; - if (shape == 0) - { - return s_bit(SCULPT_ARROWBACK[clamp(r, 0, 17)], r, c, 18, 17); - } - return s_bit(SCULPT_HANDBACK[clamp(r, 0, 16)], r, c, 17, 15); + float2 o = PIX_ORIGIN[shape]; + return int2((int)floor((p.x - o.x) / SCULPT_VOX), (int)floor((o.y - p.y) / SCULPT_VOX)); } -// Chaque cellule pleine est un cube biseauté ; la couche violette est un étage plus bas. Hors du -// voisinage 3×3, une borne : une cellule au moins, et jamais plus près que la boîte de la forme. +// Un cube par pixel plein, tous de même hauteur, l'arête à peine cassée pour que chaque cube se +// lise. Hors du voisinage 3×3, une borne : une cellule au moins, et jamais plus près que la boîte +// de la forme. float2 s_voxels(float3 p, int shape) { - float2 o = s_grid_origin(shape); - float2 cell = floor((p.xy - o) / SCULPT_VOX); - float2 bc = shape == 0 ? float2(0.33, -0.5) : float2(0.25, -0.47); - float2 bh = shape == 0 ? float2(0.44, 0.61) : float2(0.48, 0.55); - float2 bq = max(abs(p.xy - bc) - bh, 0.0); + float4 b = PIX_BOX[shape]; + float2 bq = max(abs(p.xy - b.xy) - b.zw, 0.0); float bz = max(abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07, 0.0); - float far = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); - float dF = far; - float dB = far; - const float3 half_cell = float3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.04); + float d = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); + float2 o = PIX_ORIGIN[shape]; + int2 cell = s_pix_cell(p.xy, shape); + const float3 cube = float3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.07); [loop] for (int j = -1; j <= 1; j++) { [loop] for (int i = -1; i <= 1; i++) { - float2 id = cell + float2(i, j); - float3 q = float3(p.xy - (o + (id + 0.5) * SCULPT_VOX), p.z); - if (s_occ(id, shape)) - { - dF = min(dF, s_round_box(q - float3(0.0, 0.0, SCULPT_HOVER + 0.1), half_cell, 0.009)); - } - if (s_occ_back(id, shape)) + int c = cell.x + i; + int r = cell.y + j; + if (s_pix_body(shape, c, r)) { - dB = min(dB, s_round_box(q - float3(0.0, 0.0, SCULPT_HOVER + 0.04), half_cell, 0.009)); + float3 q = float3(p.x - o.x - (c + 0.5) * SCULPT_VOX, p.y - o.y + (r + 0.5) * SCULPT_VOX, + p.z - SCULPT_HOVER - 0.07); + d = min(d, s_round_box(q, cube, 0.006)); } } } - return dF < dB ? float2(dF, 6.0) : float2(dB, 7.0); + return float2(d, 6.0); } -// Distance, dans le plan de l'écran, au contour de la couche violette de la main. -float s_pixel_hand_dist(float2 p) +// Distance, dans le plan de l'écran, au contour de la forme. +float s_pixel_outline(float2 p, int shape) { float d = 1e9; - [loop] for (int r = 0; r < 17; r++) + [loop] for (int i = PIX_RECT_N[shape]; i < PIX_RECT_N[shape + 1]; i++) { - float2 x = (SCULPT_HANDSPAN[r] + float2(-3.5, -2.5)) * SCULPT_VOX; - float2 y = float2(-r, 1.0 - r) * SCULPT_VOX; - float2 q = max(max(float2(x.x, y.x) - p, p - float2(x.y, y.y)), 0.0); - d = min(d, length(q)); - } - return d; -} - -// Prism Glow est taillé, pas peint : chaque pièce est un polyèdre convexe (le max de ses plans), -// les facettes sont planes et la silhouette polygonale. - -// Une arête d'un contour convexe de gemme (a -> b, sens trigonométrique) : sa paroi, une facette -// de rondiste raide, et deux facettes de couronne qui se rejoignent au milieu de l'arête. Les -// couronnes de toutes les arêtes se rejoignent en arêtes vives au-dessus du centre. -float s_gem_edge(float3 p, float2 a, float2 b, float z0) -{ - float2 e = b - a; - float len = length(e); - float2 d = e / len; - float2 q = p.xy - a; - float dist = dot(q, float2(-d.y, d.x)); - float along = abs(dot(q, d) - 0.5 * len); - float z = p.z - z0; - float girdle = (z - 0.035 - 2.2 * dist) * 0.4138; - float crown = (z - 0.07 - 0.6 * dist + 0.18 * along) * 0.8438; - return max(-dist, max(girdle, crown)); -} - -float s_gem_arrow(float3 p) -{ - float z0 = SCULPT_HOVER + 0.03; - float slab = max(p.z - z0 - 0.2, z0 - 0.03 - p.z); - float head = max(slab, max(s_gem_edge(p, float2(-0.02, 0.03), float2(-0.02, -0.88), z0), - max(s_gem_edge(p, float2(-0.02, -0.88), float2(0.66, -0.61), z0), - s_gem_edge(p, float2(0.66, -0.61), float2(-0.02, 0.03), z0)))); - float tail = max(slab, max(max(s_gem_edge(p, float2(0.215, -0.665), float2(0.37, -1.0), z0), - s_gem_edge(p, float2(0.37, -1.0), float2(0.53, -0.93), z0)), - max(s_gem_edge(p, float2(0.53, -0.93), float2(0.38, -0.60), z0), - s_gem_edge(p, float2(0.38, -0.60), float2(0.215, -0.665), z0)))); - return min(head, tail); -} - -// Capsule taillée a -> b : un prisme hexagonal coiffé en b de deux couronnes de facettes -// décalées et d'une facette sommitale, tous les plans tangents à la sphère du bout. -float s_facet_capsule(float3 p, float3 a, float3 b, float r, float spin) -{ - float3 u = normalize(b - a); - float3 v = normalize(cross(u, float3(0.0, 0.0, 1.0))); - float3 w = cross(u, v); - float3 q = p - b; - float h = dot(q, u); - float2 rad = float2(dot(q, v), dot(q, w)); - float d = max(h - r, -dot(p - a, u) - r); - [loop] for (int k = 0; k < 6; k++) - { - float an = spin + k * 1.0471976; - float s = dot(rad, float2(cos(an), sin(an))); - d = max(d, s - r); - d = max(d, dot(rad, float2(cos(an + 0.5236), sin(an + 0.5236))) * 0.8660 + h * 0.5 - r); - d = max(d, s * 0.5 + h * 0.8660 - r); + float2 q = abs(p - PIX_RECT[i].xy) - PIX_RECT[i].zw; + d = min(d, length(max(q, 0.0)) + min(max(q.x, q.y), 0.0)); } return d; } -// Ellipsoïde taillé : les plans tangents dans dix directions du premier octant, reportées dans -// les huit. Une pierre aux facettes de taille égale. -static const float3 SCULPT_FACETS[10] = { - float3(1.0, 0.0, 0.0), float3(0.0, 1.0, 0.0), float3(0.0, 0.0, 1.0), - float3(0.7071, 0.7071, 0.0), float3(0.7071, 0.0, 0.7071), float3(0.0, 0.7071, 0.7071), - float3(0.5774, 0.5774, 0.5774), float3(0.4472, 0.0, 0.8944), float3(0.0, 0.4472, 0.8944), - float3(0.3015, 0.3015, 0.9045) +// ---- Prism Glow : un cristal en MAILLAGE ---- +// Les facettes tracées sur l'art 2D du thème (`design/cursors/prism-glow/model`), en triangles dans +// le repère du MODÈLE lui-même (unité = côté du sprite 128 px, origine au hotspot, y vers le bas, +// dessus du serti en z = 0). Le serti marine est la silhouette de l'art extrudée sous z = 0, un +// champ de distance comme les autres sculptés : il donne la couverture, l'ombre portée et l'ombre +// de contact. Le cristal est lancé de rayons, triangles rangés par boîtes : réfraction par canal +// (dispersion), réflexions totales internes, sortie par son fond plat vers l'image composée sous +// le curseur, flous de confidentialité compris (sa copie en t5). Tables : miroir de `prism_mesh.rs` ; +// un triangle = quatre +// float4 (v0, drapeaux), (e1, r), (e2, g), (normale sortante, b) ; drapeaux = paroi (1) + 2 × +// arêtes réelles (bit k : l'arête opposée au sommet k) ; une boîte = deux float4 (coin bas, +// premier triangle), (coin haut, nombre de triangles). +// prism mesh: generated by design/cursors/prism-glow/model/export_compositor.py +static const int PRISM_TRI_START[2] = { 0, 31 }; +static const int PRISM_TRI_COUNT[2] = { 31, 130 }; +static const int PRISM_SIL_START[2] = { 0, 25 }; +static const int PRISM_SIL_COUNT[2] = { 25, 42 }; +static const int PRISM_OUT_START[2] = { 67, 76 }; +static const int PRISM_OUT_COUNT[2] = { 9, 34 }; +static const int PRISM_BOX_START[2] = { 0, 4 }; +static const int PRISM_BOX_COUNT[2] = { 4, 18 }; +static const float4 PRISM_TRIS[644] = { + float4(0.252301, 0.266258, -0.042945, 11.0), + float4(-0.221166, -0.203988, 0.0, 0.01033), + float4(-0.221166, -0.203988, 0.042945, 0.045182), + float4(0.677984, -0.735077, 0.0, 0.577584), + float4(0.252301, 0.266258, -0.042945, 3.0), + float4(-0.221166, -0.203988, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.677984, -0.735077, 0.0, 0.577584), + float4(0.031135, 0.06227, -0.042945, 7.0), + float4(0.004294, 0.419785, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-0.999948, 0.01023, 0.0, 0.577584), + float4(0.169632, 0.356442, 0.115951, 14.0), + float4(0.082669, -0.090184, -0.115951, 0.590616), + float4(-0.138497, -0.294172, -0.115951, 0.737911), + float4(0.466385, -0.505659, 0.725806, 0.973449), + float4(0.035429, 0.482055, 0.0, 14.0), + float4(0.134202, -0.125614, 0.115951, 0.000305), + float4(-0.004294, -0.419785, 0.0, 0.938689), + float4(-0.650191, 0.006652, 0.759742, 0.854994), + float4(0.031135, 0.06227, -0.042945, 9.0), + float4(0.004294, 0.419785, 0.0, 0.01033), + float4(0.004294, 0.419785, 0.042945, 0.045182), + float4(-0.999948, 0.01023, 0.0, 0.577584), + float4(0.169632, 0.356442, 0.115951, 14.0), + float4(0.159969, 0.080521, -0.045092, 0.381323), + float4(0.082669, -0.090184, -0.115951, 0.056123), + float4(0.461407, -0.510214, 0.725799, 0.973449), + float4(0.541104, 0.544325, -0.042945, 9.0), + float4(-0.288804, -0.278068, 0.0, 0.01033), + float4(-0.288804, -0.278068, 0.042945, 0.045182), + float4(0.69359, -0.72037, 0.0, 0.577584), + float4(0.329601, 0.436963, 0.070859, 14.0), + float4(0.211503, 0.107362, -0.070859, 0.545724), + float4(-0.077301, -0.170706, -0.070859, 0.064805), + float4(0.49567, -0.514808, 0.699488, 0.973449), + float4(0.268405, 0.533589, 0.081595, 14.0), + float4(0.061196, -0.096626, -0.010736, 0.000916), + float4(-0.098773, -0.177147, 0.034356, 0.046662), + float4(0.247834, 0.049458, 0.967539, 0.904662), + float4(0.541104, 0.544325, -0.042945, 7.0), + float4(-0.288804, -0.278068, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.69359, -0.72037, 0.0, 0.577584), + float4(0.199693, 0.595859, 0.0, 14.0), + float4(0.068712, -0.06227, 0.081595, 0.0), + float4(-0.030061, -0.239417, 0.115951, 0.01445), + float4(-0.504461, 0.426837, 0.750553, 0.254154), + float4(0.268405, 0.533589, 0.081595, 14.0), + float4(0.272699, 0.010736, -0.081595, 0.0), + float4(0.061196, -0.096626, -0.010736, 0.025193), + float4(0.283244, 0.073137, 0.956255, 0.527108), + float4(0.035429, 0.482055, 0.0, 14.0), + float4(0.0, 0.258742, 0.0, 0.0), + float4(0.134202, -0.125614, 0.115951, 0.473533), + float4(-0.653777, 0.0, 0.756687, 0.964691), + float4(0.359663, 0.559356, 0.0, 14.0), + float4(0.181442, -0.015031, 0.0, 0.006043), + float4(-0.091258, -0.025767, 0.081595, 0.076188), + float4(0.076465, 0.923048, 0.37701, 0.964691), + float4(0.035429, 0.482055, -0.042945, 3.0), + float4(0.0, 0.258742, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-1.0, 0.0, 0.0, 0.577584), + float4(0.035429, 0.482055, -0.042945, 11.0), + float4(0.0, 0.258742, 0.0, 0.01033), + float4(0.0, 0.258742, 0.042945, 0.045182), + float4(-1.0, 0.0, 0.0, 0.577584), + float4(0.035429, 0.740798, -0.042945, 7.0), + float4(0.164264, -0.144939, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.661622, 0.749838, 0.0, 0.577584), + float4(0.035429, 0.740798, 0.0, 14.0), + float4(0.164264, -0.144939, 0.0, 0.0), + float4(0.134202, -0.384356, 0.115951, 0.010956), + float4(0.332572, 0.376915, 0.864483, 0.215854), + float4(0.035429, 0.740798, -0.042945, 11.0), + float4(0.164264, -0.144939, 0.0, 0.01033), + float4(0.164264, -0.144939, 0.042945, 0.045182), + float4(0.661622, 0.749838, 0.0, 0.577584), + float4(0.199693, 0.595859, -0.042945, 7.0), + float4(0.105215, 0.22546, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-0.906183, 0.422886, 0.0, 0.577584), + float4(0.199693, 0.595859, 0.0, 14.0), + float4(0.105215, 0.22546, 0.0, 0.0), + float4(0.068712, -0.06227, 0.081595, 0.015991), + float4(-0.613882, 0.286478, 0.735581, 0.473533), + float4(0.199693, 0.595859, -0.042945, 11.0), + float4(0.105215, 0.22546, 0.0, 0.01033), + float4(0.105215, 0.22546, 0.042945, 0.045182), + float4(-0.906183, 0.422886, 0.0, 0.577584), + float4(0.304908, 0.821319, 0.0, 14.0), + float4(0.128834, -0.067638, 0.0, 0.0), + float4(-0.036503, -0.28773, 0.081595, 0.254154), + float4(0.133686, 0.254641, 0.957751, 0.973449), + float4(0.304908, 0.821319, -0.042945, 7.0), + float4(0.128834, -0.067638, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.464834, 0.885398, 0.0, 0.577584), + float4(0.433742, 0.753681, 0.0, 14.0), + float4(-0.07408, -0.194325, 0.0, 0.0), + float4(-0.165337, -0.220092, 0.081595, 0.964691), + float4(0.683362, -0.260508, 0.682021, 0.964691), + float4(0.433742, 0.753681, -0.042945, 11.0), + float4(-0.07408, -0.194325, 0.0, 0.01033), + float4(-0.07408, -0.194325, 0.042945, 0.045182), + float4(0.934406, -0.35621, 0.0, 0.577584), + float4(0.304908, 0.821319, -0.042945, 11.0), + float4(0.128834, -0.067638, 0.0, 0.01033), + float4(0.128834, -0.067638, 0.042945, 0.045182), + float4(0.464834, 0.885398, 0.0, 0.577584), + float4(0.433742, 0.753681, -0.042945, 7.0), + float4(-0.07408, -0.194325, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.934406, -0.35621, 0.0, 0.577584), + float4(0.359663, 0.559356, -0.042945, 7.0), + float4(0.181442, -0.015031, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.082557, 0.996586, 0.0, 0.577584), + float4(0.359663, 0.559356, -0.042945, 11.0), + float4(0.181442, -0.015031, 0.0, 0.01033), + float4(0.181442, -0.015031, 0.042945, 0.045182), + float4(0.082557, 0.996586, 0.0, 0.577584), + float4(0.044345, 0.043263, -0.043263, 11.0), + float4(-0.043263, -0.005408, 0.0, 0.01033), + float4(-0.043263, -0.005408, 0.043263, 0.045182), + float4(0.124035, -0.992278, 0.0, 0.577584), + float4(0.044345, 0.043263, -0.043263, 7.0), + float4(-0.043263, -0.005408, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.124035, -0.992278, 0.0, 0.577584), + float4(0.001082, 0.037855, -0.043263, 7.0), + float4(-0.044345, 0.040019, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.669964, -0.742393, 0.0, 0.577584), + float4(0.044345, 0.043263, 0.0, 10.0), + float4(-0.043263, -0.005408, 0.0, 0.000305), + float4(-0.040019, 0.030284, 0.049753, 0.964691), + float4(0.101434, -0.811469, 0.575525, 0.973449), + float4(0.004326, 0.073548, 0.049753, 14.0), + float4(-0.003245, -0.035692, -0.049753, 0.0), + float4(-0.04759, 0.004326, -0.049753, 0.955978), + float4(-0.580475, -0.643229, 0.499304, 0.896269), + float4(0.001082, 0.037855, -0.043263, 11.0), + float4(-0.044345, 0.040019, 0.0, 0.01033), + float4(-0.044345, 0.040019, 0.043263, 0.045182), + float4(-0.669964, -0.742393, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.035692, -0.155748, -0.008653, 0.000305), + float4(-0.011897, -0.151422, -0.058405, 0.955978), + float4(-0.716533, -0.20131, 0.667874, 0.904662), + float4(-0.043263, 0.077874, -0.043263, 7.0), + float4(-0.001082, 0.154666, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999976, -0.006993, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.089771, -0.137361, -0.02055, 0.445197), + float4(0.035692, -0.155748, -0.008653, 0.973449), + float4(0.216369, -0.004654, 0.976301, 0.973449), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(-0.011897, -0.151422, -0.058405, 0.000305), + float4(-0.012979, 0.003245, -0.058405, 0.955978), + float4(-0.976245, -0.006827, 0.216564, 0.904662), + float4(-0.043263, 0.077874, -0.043263, 9.0), + float4(-0.001082, 0.154666, 0.0, 0.01033), + float4(-0.001082, 0.154666, 0.043263, 0.045182), + float4(-0.999976, -0.006993, 0.0, 0.577584), + float4(-0.045426, 0.564586, 0.052998, 14.0), + float4(0.014061, -0.335291, 0.005408, 0.520989), + float4(0.001082, -0.332046, -0.052998, 0.088655), + float4(-0.975941, -0.037462, 0.214794, 0.973449), + float4(-0.044345, 0.23254, 0.0, 3.0), + float4(0.0, 0.0, -0.043263, 0.01033), + float4(-0.001082, 0.332046, -0.043263, 0.045182), + float4(-0.999995, -0.003257, 0.0, 0.577584), + float4(-0.163319, 0.380717, -0.043263, 7.0), + float4(-0.060569, 0.02704, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.407651, -0.913138, 0.0, 0.577584), + float4(-0.094098, 0.431551, -0.043263, 11.0), + float4(-0.069221, -0.050834, 0.0, 0.01033), + float4(-0.069221, -0.050834, 0.043263, 0.045182), + float4(0.591909, -0.806004, 0.0, 0.577584), + float4(-0.163319, 0.380717, -0.043263, 11.0), + float4(-0.060569, 0.02704, 0.0, 0.01033), + float4(-0.060569, 0.02704, 0.043263, 0.045182), + float4(-0.407651, -0.913138, 0.0, 0.577584), + float4(0.075711, 0.08869, -0.043263, 11.0), + float4(-0.031366, -0.045426, 0.0, 0.01033), + float4(-0.031366, -0.045426, 0.043263, 0.045182), + float4(0.822897, -0.568191, 0.0, 0.577584), + float4(0.044345, 0.043263, 0.0, 6.0), + float4(-0.040019, 0.030284, 0.049753, 0.000305), + float4(0.014061, 0.048671, 0.037855, 0.964691), + float4(0.365607, -0.634949, 0.680566, 0.973449), + float4(0.075711, 0.08869, -0.043263, 7.0), + float4(-0.031366, -0.045426, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.822897, -0.568191, 0.0, 0.577584), + float4(0.058405, 0.091934, 0.037855, 14.0), + float4(0.017305, -0.003245, -0.037855, 0.0), + float4(-0.014061, -0.048671, -0.037855, 0.473533), + float4(0.757369, -0.522945, 0.39105, 0.904662), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.107077, -0.140606, -0.058405, 0.0), + float4(0.089771, -0.137361, -0.02055, 0.376257), + float4(0.812052, 0.481346, 0.329966, 0.830777), + float4(0.077874, 0.407756, -0.043263, 11.0), + float4(-0.002163, -0.319067, 0.0, 0.01033), + float4(-0.002163, -0.319067, 0.043263, 0.045182), + float4(0.999977, -0.006779, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.10924, 0.178461, -0.058405, 0.0), + float4(0.107077, -0.140606, -0.058405, 0.376257), + float4(0.475583, -0.003224, 0.879665, 0.830777), + float4(0.147095, 0.272559, -0.043263, 7.0), + float4(-0.035692, 0.034611, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.696146, -0.7179, 0.0, 0.577584), + float4(0.147095, 0.272559, -0.043263, 11.0), + float4(-0.035692, 0.034611, 0.0, 0.01033), + float4(-0.035692, 0.034611, 0.043263, 0.045182), + float4(-0.696146, -0.7179, 0.0, 0.577584), + float4(0.144932, 0.307169, 0.049753, 14.0), + float4(0.002163, -0.034611, -0.049753, 0.0), + float4(-0.033529, 0.0, -0.049753, 0.9131), + float4(-0.630237, -0.649931, 0.424725, 0.904662), + float4(0.183869, 0.317985, 0.037855, 6.0), + float4(-0.036774, -0.045426, -0.037855, 0.0), + float4(-0.038937, -0.010816, 0.011897, 0.806958), + float4(0.374433, -0.753477, 0.540438, 0.854994), + float4(0.077874, 0.407756, -0.043263, 7.0), + float4(-0.002163, -0.319067, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.999977, -0.006779, 0.0, 0.577584), + float4(0.111403, 0.307169, -0.043263, 7.0), + float4(-0.002163, 0.131953, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999866, -0.016391, 0.0, 0.577584), + float4(0.10924, 0.439122, 0.0, 6.0), + float4(0.035692, -0.131953, 0.049753, 0.18117), + float4(0.002163, -0.131953, 0.0, 0.955978), + float4(-0.82919, -0.013593, 0.558802, 0.964691), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(0.055161, -0.146014, -0.025958, 0.246204), + float4(0.016224, -0.156829, -0.014061, 0.964691), + float4(0.305397, -0.053644, 0.950713, 0.973449), + float4(0.111403, 0.307169, -0.043263, 11.0), + float4(-0.002163, 0.131953, 0.0, 0.01033), + float4(-0.002163, 0.131953, 0.043263, 0.045182), + float4(-0.999866, -0.016391, 0.0, 0.577584), + float4(0.189277, 0.28013, -0.043263, 11.0), + float4(-0.042182, -0.007571, 0.0, 0.01033), + float4(-0.042182, -0.007571, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.183869, 0.317985, 0.037855, 10.0), + float4(0.005408, -0.037855, -0.037855, 0.0), + float4(-0.036774, -0.045426, -0.037855, 0.806958), + float4(0.124328, -0.692684, 0.710445, 0.854994), + float4(0.189277, 0.28013, -0.043263, 7.0), + float4(-0.042182, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.183869, 0.317985, 0.037855, 14.0), + float4(0.02704, -0.004326, -0.037855, 0.0), + float4(0.005408, -0.037855, -0.037855, 0.514916), + float4(0.70062, -0.452013, 0.552101, 0.846876), + float4(0.210908, 0.313659, -0.043263, 11.0), + float4(-0.021632, -0.033529, 0.0, 0.01033), + float4(-0.021632, -0.033529, 0.043263, 0.045182), + float4(0.840297, -0.542127, 0.0, 0.577584), + float4(0.210908, 0.445612, 0.0, 10.0), + float4(0.0, -0.131953, 0.0, 0.0), + float4(-0.02704, -0.127627, 0.037855, 0.439658), + float4(0.813734, 0.0, 0.581238, 0.871366), + float4(0.210908, 0.313659, -0.043263, 7.0), + float4(-0.021632, -0.033529, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.840297, -0.542127, 0.0, 0.577584), + float4(0.210908, 0.445612, -0.043263, 11.0), + float4(0.0, -0.131953, 0.0, 0.01033), + float4(0.0, -0.131953, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.241193, 0.343943, -0.043263, 7.0), + float4(-0.002163, 0.107077, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999796, -0.020198, 0.0, 0.577584), + float4(0.276885, 0.311496, -0.043263, 11.0), + float4(-0.035692, 0.032447, 0.0, 0.01033), + float4(-0.035692, 0.032447, 0.043263, 0.045182), + float4(-0.672673, -0.73994, 0.0, 0.577584), + float4(0.23903, 0.45102, 0.0, 14.0), + float4(0.042182, -0.108158, 0.046508, 0.000305), + float4(0.002163, -0.107077, 0.0, 0.846876), + float4(-0.758098, -0.015315, 0.651961, 0.896269), + float4(0.276885, 0.311496, -0.043263, 7.0), + float4(-0.035692, 0.032447, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.672673, -0.73994, 0.0, 0.577584), + float4(0.281211, 0.342862, 0.046508, 14.0), + float4(-0.004326, -0.031366, -0.046508, 0.0), + float4(-0.040019, 0.001082, -0.046508, 0.752941), + float4(-0.58651, -0.645161, 0.489667, 0.846876), + float4(0.246601, 0.459672, 0.06165, 14.0), + float4(0.070303, -0.101669, -0.024876, 0.262257), + float4(0.034611, -0.116811, -0.015142, 0.973449), + float4(0.27928, -0.041605, 0.959308, 0.973449), + float4(0.319067, 0.319067, -0.043263, 11.0), + float4(-0.042182, -0.007571, 0.0, 0.01033), + float4(-0.042182, -0.007571, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.319067, 0.319067, 0.0, 10.0), + float4(-0.042182, -0.007571, 0.0, 0.0), + float4(-0.037855, 0.023795, 0.046508, 0.896269), + float4(0.1483, -0.826245, 0.54344, 0.896269), + float4(0.319067, 0.319067, -0.043263, 7.0), + float4(-0.042182, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.319067, 0.319067, 0.0, 6.0), + float4(-0.037855, 0.023795, 0.046508, 0.0), + float4(-0.002163, 0.038937, 0.036774, 0.896269), + float4(0.437906, -0.604311, 0.665618, 0.896269), + float4(0.316904, 0.358004, 0.036774, 14.0), + float4(0.024876, -0.004326, -0.036774, 0.0), + float4(0.002163, -0.038937, -0.036774, 0.617212), + float4(0.707338, -0.464191, 0.533104, 0.921584), + float4(0.34178, 0.353677, -0.043263, 11.0), + float4(-0.022713, -0.034611, 0.0, 0.01033), + float4(-0.022713, -0.034611, 0.043263, 0.045182), + float4(0.836048, -0.548657, 0.0, 0.577584), + float4(0.340698, 0.474815, 0.0, 14.0), + float4(0.001082, -0.121137, 0.0, 0.0), + float4(-0.023795, -0.116811, 0.036774, 0.473533), + float4(0.828664, 0.007399, 0.559697, 0.930114), + float4(0.34178, 0.353677, -0.043263, 7.0), + float4(-0.022713, -0.034611, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.836048, -0.548657, 0.0, 0.577584), + float4(0.340698, 0.474815, -0.043263, 11.0), + float4(0.001082, -0.121137, 0.0, 0.01033), + float4(0.001082, -0.121137, 0.043263, 0.045182), + float4(0.99996, 0.008928, 0.0, 0.577584), + float4(0.411001, 0.351514, -0.043263, 11.0), + float4(-0.034611, 0.030284, 0.0, 0.01033), + float4(-0.034611, 0.030284, 0.043263, 0.045182), + float4(-0.658505, -0.752577, 0.0, 0.577584), + float4(0.411001, 0.351514, -0.043263, 7.0), + float4(-0.034611, 0.030284, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.658505, -0.752577, 0.0, 0.577584), + float4(0.412083, 0.38288, 0.043263, 14.0), + float4(-0.001082, -0.031366, -0.043263, 0.0), + float4(-0.035692, -0.001082, -0.043263, 0.745403), + float4(-0.574039, -0.656045, 0.489984, 0.854994), + float4(0.45102, 0.359085, -0.043263, 11.0), + float4(-0.040019, -0.007571, 0.0, 0.01033), + float4(-0.040019, -0.007571, 0.043263, 0.045182), + float4(0.185892, -0.98257, 0.0, 0.577584), + float4(0.45102, 0.359085, 0.0, 10.0), + float4(-0.040019, -0.007571, 0.0, 0.000305), + float4(-0.038937, 0.023795, 0.043263, 0.879622), + float4(0.151736, -0.802035, 0.577682, 0.9131), + float4(0.45102, 0.359085, -0.043263, 7.0), + float4(-0.040019, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.185892, -0.98257, 0.0, 0.577584), + float4(0.45102, 0.359085, 0.0, 6.0), + float4(-0.038937, 0.023795, 0.043263, 0.000305), + float4(0.003245, 0.0411, 0.032447, 0.879622), + float4(0.417852, -0.583049, 0.696744, 0.9131), + float4(0.473733, 0.391533, -0.043263, 11.0), + float4(-0.022713, -0.032447, 0.0, 0.01033), + float4(-0.022713, -0.032447, 0.043263, 0.045182), + float4(0.819232, -0.573462, 0.0, 0.577584), + float4(0.454264, 0.400185, 0.032447, 14.0), + float4(0.019468, -0.008653, -0.032447, 0.0), + float4(-0.003245, -0.0411, -0.032447, 0.527108), + float4(0.688617, -0.482032, 0.541712, 0.887922), + float4(0.473733, 0.391533, -0.043263, 7.0), + float4(-0.022713, -0.032447, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.819232, -0.573462, 0.0, 0.577584), + float4(-0.223888, 0.407756, -0.043263, 7.0), + float4(0.002163, 0.06814, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999496, 0.03173, 0.0, 0.577584), + float4(-0.223888, 0.407756, -0.043263, 11.0), + float4(0.002163, 0.06814, 0.0, 0.01033), + float4(0.002163, 0.06814, 0.043263, 0.045182), + float4(-0.999496, 0.03173, 0.0, 0.577584), + float4(-0.167645, 0.416409, 0.046508, 14.0), + float4(-0.056242, -0.008653, -0.046508, 0.000305), + float4(-0.054079, 0.059487, -0.046508, 0.955978), + float4(-0.638985, 0.020285, 0.768952, 0.973449), + float4(-0.167645, 0.416409, 0.046508, 14.0), + float4(0.004326, -0.035692, -0.046508, 0.000305), + float4(-0.056242, -0.008653, -0.046508, 0.955978), + float4(-0.339782, -0.761112, 0.552501, 0.973449), + float4(-0.221724, 0.475896, 0.0, 14.0), + float4(0.139524, 0.055161, 0.043263, 0.520989), + float4(0.054079, -0.059487, 0.046508, 0.973449), + float4(-0.39307, 0.317373, 0.863001, 0.973449), + float4(-0.167645, 0.416409, 0.046508, 6.0), + float4(0.073548, 0.015142, -0.046508, 0.0), + float4(0.004326, -0.035692, -0.046508, 0.768154), + float4(0.490917, -0.668482, 0.558688, 0.871366), + float4(-0.094098, 0.431551, -0.043263, 7.0), + float4(-0.069221, -0.050834, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.591909, -0.806004, 0.0, 0.577584), + float4(-0.167645, 0.416409, 0.046508, 10.0), + float4(0.085445, 0.114648, -0.003245, 0.0), + float4(0.073548, 0.015142, -0.046508, 0.768154), + float4(0.548356, -0.38771, 0.740936, 0.871366), + float4(-0.0822, 0.531057, -0.043263, 11.0), + float4(-0.011897, -0.099506, 0.0, 0.01033), + float4(-0.011897, -0.099506, 0.043263, 0.045182), + float4(0.992928, -0.11872, 0.0, 0.577584), + float4(-0.044345, 0.23254, 0.0, 7.0), + float4(-0.001082, 0.332046, -0.043263, 0.01033), + float4(-0.001082, 0.332046, 0.052998, 0.045182), + float4(-0.999995, -0.003257, 0.0, 0.577584), + float4(-0.045426, 0.564586, 0.052998, 14.0), + float4(0.1233, -0.156829, -0.052998, 0.520989), + float4(0.014061, -0.335291, 0.005408, 0.088655), + float4(0.429355, 0.032562, 0.902549, 0.973449), + float4(0.10924, 0.439122, 0.0, 14.0), + float4(-0.031366, -0.031366, 0.0, 0.132876), + float4(-0.154666, 0.125464, 0.052998, 0.028428), + float4(0.182761, -0.182761, 0.966021, 0.955978), + float4(0.10924, 0.439122, -0.043263, 11.0), + float4(-0.031366, -0.031366, 0.0, 0.01033), + float4(-0.031366, -0.031366, 0.043263, 0.045182), + float4(0.707107, -0.707107, 0.0, 0.577584), + float4(0.10924, 0.439122, -0.043263, 7.0), + float4(-0.031366, -0.031366, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.707107, -0.707107, 0.0, 0.577584), + float4(0.10924, 0.439122, 0.0, 10.0), + float4(0.019468, 0.024876, 0.063813, 0.18117), + float4(0.035692, -0.131953, 0.049753, 0.955978), + float4(-0.933936, -0.126584, 0.334276, 0.964691), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(0.050834, 0.002163, -0.005408, 0.0), + float4(0.0822, -0.018387, -0.063813, 0.439658), + float4(0.07859, -0.92643, 0.368172, 0.871366), + float4(-0.221724, 0.475896, -0.043263, 7.0), + float4(0.122219, 0.207664, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.861819, 0.507216, 0.0, 0.577584), + float4(-0.221724, 0.475896, -0.043263, 9.0), + float4(0.122219, 0.207664, 0.0, 0.01033), + float4(0.122219, 0.207664, 0.043263, 0.045182), + float4(-0.861819, 0.507216, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.017305, -0.152503, 0.043263, 0.0), + float4(-0.122219, -0.207664, 0.0, 0.723049), + float4(-0.365879, 0.215335, 0.905408, 0.854994), + float4(-0.0822, 0.531057, -0.043263, 7.0), + float4(-0.011897, -0.099506, 0.043263, 0.01033), + float4(0.0, 0.0, 0.086527, 0.045182), + float4(0.992928, -0.11872, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.054079, -0.118974, 0.052998, 0.0), + float4(0.017305, -0.152503, 0.043263, 0.514916), + float4(-0.41958, 0.203352, 0.884647, 0.863157), + float4(-0.099506, 0.68356, -0.043263, 3.0), + float4(0.086527, 0.149258, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.86514, 0.501531, 0.0, 0.577584), + float4(-0.045426, 0.564586, -0.043263, 11.0), + float4(-0.036774, -0.033529, 0.0, 0.01033), + float4(-0.036774, -0.033529, 0.086527, 0.045182), + float4(0.673754, -0.738956, 0.0, 0.577584), + float4(-0.045426, 0.564586, -0.043263, 7.0), + float4(-0.036774, -0.033529, 0.086527, 0.01033), + float4(0.0, 0.0, 0.096261, 0.045182), + float4(0.673754, -0.738956, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.086527, 0.149258, 0.0, 0.0), + float4(0.054079, -0.118974, 0.052998, 0.262257), + float4(-0.385563, 0.223515, 0.8952, 0.9131), + float4(-0.099506, 0.68356, -0.043263, 11.0), + float4(0.086527, 0.149258, 0.0, 0.01033), + float4(0.086527, 0.149258, 0.043263, 0.045182), + float4(-0.86514, 0.501531, 0.0, 0.577584), + float4(-0.012979, 0.832818, 0.0, 14.0), + float4(0.214153, -0.194685, 0.0822, 0.000305), + float4(-0.032447, -0.268232, 0.052998, 0.930114), + float4(-0.176863, 0.211325, 0.961281, 0.973449), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(-0.019468, -0.024876, -0.063813, 0.168276), + float4(-0.174135, 0.100587, -0.010816, 0.038209), + float4(-0.469238, -0.764881, 0.441331, 0.955978), + float4(0.201174, 0.638133, 0.0822, 14.0), + float4(-0.072466, -0.174135, -0.018387, 0.18117), + float4(-0.246601, -0.073548, -0.029203, 0.082277), + float4(-0.098561, -0.063844, 0.993081, 0.973449), + float4(-0.012979, 0.832818, -0.043263, 7.0), + float4(0.339617, -0.002163, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.006369, 0.99998, 0.0, 0.577584), + float4(0.179543, 0.466162, 0.058405, 10.0), + float4(-0.050834, -0.002163, 0.005408, 0.000305), + float4(0.021632, 0.171972, 0.023795, 0.022171), + float4(0.110896, -0.149888, 0.982464, 0.381323), + float4(-0.012979, 0.832818, 0.0, 14.0), + float4(0.339617, -0.002163, 0.0, 0.0), + float4(0.214153, -0.194685, 0.0822, 0.027314), + float4(0.002492, 0.391296, 0.920261, 0.49102), + float4(0.183869, 0.317985, 0.037855, 10.0), + float4(-0.055161, 0.146014, 0.025958, 0.0), + float4(0.02704, 0.127627, -0.037855, 0.439658), + float4(0.623841, 0.097823, 0.775405, 0.871366), + float4(0.210908, 0.445612, -0.043263, 7.0), + float4(0.0, -0.131953, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.210908, 0.445612, 0.0, 10.0), + float4(-0.031366, 0.02055, 0.058405, 0.000305), + float4(0.035692, 0.014061, 0.06165, 0.022171), + float4(-0.105864, -0.954447, 0.278971, 0.381323), + float4(0.23903, 0.45102, -0.043263, 11.0), + float4(-0.028121, -0.005408, 0.0, 0.01033), + float4(-0.028121, -0.005408, 0.043263, 0.045182), + float4(0.188847, -0.982006, 0.0, 0.577584), + float4(0.23903, 0.45102, -0.043263, 7.0), + float4(-0.028121, -0.005408, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.188847, -0.982006, 0.0, 0.577584), + float4(0.246601, 0.459672, 0.06165, 11.0), + float4(-0.007571, -0.008653, -0.06165, 0.000305), + float4(-0.035692, -0.014061, -0.06165, 0.022171), + float4(0.187618, -0.975617, 0.113888, 0.381323), + float4(0.241193, 0.343943, -0.043263, 11.0), + float4(-0.002163, 0.107077, 0.0, 0.01033), + float4(-0.002163, 0.107077, 0.043263, 0.045182), + float4(-0.999796, -0.020198, 0.0, 0.577584), + float4(0.23903, 0.45102, 0.0, 15.0), + float4(0.007571, 0.008653, 0.06165, 0.000305), + float4(0.042182, -0.108158, 0.046508, 0.846876), + float4(-0.94107, -0.299262, 0.157572, 0.896269), + float4(0.316904, 0.358004, 0.036774, 14.0), + float4(-0.070303, 0.101669, 0.024876, 0.0), + float4(0.023795, 0.116811, -0.036774, 0.473533), + float4(0.523424, 0.157027, 0.837478, 0.930114), + float4(0.340698, 0.474815, -0.043263, 7.0), + float4(0.001082, -0.121137, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.99996, 0.008928, 0.0, 0.577584), + float4(0.376391, 0.381799, -0.043263, 11.0), + float4(-0.001082, 0.107077, 0.0, 0.01033), + float4(-0.001082, 0.107077, 0.043263, 0.045182), + float4(-0.999949, -0.0101, 0.0, 0.577584), + float4(0.376391, 0.381799, -0.043263, 7.0), + float4(-0.001082, 0.107077, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999949, -0.0101, 0.0, 0.577584), + float4(0.375309, 0.488875, 0.0, 14.0), + float4(0.036774, -0.105995, 0.043263, 0.000305), + float4(0.001082, -0.107077, 0.0, 0.879622), + float4(-0.771254, -0.00779, 0.63648, 0.896269), + float4(0.375309, 0.488875, 0.0, 14.0), + float4(0.078955, -0.08869, 0.032447, 0.223224), + float4(0.036774, -0.105995, 0.043263, 0.964691), + float4(0.071225, 0.39802, 0.914608, 0.973449), + float4(0.473733, 0.603523, 0.0, 14.0), + float4(0.0, -0.21199, 0.0, 0.158953), + float4(-0.019468, -0.203337, 0.032447, 0.030716), + float4(0.857493, 0.0, 0.514496, 0.964691), + float4(0.473733, 0.603523, -0.043263, 11.0), + float4(0.0, -0.21199, 0.0, 0.01033), + float4(0.0, -0.21199, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.276885, 0.484549, 0.058405, 10.0), + float4(-0.030284, -0.024876, 0.003245, 0.000305), + float4(-0.097342, -0.018387, 0.0, 0.022171), + float4(-0.03153, 0.166923, 0.985466, 0.381323), + float4(0.246601, 0.459672, 0.06165, 14.0), + float4(0.030284, 0.024876, -0.003245, 0.0), + float4(0.094098, 0.015142, -0.06165, 0.473533), + float4(0.518869, -0.545855, 0.657888, 0.930114), + float4(0.375309, 0.488875, -0.043263, 11.0), + float4(-0.034611, -0.014061, 0.0, 0.01033), + float4(-0.034611, -0.014061, 0.043263, 0.045182), + float4(0.376377, -0.926467, 0.0, 0.577584), + float4(0.340698, 0.474815, 0.0, 14.0), + float4(-0.063813, 0.009734, 0.058405, 0.000305), + float4(0.034611, 0.014061, 0.0, 0.022171), + float4(0.3276, -0.806402, 0.492334, 0.381323), + float4(0.375309, 0.488875, -0.043263, 7.0), + float4(-0.034611, -0.014061, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.376377, -0.926467, 0.0, 0.577584), + float4(0.454264, 0.400185, 0.032447, 12.0), + float4(-0.078955, 0.08869, -0.032447, 0.158953), + float4(0.019468, 0.203337, -0.032447, 0.030716), + float4(-0.201684, 0.173144, 0.964025, 0.964691), + float4(0.201174, 0.638133, 0.0822, 8.0), + float4(0.075711, -0.153585, -0.023795, 0.000305), + float4(-0.021632, -0.171972, -0.023795, 0.022171), + float4(0.026497, -0.140281, 0.989757, 0.381323), + float4(0.473733, 0.603523, -0.043263, 7.0), + float4(0.0, -0.21199, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.276885, 0.484549, 0.058405, 14.0), + float4(-0.075711, 0.153585, 0.023795, 0.000305), + float4(0.098424, 0.004326, -0.058405, 0.022171), + float4(0.50316, 0.115345, 0.856461, 0.381323), + float4(0.375309, 0.488875, 0.0, 10.0), + float4(0.090853, 0.12979, 0.0, 0.158953), + float4(0.098424, 0.114648, 0.0, 0.030716), + float4(0.0, 0.0, 1.0, 0.964691), + float4(0.466162, 0.618665, -0.043263, 11.0), + float4(0.007571, -0.015142, 0.0, 0.01033), + float4(0.007571, -0.015142, 0.043263, 0.045182), + float4(0.894427, 0.447214, 0.0, 0.577584), + float4(0.466162, 0.618665, -0.043263, 3.0), + float4(0.007571, -0.015142, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.894427, 0.447214, 0.0, 0.577584), + float4(0.326638, 0.830655, 0.0, 14.0), + float4(0.139524, -0.21199, 0.0, 0.0), + float4(0.048671, -0.34178, 0.0, 0.015213), + float4(0.0, 0.0, 1.0, 0.428687), + float4(0.326638, 0.830655, 0.0, 14.0), + float4(0.048671, -0.34178, 0.0, 0.007492), + float4(-0.125464, -0.192522, 0.0822, 0.054475), + float4(0.472493, 0.067285, 0.878762, 0.964691), + float4(0.326638, 0.830655, -0.043263, 9.0), + float4(0.139524, -0.21199, 0.0, 0.01033), + float4(0.139524, -0.21199, 0.043263, 0.045182), + float4(0.835314, 0.549773, 0.0, 0.577584), + float4(0.326638, 0.830655, -0.043263, 7.0), + float4(0.139524, -0.21199, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.835314, 0.549773, 0.0, 0.577584), + float4(-0.012979, 0.832818, -0.043263, 11.0), + float4(0.339617, -0.002163, 0.0, 0.01033), + float4(0.339617, -0.002163, 0.043263, 0.045182), + float4(0.006369, 0.99998, 0.0, 0.577584) }; +static const float4 PRISM_BOXES[44] = { + float4(0.030135, 0.06127, -0.043945, 0.0), + float4(0.330601, 0.483055, 0.116951, 7.0), + float4(0.034429, 0.265258, -0.043945, 7.0), + float4(0.542104, 0.741798, 0.116951, 8.0), + float4(0.034429, 0.355442, -0.043945, 15.0), + float4(0.305908, 0.822319, 0.116951, 8.0), + float4(0.267405, 0.532589, -0.043945, 23.0), + float4(0.542104, 0.822319, 0.082595, 8.0), + float4(-0.045345, 0.036855, -0.044263, 31.0), + float4(0.045345, 0.23354, 0.059405, 8.0), + float4(-0.224888, 0.072548, -0.044263, 39.0), + float4(0.059405, 0.565586, 0.059405, 8.0), + float4(-0.032366, 0.042263, -0.044263, 47.0), + float4(0.148095, 0.408756, 0.059405, 8.0), + float4(0.074711, 0.08769, -0.044263, 55.0), + float4(0.184869, 0.464999, 0.064813, 8.0), + float4(0.146095, 0.271559, -0.044263, 63.0), + float4(0.211908, 0.446612, 0.038855, 8.0), + float4(0.23803, 0.310496, -0.044263, 71.0), + float4(0.320067, 0.460672, 0.06265, 8.0), + float4(0.275885, 0.310496, -0.044263, 79.0), + float4(0.412001, 0.475815, 0.047508, 8.0), + float4(0.375391, 0.350514, -0.044263, 87.0), + float4(0.45202, 0.38388, 0.044263, 4.0), + float4(0.410001, 0.350514, -0.044263, 91.0), + float4(0.474733, 0.401185, 0.044263, 5.0), + float4(-0.224888, 0.379717, -0.044263, 96.0), + float4(-0.0812, 0.532057, 0.047508, 8.0), + float4(-0.095098, 0.228295, -0.044263, 104.0), + float4(0.211908, 0.565586, 0.064813, 8.0), + float4(-0.222724, 0.430551, -0.044263, 112.0), + float4(-0.011979, 0.833818, 0.053998, 8.0), + float4(-0.100506, 0.438122, -0.044263, 120.0), + float4(0.327638, 0.833818, 0.0832, 8.0), + float4(0.127708, 0.312659, -0.044263, 128.0), + float4(0.282211, 0.467162, 0.064813, 8.0), + float4(0.245601, 0.352677, -0.044263, 136.0), + float4(0.474733, 0.604523, 0.06265, 8.0), + float4(0.178543, 0.390533, -0.044263, 144.0), + float4(0.474733, 0.639133, 0.0832, 8.0), + float4(0.200174, 0.483549, -0.044263, 152.0), + float4(0.474733, 0.639133, 0.0832, 4.0), + float4(-0.013979, 0.487875, -0.044263, 156.0), + float4(0.467162, 0.833818, 0.0832, 5.0) +}; +static const float2 PRISM_POLY[110] = { + float2(0.006442, -0.002147), + float2(-0.001074, 0.001074), + float2(-0.004294, 0.007515), + float2(-0.004294, 0.806288), + float2(-0.001074, 0.809509), + float2(0.004294, 0.809509), + float2(0.022546, 0.796626), + float2(0.185736, 0.655982), + float2(0.281288, 0.86319), + float2(0.286656, 0.870706), + float2(0.293098, 0.871779), + float2(0.465951, 0.781595), + float2(0.478834, 0.773006), + float2(0.478834, 0.763344), + float2(0.407975, 0.592638), + float2(0.45092, 0.586196), + float2(0.601227, 0.573313), + float2(0.606595, 0.569018), + float2(0.606595, 0.564724), + float2(0.601227, 0.556135), + float2(0.50138, 0.453067), + float2(0.425153, 0.37684), + float2(0.289877, 0.246933), + float2(0.081595, 0.057975), + float2(0.013957, 0.0), + float2(-0.011897, 0.0), + float2(-0.055161, 0.037855), + float2(-0.081119, 0.063813), + float2(-0.083282, 0.06814), + float2(-0.084363, 0.38288), + float2(-0.089771, 0.38288), + float2(-0.158993, 0.33529), + float2(-0.166564, 0.336372), + float2(-0.250927, 0.379635), + float2(-0.261743, 0.388288), + float2(-0.263906, 0.461836), + float2(-0.262824, 0.486712), + float2(-0.173053, 0.638133), + float2(-0.044345, 0.860939), + float2(-0.036774, 0.867429), + float2(-0.024876, 0.871755), + float2(0.33529, 0.871755), + float2(0.349351, 0.867429), + float2(0.363412, 0.853368), + float2(0.513752, 0.619747), + float2(0.513752, 0.379635), + float2(0.466162, 0.322311), + float2(0.461836, 0.320148), + float2(0.399104, 0.311496), + float2(0.393696, 0.313659), + float2(0.374227, 0.332046), + float2(0.343943, 0.288782), + float2(0.336372, 0.282293), + float2(0.269314, 0.271477), + float2(0.263906, 0.27364), + float2(0.243356, 0.292027), + float2(0.21199, 0.244438), + float2(0.207664, 0.241193), + float2(0.149258, 0.231459), + float2(0.138443, 0.231459), + float2(0.118974, 0.248764), + float2(0.116811, 0.222806), + float2(0.117892, 0.162237), + float2(0.115729, 0.076792), + float2(0.06814, 0.011897), + float2(0.062732, 0.007571), + float2(0.005408, 0.0), + float2(0.031135, 0.06227), + float2(0.252301, 0.266258), + float2(0.541104, 0.544325), + float2(0.359663, 0.559356), + float2(0.433742, 0.753681), + float2(0.304908, 0.821319), + float2(0.199693, 0.595859), + float2(0.035429, 0.740798), + float2(0.035429, 0.482055), + float2(-0.043263, 0.077874), + float2(0.001082, 0.037855), + float2(0.044345, 0.043263), + float2(0.075711, 0.08869), + float2(0.077874, 0.407756), + float2(0.10924, 0.439122), + float2(0.111403, 0.307169), + float2(0.147095, 0.272559), + float2(0.189277, 0.28013), + float2(0.210909, 0.313659), + float2(0.210909, 0.445612), + float2(0.23903, 0.45102), + float2(0.241193, 0.343943), + float2(0.276885, 0.311496), + float2(0.319067, 0.319067), + float2(0.34178, 0.353677), + float2(0.340698, 0.474815), + float2(0.375309, 0.488875), + float2(0.376391, 0.381799), + float2(0.411001, 0.351514), + float2(0.45102, 0.359085), + float2(0.473733, 0.391533), + float2(0.473733, 0.603523), + float2(0.466162, 0.618665), + float2(0.326638, 0.830655), + float2(-0.012979, 0.832818), + float2(-0.099506, 0.68356), + float2(-0.221724, 0.475896), + float2(-0.223888, 0.407756), + float2(-0.163319, 0.380717), + float2(-0.094098, 0.431551), + float2(-0.0822, 0.531057), + float2(-0.045426, 0.564586), + float2(-0.044345, 0.23254) +}; +static const float PRISM_BEVEL = 0.012883; +// end of the prism mesh + +// Le verre validé sur le prototype (30/09/2026). +static const float PRISM_IOR = 1.61; +static const float PRISM_DISPERSION = 0.035; +static const float PRISM_GLOW = 0.45; +static const float PRISM_FOLD = 0.35; +// Rebonds au plus dans le cristal, par canal ; l'id de matière du cristal. +static const int PRISM_BOUNCES = 6; +static const float PRISM_MAT = 9.0; + +// Le modèle de Prism Glow de ce dessin : 0 flèche, 1 main, -1 un autre curseur. +int prism_shape() +{ + int id = sculpt_id(); + return id == 3 || id == 4 ? id - 3 : -1; +} -float s_facet_ellipsoid(float3 p, float3 r) +// Distance signée au polygone `n` sommets de PRISM_POLY à partir de `i0` (plan xy du modèle), +// négative dedans : la silhouette du serti, ou le contour du cristal. +float prism_poly_dist(float2 p, int i0, int n) { - float3 q = abs(p); - float d = -1e9; - [loop] for (int i = 0; i < 10; i++) + float2 vj = PRISM_POLY[i0 + n - 1]; + float d = 1e9; + float sgn = 1.0; + [loop] for (int i = 0; i < n; i++) { - d = max(d, dot(q, SCULPT_FACETS[i]) - length(r * SCULPT_FACETS[i])); + float2 vi = PRISM_POLY[i0 + i]; + float2 e = vj - vi; + float2 w = p - vi; + float2 b = w - e * saturate(dot(w, e) / dot(e, e)); + d = min(d, dot(b, b)); + bool c0 = p.y >= vi.y; + bool c1 = p.y < vj.y; + bool c2 = e.x * w.y > e.y * w.x; + if ((c0 && c1 && c2) || (!c0 && !c1 && !c2)) + { + sgn = -sgn; + } + vj = vi; } - return d; + return sgn * sqrt(d); } -// Le gant des autres thèmes, mêmes os, taillé en facettes. -float s_crystal_hand(float3 p) +// Le serti : la silhouette extrudée de z = -épaisseur à 0, arêtes arrondies de PRISM_BEVEL. +float prism_rim(float3 p, int shape) { - float zc = SCULPT_HAND_ZC; - float d = s_facet_ellipsoid(p - float3(0.185, -0.70, zc), float3(0.3, 0.235, 0.125)); - d = min(d, s_facet_capsule(p, float3(0.0, -0.52, zc), float3(0.0, -0.10, zc), 0.098, 0.3)); - d = min(d, s_facet_capsule(p, float3(0.17, -0.57, zc + 0.012), float3(0.17, -0.41, zc + 0.045), 0.086, 0.1)); - d = min(d, s_facet_capsule(p, float3(0.31, -0.59, zc + 0.01), float3(0.31, -0.45, zc + 0.04), 0.08, 0.5)); - d = min(d, s_facet_capsule(p, float3(0.435, -0.625, zc + 0.005), float3(0.435, -0.52, zc + 0.03), 0.07, 0.2)); - return min(d, s_facet_capsule(p, float3(0.03, -0.77, zc + 0.03), float3(-0.165, -0.60, zc + 0.065), 0.082, 0.7)); + float half_t = model_thick() * 0.5; + float sil = prism_poly_dist(p.xy, PRISM_SIL_START[shape], PRISM_SIL_COUNT[shape]); + float2 w = float2(sil + PRISM_BEVEL, abs(p.z + half_t) - (half_t - PRISM_BEVEL)); + return min(max(w.x, w.y), 0.0) + length(max(w, 0.0)) - PRISM_BEVEL; } -// Le modèle `shape` du thème `theme`, repère du prototype : (distance, id de matière). -float2 sculpt_proto(float3 p, int theme, int shape) +// Distance le long de (ro, rd) au triangle `k` (Möller–Trumbore), -1 s'il le rate. Un rien de jeu +// sur les bords : un rayon ne passe pas entre deux facettes voisines. +float prism_tri(float3 ro, float3 rd, int k) { - if (theme == 3) + float3 v0 = PRISM_TRIS[4 * k].xyz; + float3 e1 = PRISM_TRIS[4 * k + 1].xyz; + float3 e2 = PRISM_TRIS[4 * k + 2].xyz; + float3 pv = cross(rd, e2); + float det = dot(e1, pv); + if (abs(det) < 1e-10) { - return s_voxels(p, shape); + return -1.0; } - if (theme == 1) + float inv = 1.0 / det; + float3 sv = ro - v0; + float u = dot(sv, pv) * inv; + float3 qv = cross(sv, e1); + float v = dot(rd, qv) * inv; + if (u < -2e-4 || v < -2e-4 || u + v > 1.0004) { - return float2(shape == 0 ? s_gem_arrow(p) : s_crystal_hand(p), 8.0); + return -1.0; } - float2 body; - float3 star; - if (shape == 0) + return dot(e2, qv) * inv; +} + +// Le triangle du cristal `shape` le plus proche devant (ro, rd) : sa distance (1e9 si aucun), son +// indice dans `hit` (-1 si aucun). Les triangles sont rangés par voisinage, une boîte englobante +// par groupe (PRISM_BOXES) : un rayon qui rate une boîte, ou n'y entre qu'au-delà du meilleur +// point, saute tous ses triangles. L'inverse de la direction sert à toutes les boîtes. +float prism_trace(float3 ro, float3 rd, int shape, out int hit) +{ + float3 inv = 1.0 / (abs(rd) > 1e-6 ? rd : 1e-6); + int b0 = PRISM_BOX_START[shape]; + int nb = PRISM_BOX_COUNT[shape]; + float best = 1e9; + hit = -1; + [loop] for (int j = b0; j < b0 + nb; j++) { - float h = 0.075, re = 0.03, dome = 0.0; - if (theme == 2) + float4 lo = PRISM_BOXES[2 * j]; + float4 hi = PRISM_BOXES[2 * j + 1]; + float3 t0 = (lo.xyz - ro) * inv; + float3 t1 = (hi.xyz - ro) * inv; + float3 tn = min(t0, t1); + float3 tf = max(t0, t1); + float tmin = max(max(tn.x, tn.y), tn.z); + float tmax = min(min(tf.x, tf.y), tf.z); + if (tmax < max(tmin, 1e-5) || tmin > best) { - h = 0.07; - re = 0.06; - dome = 0.035; + continue; } - if (theme == 4) + int k0 = (int)lo.w; + int n = (int)hi.w; + [loop] for (int i = k0; i < k0 + n; i++) { - h = 0.07; - re = 0.055; - dome = 0.025; - } - float2 r = float2(s_arrow_solid(p, h, re, dome), 1.0); - if (theme == 0) - { - r = s_opu(r, float2(s_piping(p, SCULPT_HOVER + 2.0 * h - 0.004), 2.0)); + float t = prism_tri(ro, rd, i); + if (t > 1e-5 && t < best) + { + best = t; + hit = i; + } } - body = r; - star = float3(0.57, -0.86, SCULPT_HOVER + 2.0 * h + dome); } - else + return best; +} + +// Le modèle `shape` du thème `theme`, repère du prototype : (distance, id de matière). +float2 sculpt_proto(float3 p, int theme, int shape) +{ + if (theme == 3) { - body = s_opu(float2(s_glove(p, SCULPT_HAND_ZC), 1.0), float2(s_cuff(p, SCULPT_HAND_ZC), 2.0)); - star = float3(0.185, -0.93, SCULPT_HAND_ZC + 0.15); + return s_voxels(p, shape); } - // L'étoile de Star Sprout : un seul appel pour les deux formes (FXC recopie chaque appel). - return theme == 4 ? s_opu(body, s_sprout(p, star)) : body; + // Un seul appel pour les trois thèmes cerclés et leurs deux formes (FXC recopie chaque appel). + return s_rimmed(p, theme, shape); } // Repère du modèle -> prototype. L'écrasement du clic ne raccourcit que z, autour du hotspot. @@ -1005,23 +2052,28 @@ float sculpt_units() // Le curseur sculpté en `q` (repère du modèle) : distance et id de matière. `occ` : ce qui porte // l'ombre sur l'écran. La borne des plans d'une gemme et le champ des voxels sont de mauvaises // distances loin de la surface : la pénombre s'y strie, ou s'arrête net au bord de la boîte des -// voxels. Le cristal porte donc l'ombre de la forme lisse de Pop Coral, les voxels celle de leur -// contour extrudé sur toute leur hauteur. +// voxels. Pixel Candy porte donc l'ombre de sa forme, extrudée sur la hauteur des cubes. Prism +// Glow n'a ici que son serti, dans le repère du modèle : son cristal est tracé à part +// (`prism_primary`), et l'ombre est celle de toute sa silhouette. float2 sculpt_eval(float3 q, bool occ) { int id = sculpt_id() - 1; int theme = id / 2, shape = id % 2; + if (theme == 1) + { + return float2(prism_rim(q, shape), 8.0); + } float3 p = sculpt_point(q); float2 r; if (occ && theme == 3) { - float d2 = shape == 0 ? s_arrow2(p.xy) - 1.2 * SCULPT_VOX : s_pixel_hand_dist(p.xy); + float d2 = s_pixel_outline(p.xy, shape); float dz = abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07; r = float2(length(max(float2(d2, dz), 0.0)) + min(max(d2, dz), 0.0), 7.0); } else { - r = sculpt_proto(p, occ && theme == 1 ? 2 : theme, shape); + r = sculpt_proto(p, theme, shape); } return float2(r.x * sculpt_units(), r.y); } @@ -1138,20 +2190,16 @@ SculptMat s_mat(float3 alb, float rough, float spec, float sss, float refl) return m; } -// Couleur de face du voxel qui porte `p` (prototype) : menthe où le bord regarde en bas à gauche, -// rose pâle où il regarde en haut à droite, rose dedans. +// Couleur du cube qui porte `p` (prototype) : celle de son pixel. float3 s_pixel_colour(float3 p, int shape) { - float2 id = floor((p.xy - s_grid_origin(shape)) / SCULPT_VOX); - if (!s_occ(id + float2(-1.0, 0.0), shape) || !s_occ(id + float2(0.0, -1.0), shape)) - { - return s_lin(0.52, 0.91, 0.77); - } - if (!s_occ(id + float2(1.0, 0.0), shape) || !s_occ(id + float2(0.0, 1.0), shape)) - { - return s_lin(1.0, 0.78, 0.87); - } - return s_lin(1.0, 0.50, 0.71); + int2 cell = s_pix_cell(p.xy, shape); + int4 g = PIX_GRID[shape]; + int row = g.z + clamp(cell.y, 0, g.y - 1); + if (s_pix_bit(PIX_LINE[row], cell.x)) return s_lin(0.29, 0.12, 0.36); + if (s_pix_bit(PIX_HI[row], cell.x)) return s_lin(1.0, 0.78, 0.87); + if (s_pix_bit(PIX_SHADE[row], cell.x)) return s_lin(0.87, 0.27, 0.51); + return s_lin(1.0, 0.435, 0.66); } // La matière `mat` (id de `sculpt_proto`) au point `p` du prototype. @@ -1160,28 +2208,36 @@ SculptMat sculpt_material(float mat, float3 p, int theme, int shape) bool primary = mat < 1.5; if (theme == 0) { - if (shape == 0 && primary) return s_mat(s_lin(0.10, 0.10, 0.115), 0.3, 0.2, 0.0, 0.9); - if (shape == 0) return s_mat(s_lin(0.94, 0.91, 0.84), 0.45, 0.4, 0.2, 0.3); - if (primary) return s_mat(s_lin(0.95, 0.92, 0.85), 0.55, 0.35, 0.35, 0.25); - return s_mat(s_lin(0.17, 0.18, 0.22), 0.35, 0.6, 0.0, 0.6); + // Ivoire (le gant, la bande de la flèche) ; noir satiné, presque sans reflet : un reflet + // large sur le champ plat de la flèche le virait au gris. + if (mat < 2.5) return s_mat(s_lin(0.95, 0.92, 0.85), 0.3, 0.6, 0.35, 0.25); + return s_mat(s_lin(0.1, 0.1, 0.11), 0.55, 0.05, 0.0, 0.1); + } + if (theme == 1) + { + // Le serti de Prism Glow : un émail marine, brillant sur son arrondi seulement (`model_shade`). + return s_mat(s_lin(0.02, 0.05, 0.33), 0.35, 0.5, 0.05, 0.3); } if (theme == 2) { - if (shape == 0) return s_mat(s_lin(1.0, 0.40, 0.30), 0.5, 0.45, 0.4, 0.25); - if (primary) return s_mat(s_lin(1.0, 0.79, 0.16), 0.5, 0.45, 0.4, 0.25); - return s_mat(s_lin(0.18, 0.20, 0.29), 0.4, 0.5, 0.0, 0.4); + // Corail : la flèche, le calque et les tirets de la main ; jaune : le gant, le calque et les + // tirets de la flèche ; le trait marine. + if ((primary && shape == 0) || (mat > 2.5 && mat < 3.5 && shape == 1)) return s_mat(s_lin(1.0, 0.40, 0.30), 0.85, 0.08, 0.15, 0.03); + if (mat < 3.5) return s_mat(s_lin(1.0, 0.80, 0.10), 0.85, 0.08, 0.15, 0.03); + return s_mat(s_lin(0.09, 0.13, 0.45), 0.9, 0.05, 0.05, 0.02); } if (theme == 4) { - if (primary && shape == 0) return s_mat(s_lin(0.62, 0.91, 0.78), 0.22, 0.8, 0.25, 0.6); - if (primary) return s_mat(s_lin(0.96, 0.94, 0.88), 0.5, 0.35, 0.35, 0.25); - if (mat < 2.5) return s_mat(s_lin(0.62, 0.91, 0.78), 0.25, 0.7, 0.25, 0.5); - if (mat < 3.5) return s_mat(s_lin(1.0, 0.80, 0.20), 0.3, 0.6, 0.3, 0.4); - if (mat < 4.5) return s_mat(s_lin(0.38, 0.80, 0.55), 0.35, 0.5, 0.35, 0.3); - return s_mat(s_lin(0.16, 0.12, 0.10), 0.2, 0.8, 0.0, 0.5); + // Coussins laqués ; le marine satiné, un reflet large et sombre plutôt qu'un filet blanc. + if (primary && shape == 0) return s_mat(s_lin(0.68, 0.93, 0.80), 0.25, 0.7, 0.3, 0.3); + if (primary) return s_mat(s_lin(0.97, 0.95, 0.90), 0.3, 0.6, 0.35, 0.25); + if (mat < 2.5) return s_mat(s_lin(0.52, 0.87, 0.78), 0.25, 0.7, 0.3, 0.3); + if (mat < 3.5) return s_mat(s_lin(1.0, 0.75, 0.25), 0.25, 0.7, 0.3, 0.3); + if (mat < 4.5) return s_mat(s_lin(0.62, 0.92, 0.72), 0.3, 0.6, 0.3, 0.35); + return s_mat(s_lin(0.07, 0.15, 0.33), 0.7, 0.15, 0.05, 0.08); } - if (mat < 6.5) return s_mat(s_pixel_colour(p, shape), 0.45, 0.35, 0.15, 0.2); - return s_mat(s_lin(0.36, 0.18, 0.54), 0.45, 0.35, 0.1, 0.2); + // Plastique mat : sur les faces plates des cubes, un reflet de la lampe blanchirait le prune. + return s_mat(s_pixel_colour(p, shape), 0.65, 0.1, 0.12, 0.04); } // L'environnement du studio vu du modèle : l'écran dessous, la pièce au-dessus (z du modèle), une @@ -1197,39 +2253,6 @@ float3 model_env(float3 d, float rough, float3 l, float3 fill) return col; } -// Cyan, bleu, violet, orchidée. -float3 s_gem(float k) -{ - k = saturate(k) * 3.0; - float3 a = s_lin(0.20, 0.95, 1.0); - float3 b = s_lin(0.15, 0.42, 1.0); - float3 c = s_lin(0.45, 0.25, 0.95); - float3 d = s_lin(0.88, 0.50, 1.0); - if (k < 1.0) return lerp(a, b, k); - if (k < 2.0) return lerp(b, c, k - 1.0); - return lerp(c, d, k - 2.0); -} - -// Le cristal : la teinte vient d'où la facette regarde et d'où elle plie la vue (lues y vers le -// haut, comme au prototype) ; la dispersion sépare le décalage par canal. Les facettes tournées -// vers la lampe sont cyan, les autres violettes. -float3 model_shade_crystal(float3 n, float3 rd, float3 L, float fall, float3 l, float3 fill) -{ - float cosi = saturate(dot(-rd, n)); - float F = 0.04 + 0.96 * pow(1.0 - cosi, 5.0); - float3 t = refract(rd, n, 1.0 / 1.6); - float k = 0.42 + 0.9 * dot(float2(n.x, -n.y), float2(0.7557, -0.6549)) - + 0.6 * dot(float2(t.x, -t.y), float2(0.6, -0.8)); - float3 body = float3(s_gem(k - 0.08).r, s_gem(k).g, s_gem(k + 0.08).b); - float3 col = body * (0.1 + 1.8 * fall * pow(max(dot(n, L), 0.0), 2.5)); - // Éclat : la vue rebondit sur le dos plat et allume la facette quand elle trouve la lampe. - col += body * model_env(reflect(t, float3(0.0, 0.0, 1.0)) * float3(1.0, 1.0, -1.0), 0.15, l, fill) * 0.5; - col += pow(max(dot(reflect(rd, n), L), 0.0), 30.0) * 2.0; - col += model_env(reflect(rd, n), 0.05, l, fill) * F; - col += s_lin(0.5, 0.9, 1.0) * pow(1.0 - cosi, 4.0) * 0.6; - return col; -} - // Khronos PBR Neutral : garde les teintes, ne comprime que les hautes lumières ; puis sRGB. float3 model_tonemap(float3 c) { @@ -1260,10 +2283,6 @@ float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, float sh float3 l, float3 fill) { int id = sculpt_id(); - if (id > 0 && (id - 1) / 2 == 1) - { - return model_tonemap(model_shade_crystal(n, rd, L, fall, l, fill)); - } SculptMat m; // Sur un sprite, reflets et brillance sur les arrondis seulement : le dessus plat d'un sprite // sombre virerait au gris sous la lampe. @@ -1273,6 +2292,12 @@ float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, float sh // La matière d'un voxel se lit juste sous la surface : son flanc lit sa propre cellule. float3 p = sculpt_point(q) - float3(n.x, -n.y, n.z) * 0.01; m = sculpt_material(mat, p, (id - 1) / 2, (id - 1) % 2); + if (id == 3 || id == 4) + { + // Le serti de Prism Glow, comme un sprite : son dessus plat, face à la lampe, prendrait + // tout son reflet et virerait au gris. + gloss = 1.0 - smoothstep(0.97, 0.995, abs(n.z)); + } } else { @@ -1298,6 +2323,177 @@ float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, float sh return model_tonemap(col); } +// ---- Le cristal de Prism Glow ---- + +float prism_fresnel(float c, float ior) +{ + float f0 = (ior - 1.0) / (ior + 1.0); + f0 *= f0; + return f0 + (1.0 - f0) * pow(1.0 - c, 5.0); +} + +// Le cristal sur le rayon (ro, rd) : distance au point touché (1e9 si raté ou hors de Prism Glow), +// facette dans `hc`. Une paroi sous le dessus du serti ne compte pas : le rayon a traversé le +// serti avant, c'est lui qu'on voit. +float prism_primary(float3 ro, float3 rd, out int hc) +{ + hc = -1; + int shape = prism_shape(); + if (shape < 0) + { + return 1e9; + } + float t = prism_trace(ro, rd, shape, hc); + if (hc >= 0 && fmod(PRISM_TRIS[4 * hc].w, 2.0) > 0.5 && ro.z + rd.z * t < 0.0) + { + hc = -1; + return 1e9; + } + return t; +} + +// La normale de la facette `h`, tournée vers le rayon qui arrive le long de `rd`. +float3 prism_normal(int h, float3 rd) +{ + float3 n = PRISM_TRIS[4 * h + 3].xyz; + return dot(n, rd) > 0.0 ? -n : n; +} + +// Distance de `p` au segment [a, b]. +float prism_seg(float3 p, float3 a, float3 b) +{ + float3 pa = p - a, ba = b - a; + return length(pa - ba * saturate(dot(pa, ba) / dot(ba, ba))); +} + +// Distance, en pixels (`px` = unités du modèle par pixel au point), de `p` aux arêtes réelles de +// la facette `h` : les plis de l'art et le bord du serti. +float prism_edge_px(float3 p, int h, float px) +{ + float4 r0 = PRISM_TRIS[4 * h]; + float3 a = r0.xyz; + float3 b = a + PRISM_TRIS[4 * h + 1].xyz; + float3 c = a + PRISM_TRIS[4 * h + 2].xyz; + int bits = (int)(r0.w * 0.5); + float d = 1e9; + if ((bits & 1) != 0) d = min(d, prism_seg(p, b, c)); + if ((bits & 2) != 0) d = min(d, prism_seg(p, c, a)); + if ((bits & 4) != 0) d = min(d, prism_seg(p, a, b)); + return d / px; +} + +// La rotation inverse de `world_to_plane` : du repère du plan à celui de la caméra. +float3 plane_to_world(float3 v, ModelFrame f) +{ + float x = v.x * f.c.z - v.y * f.s.z; + float y = v.x * f.s.z + v.y * f.c.z; + float z = -x * f.s.y + v.z * f.c.y; + return float3(x * f.c.y + v.z * f.s.y, y * f.c.x - z * f.s.x, y * f.s.x + z * f.c.x); +} + +// Ce que montre l'image là où le rayon (q, d) du modèle retombe sur le plan : la copie de l'image +// composée faite juste avant le curseur (t5), flous de confidentialité compris, lue au pixel de +// sortie de ce point, en linéaire. Seuls des pixels déjà floutés passent donc à travers le verre. +// L'œil est en (-mb.zw, persp) du repère caméra, le rayon du pixel `local` y va vers +// (local + src.xy, -persp) : on remonte de ce point du plan à son `local`, puis à son uv de sortie. +// Hors de l'image, l'écran sombre du studio. +float3 prism_screen(float3 q, float3 d, float3 nz, float hz, float3 tip, float unit, ModelFrame f) +{ + float denom = dot(d, nz); + if (denom > -1e-4) + { + return SCULPT_SCREEN * 0.2; + } + float3 g = q + d * ((hz - dot(q, nz)) / denom); + float3 w = plane_to_world(tip + unit * model_to_plane(g, f), f); + float k = 1.0 - w.z / src.z; + if (k < 1e-3) + { + return SCULPT_SCREEN * 0.2; + } + float2 uv = dst.xy + ((w.xy + mb.zw) / k - src.xy) / quad_px * dst.zw; + if (any(uv < 0.0) || any(uv > 1.0)) + { + return SCULPT_SCREEN * 0.2; + } + return pow(texDof.SampleLevel(samp, uv, 0.0).rgb, 2.2); +} + +// La couleur (sRGB, comme `model_shade`) du cristal au point `p` de la facette `h`, vu le long de +// `rd` : reflet du studio, l'enregistrement réfracté par canal (la dispersion) à travers les +// facettes et le fond plat, la lueur de la couleur de la facette dans l'art, ses plis en liseré. +float3 prism_shade(float3 p, float3 n, float3 rd, int h, float px, float3 l, float3 fill, float3 nz, + float hz, float3 tip, float unit, ModelFrame f) +{ + int shape = prism_shape(); + float4 r3 = PRISM_TRIS[4 * h + 3]; + float3 facet = float3(PRISM_TRIS[4 * h + 1].w, PRISM_TRIS[4 * h + 2].w, r3.w); + float F = prism_fresnel(saturate(-dot(rd, n)), PRISM_IOR); + float zb = -model_thick(); + float3 trans = 0.0; + [loop] for (int ch = 0; ch < 3; ch++) + { + // Le rouge plie le moins, le bleu le plus. FXC n'indexe pas un vecteur par une variable en + // écriture (il déroulerait la boucle) : le canal passe par un masque. + float3 mask = float3(ch == 0, ch == 1, ch == 2); + float ior = PRISM_IOR + PRISM_DISPERSION * (ch - 1); + float3 d = refract(rd, n, 1.0 / ior); + float3 pos = p; + float thr = 1.0; + float acc = 0.0; + [loop] for (int b = 0; b < PRISM_BOUNCES; b++) + { + int hh; + float th = prism_trace(pos, d, shape, hh); + float tb = d.z < -1e-6 ? (zb - pos.z) / d.z : 1e9; + if (tb <= th) + { + // Le fond plat : le rayon sort vers l'écran, ou s'y réfléchit tout entier. + float3 qb = pos + d * tb; + float3 d2 = refract(d, float3(0.0, 0.0, 1.0), ior); + if (dot(d2, d2) < 1e-8) + { + pos = qb; + d = reflect(d, float3(0.0, 0.0, 1.0)); + continue; + } + acc += thr * dot(prism_screen(qb, d2, nz, hz, tip, unit, f), mask); + thr = 0.0; + break; + } + if (hh < 0) + { + break; + } + pos += d * th; + float4 rn = PRISM_TRIS[4 * hh + 3]; + if (fmod(PRISM_TRIS[4 * hh].w, 2.0) > 0.5 && pos.z < 0.0) + { + // Une paroi sous le dessus du serti : le marine de son flanc intérieur. + acc += thr * dot(s_lin(0.03, 0.06, 0.37) * 0.35, mask); + thr = 0.0; + break; + } + float3 dout = refract(d, -rn.xyz, ior); + if (dot(dout, dout) < 1e-8) + { + d = reflect(d, -rn.xyz); + continue; + } + float fi = prism_fresnel(saturate(dot(dout, rn.xyz)), ior); + acc += thr * (1.0 - fi) * dot(model_env(dout, 0.05, l, fill), mask); + thr *= fi; + d = reflect(d, -rn.xyz); + } + trans += acc * mask; + } + float3 col = F * model_env(reflect(rd, n), 0.05, l, fill) + (1.0 - F) * trans + facet * PRISM_GLOW; + // Les plis de l'art : un liseré clair sur chaque arête réelle, fin d'un pixel. + float fold = 1.0 - smoothstep(0.2, 1.0, prism_edge_px(p, h, px)); + col = lerp(col, s_lin(0.96, 0.99, 1.0) * 1.3, PRISM_FOLD * fold * (0.35 + 0.65 * saturate(dot(n, l)))); + return model_tonemap(col); +} + // Les passes de la boucle unique de `cursor_model`, dans l'ordre. Chaque tour n'évalue le modèle // QU'UNE fois (`model_eval`) : FXC recopie une fonction à chaque appel, et un curseur sculpté est // assez gros pour qu'un appel par passe multiplie par neuf le temps de compilation de ps_main. @@ -1306,7 +2502,12 @@ static const int STAGE_NORMAL = 1; // quatre sondes en tétraèdre autour du poi static const int STAGE_AO = 2; // cinq sondes le long de la normale, jusqu'à 0,08 du prototype static const int STAGE_SELF = 3; // pénombre du modèle sur lui-même, vers la lampe (32 pas) static const int STAGE_PLANE = 4; // le point du plan : son contact, puis sa pénombre (32 pas) -static const int STAGE_DONE = 5; +static const int STAGE_EDGE = 5; // quatre sondes à un demi-pixel du point touché : un bord ? +static const int STAGE_SHADE = 6; // l'ombrage d'un échantillon, puis le rayon suivant s'il en reste +static const int STAGE_DONE = 7; + +// Les trois rayons de plus d'un pixel au bord, en pixels autour de son centre. +static const float2 MODEL_SUBPIXEL[3] = { float2(0.35, 0.2), float2(-0.35, 0.2), float2(0.0, -0.4) }; // Sonde `k` de la normale, sommet d'un tétraèdre. float3 model_tetra(int k) @@ -1347,17 +2548,30 @@ float4 cursor_model(float2 local) float3 lamp = float3(color.rg + sprite_size() * 0.5, 0.0) + l * SCULPT_LAMP_DIST; // Silhouette antialiasée : un rayon qui frôle le modèle à moins d'un pixel le couvre en partie - // (`best`, la plus petite distance rencontrée, en pixels). - float2 tb = ray_box(ro, rd, lo - 0.02, hi + 0.02); + // (`best`, la plus petite distance rencontrée, en pixels). Les bords intérieurs (deux matières, + // une arête, un arrondi serré) sont suréchantillonnés : `STAGE_EDGE` sonde le modèle à un + // demi-pixel du point touché, dans son plan tangent ; si la matière change ou que la surface + // s'écarte, trois rayons de plus (`MODEL_SUBPIXEL`) sont marchés et ombrés, avec l'occlusion et + // les ombres du premier. Chaque échantillon passe par `STAGE_SHADE`, seul appel de + // `model_shade` ; le pixel est leur moyenne. + float3 ray = rd; + float2 tb = ray_box(ro, ray, lo - 0.02, hi + 0.02); int stage = tb.x < tb.y && tb.y > 0.0 ? STAGE_MARCH : STAGE_DONE; + // Prism Glow : le cristal est tracé au départ de chaque rayon ; `tc` est + // son point touché (1e9 : raté), `hc` sa facette. La marche du serti s'arrête à lui. + bool need_tc = stage == STAGE_MARCH; + float tc = 1e9; + int hc = -1; // Le point du plan derrière le pixel est à régler avant le prochain tour. bool plane_next = stage == STAGE_DONE; int k = 0; float t = max(tb.x, 0.0); float best = 1e9; float t_best = t; + float d_best = 0.0; float mat = 0.0; float cov = 0.0; + float cov_s = 0.0; float3 q = 0.0; float3 n = 0.0; float3 L = 0.0; @@ -1371,14 +2585,24 @@ float4 cursor_model(float2 local) float inside = 0.0; float contact = 0.0; float dropped = 0.0; - [loop] for (int it = 0; it < 180; it++) + // Sous-échantillons restants, échantillons ombrés, et leur somme (couleur × couverture, + // couverture). + int sub = 0; + int shaded = 0; + float4 acc = 0.0; + [loop] for (int it = 0; it < 500; it++) { + if (need_tc) + { + need_tc = false; + tc = prism_primary(ro, ray, hc); + } if (plane_next) { // Le plan, là où le modèle ne couvre pas tout le pixel : ombre portée et ombre de // contact, seulement à l'intérieur de l'écran (`mb.xy` = sa demi-taille, px du plan). plane_next = false; - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; float denom = dot(rd, nz); if (cov < 1.0 && denom < -1e-4) { @@ -1393,15 +2617,63 @@ float4 cursor_model(float2 local) } } } + if (stage == STAGE_SHADE) + { + if (cov_s > 0.0) + { + float3 c; + if (mat > PRISM_MAT - 0.5) + { + c = prism_shade(q, n, ray, hc, t_best / dlen, l, fill, nz, hz, tip, unit, f); + } + else + { + float3 tl = lamp - q; + float fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); + c = model_shade(q, n, ray, normalize(tl), fall, sh, ao, mat, l, fill); + } + acc += float4(c * cov_s, cov_s); + } + shaded++; + stage = STAGE_DONE; + if (sub > 0) + { + // Le rayon suivant, décalé dans le pixel ; il repart de sa propre entrée dans la + // boîte, pour ne pas manquer ce qui passerait devant le premier. + float3 dws = float3(local + MODEL_SUBPIXEL[3 - sub] + src.xy, -persp); + sub--; + dlen = length(dws); + ray = plane_to_model(world_to_plane(dws / dlen, f), f); + tb = ray_box(ro, ray, lo - 0.02, hi + 0.02); + cov_s = 0.0; + stage = STAGE_SHADE; + if (tb.x < tb.y && tb.y > 0.0) + { + stage = STAGE_MARCH; + k = 0; + t = max(tb.x, 0.0); + best = 1e9; + t_best = t; + need_tc = true; + } + } + continue; + } float3 pos; if (stage == STAGE_MARCH) { - pos = ro + rd * t; + pos = ro + ray * t; } else if (stage == STAGE_NORMAL) { pos = q + 0.002 * model_tetra(k); } + else if (stage == STAGE_EDGE) + { + float3 side = normalize(cross(n, ray)); + float3 e = (k >= 2 ? cross(n, side) : side) * (k % 2 == 1 ? -1.0 : 1.0); + pos = q + e * 0.5 * t_best / dlen; + } else if (stage == STAGE_AO) { pos = q + (0.01 + 0.0175 * k) * SCULPT_SCALE * n; @@ -1428,22 +2700,68 @@ float4 cursor_model(float2 local) { best = hit ? 0.0 : d / fp; t_best = t; + d_best = d; mat = m.y; } t += d * stride; k++; + if (!hit && t >= tc) + { + // Le pas dépasse le cristal : c'est lui que le rayon touche, avant le serti. + hit = true; + best = 0.0; + t_best = tc; + d_best = 0.0; + mat = PRISM_MAT; + } if (hit || t > tb.y || k == 96) { - cov = saturate(1.0 - best); - if (cov > 0.0) + cov_s = saturate(1.0 - best); + if (shaded > 0) { - q = ro + rd * t_best; - stage = STAGE_NORMAL; - k = 0; + stage = STAGE_SHADE; + if (cov_s > 0.0) + { + q = ro + ray * t_best; + n = 0.0; + stage = STAGE_NORMAL; + k = 0; + } } else { - plane_next = true; + cov = cov_s; + if (cov > 0.0) + { + q = ro + ray * t_best; + stage = STAGE_NORMAL; + k = 0; + } + else + { + plane_next = true; + } + } + if (stage == STAGE_NORMAL && mat > PRISM_MAT - 0.5) + { + // Le cristal : la normale de sa facette, sans sondes ; ni occlusion ni ombre + // propre. Un bord réel à moins d'un pixel (le serti, un pli) appelle les trois + // rayons de plus, comme les bords intérieurs des autres modèles. + n = prism_normal(hc, ray); + k = 0; + if (shaded > 0) + { + stage = STAGE_SHADE; + } + else + { + if (prism_edge_px(q, hc, t_best / dlen) < 0.75) + { + sub = 3; + } + stage = STAGE_DONE; + plane_next = true; + } } } } @@ -1454,6 +2772,34 @@ float4 cursor_model(float2 local) if (k == 4) { n = normalize(n); + k = 0; + stage = shaded > 0 ? STAGE_SHADE : STAGE_EDGE; + } + } + else if (stage == STAGE_EDGE) + { + // Autre matière, ou surface qui s'écarte du plan tangent de plus de 2 % d'un pixel + // (le point touché flotte déjà de `d_best` au-dessus d'elle). + if (m.y != mat || abs(d - d_best) > 0.02 * t_best / dlen) + { + sub = 3; + } + if (k == 0 && mat > 7.5) + { + // Le serti de Prism Glow au bord du cristal : son contour à moins d'un pixel. + int ps = prism_shape(); + if (ps >= 0) + { + float dout = prism_poly_dist(q.xy, PRISM_OUT_START[ps], PRISM_OUT_COUNT[ps]); + if (abs(dout) < 0.75 * t_best / dlen) + { + sub = 3; + } + } + } + k++; + if (k == 4) + { stage = STAGE_AO; k = 0; ao = 0.0; @@ -1504,7 +2850,7 @@ float4 cursor_model(float2 local) k = 1; if (!(tbp.x < tbp.y && tbp.y > 0.0)) { - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; } } else @@ -1516,21 +2862,15 @@ float4 cursor_model(float2 local) { res = saturate(res); dropped = 1.0 - res * res * (3.0 - 2.0 * res); - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; } } } - float3 rgb = 0.0; - if (cov > 0.0) - { - float3 tl = lamp - q; - float fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); - rgb = model_shade(q, n, rd, normalize(tl), fall, sh, ao, mat, l, fill); - } + float w = 1.0 / max(shaded, 1); float shadow = inside * max(dropped * MODEL_SHADOW_ALPHA, contact * MODEL_CONTACT_ALPHA); - float a = cov * color.a; - return float4(rgb * a, a + (1.0 - a) * shadow * color.a); // prémultiplié, ombre noire + float a = acc.a * w * color.a; + return float4(acc.rgb * w * color.a, a + (1.0 - a) * shadow * color.a); // prémultiplié, ombre noire } // ============ Impact du clic (mode 16) ============ @@ -2154,7 +3494,8 @@ float4 device_frame(float2 local) return float4(rgb * a, a); // prémultiplié } -float4 ps_main(VSOut i) : SV_Target +// The body of `ps_main`, which only adds the layer's transparency on top of it. +float4 ps_layer(VSOut i) { // mode 18 : l'écran CADRÉ (ombre, cadre, métrage, appareil) flouté comme UN objet rigide // (`FrameGeometry::screen_trail`). t2 = son rendu isolé, prémultiplié, à la taille de la @@ -2213,6 +3554,7 @@ float4 ps_main(VSOut i) : SV_Target return acc / max(n, 1.0); } +#ifdef LAYER_MODELS // mode 17 : CADRE D'APPAREIL MODELÉ (cf. `device_frame`). Testé en premier, comme le 16. if (mode > 16.5) { @@ -2236,6 +3578,14 @@ float4 ps_main(VSOut i) : SV_Target } return cursor_model(i.local); } +#else + // Les modèles 3D n'existent que dans `ps_main_models` ; un draw qui les enverrait ici ne + // peint rien plutôt qu'une ombre faite de leurs emplacements. + if (mode > 14.5) + { + return float4(0.0, 0.0, 0.0, 0.0); + } +#endif // mode 14 : CADRE DE FENÊTRE autour de l'écran (barre de titre, trois pastilles, filet), // dessiné SOUS lui. Testé avant le mode 13, dont la branche n'a pas de borne haute. @@ -2490,8 +3840,12 @@ float4 ps_main(VSOut i) : SV_Target float2 plane_px = dst_prev.xy; float2 p = float2(r.x, r.y) * plane_px - plane_px * 0.5; float rad = max(radius_px, 0.0); - float d = (dst_prev.z > 0.5) ? sd_screen_under_bar(p, plane_px * 0.5, rad, color.z) - : sd_round_rect(p, plane_px * 0.5, rad); + // Une branche et non `?:`, qui évalue ses DEUX opérandes (trois SDF par pixel). + float d; + if (dst_prev.z > 0.5) + d = sd_screen_under_bar(p, plane_px * 0.5, rad, color.z); + else + d = sd_round_rect(p, plane_px * 0.5, rad); float tilt_a = 1.0 - smoothstep(0.0, 1.5, d); // Slot d'un layout en bloc (`color.w` = rayon de ses coins, px ; 0 ailleurs, sans effet) : // `dst` EST le slot, dont le rect arrondi rogne le plan. Le conteneur masque, le métrage @@ -2653,6 +4007,8 @@ float4 ps_main(VSOut i) : SV_Target float3 rgb; // 1 sauf en mode detourage, ou il porte le masque du sujet (cf. la branche fx.z ci-dessous). float alpha_mask = 1.0; + // 0 where a camera homography finds no camera point (`persp_camera`), 1 everywhere else. + float persp_keep = 1.0; if (mode < 0.5) { // flou de mouvement par vélocité (§8) : pour CE pixel sortie, uv à la frame @@ -2661,6 +4017,17 @@ float4 ps_main(VSOut i) : SV_Target float2 uv_now = i.uv; float2 localp = (i.pout - dst_prev.xy) / dst_prev.zw; float2 uv_prev = src_prev.xy + localp * (src_prev.zw - src_prev.xy); + if (layer_fx.y > 0.5) + { + // Camera homography: the texture uv comes from the fragment's place in the quad + // (`src` is not used), scaled to the valid part of the decoder texture. The previous + // frame's uv is the same map at the quad's previous place. + float3 cam = persp_camera((i.pout - dst.xy) / dst.zw); + persp_keep = cam.z; + uv_now = cam.xy * fx.xy; + float3 cam_prev = persp_camera(localp); + uv_prev = (cam_prev.z > 0.5) ? cam_prev.xy * fx.xy : uv_now; + } float2 duv = uv_now - uv_prev; float mb_scale = saturate(mb.y); float2 duv_blur = duv * mb_scale; @@ -2701,20 +4068,28 @@ float4 ps_main(VSOut i) : SV_Target } else if (effect > 1.5) { - rgb = lerp(blur_webcam_bg(uv_now, fx.w, quad_px, i.local), rgb, person); + rgb = lerp(blur_webcam_bg(uv_now, fx.w, quad_px, i.local, fx.xy), rgb, person); } else { alpha_mask = person; } } + + // Desk-view cover: the camera is being tilted, so the whole picture is blurred and + // dimmed (cover.x = strength, cover.y = radius in quad px, cover.z = dim at full cover). + if (cover.x > 0.001) + { + float3 hidden = blur_webcam_radius(uv_now, cover.y, quad_px, i.local, fx.xy); + rgb = lerp(rgb, hidden, cover.x) * (1.0 - cover.z * cover.x); + } } else { rgb = color.rgb; } - float alpha = color.a * alpha_mask; + float alpha = color.a * alpha_mask * persp_keep; if (radius_px > 0.0) { // `quad_px` est en px de SORTIE (le render target porte la géométrie de sortie) et @@ -2726,13 +4101,24 @@ float4 ps_main(VSOut i) : SV_Target // cadre (`sd_screen_under_bar`, `mb.z` = la remontée du contour intérieur, px). float2 halfsz = quad_px * 0.5; float2 p = i.local - quad_px * 0.5; - float d = (mb.w > 0.5) ? sd_screen_under_bar(p, halfsz, radius_px, mb.z) - : sd_round_rect(p, halfsz, radius_px); + // Une branche et non `?:`, qui évalue ses DEUX opérandes : trois SDF par pixel de l'écran + // au lieu d'une -- 0,6 ms par frame 1080p sur une Radeon 610M (mesuré en WGSL). + float d; + if (mb.w > 0.5) + d = sd_screen_under_bar(p, halfsz, radius_px, mb.z); + else + d = sd_round_rect(p, halfsz, radius_px); alpha *= 1.0 - smoothstep(0.0, 1.5, d); // ~1.5px feather (§7 fwidth-like) } return float4(rgb * alpha, alpha); // prémultiplié } +float4 ps_main(VSOut i) : SV_Target +{ + // Premultiplied, so the transparency scales all four channels; 0 leaves them as they are. + return ps_layer(i) * (1.0 - layer_fx.x); +} + // ============ RGB -> NV12 (§5) : deux passes vers les plans d'une texture NV12 ============ // VS plein écran (triangle unique) qui expose l'UV. struct FSOut { float4 pos : SV_Position; float2 uv : TEXCOORD0; }; diff --git a/crates/compositor/src/shaders.metal b/crates/compositor/src/shaders.metal index 6c019c3ec..b0e2cb40e 100644 --- a/crates/compositor/src/shaders.metal +++ b/crates/compositor/src/shaders.metal @@ -33,6 +33,12 @@ // `compositor_macos::tests::every_shader_entry_point_compiles` le compile sur le device // système au `cargo test`, pour qu'une faute de syntaxe MSL ne se découvre pas à // l'ouverture de l'éditeur chez un utilisateur. +// +// Il est compilé DEUX fois (`compositor_macos.rs::new_sized`) : tel quel pour tous les calques, +// et avec `LAYER_MODELS` défini pour les seuls modèles 3D (modes 15 à 17, +// `LayerCB::needs_models`). Un shader alloue à chaque draw les registres de sa branche la plus +// lourde, et ce sont ces modèles ray-tracés : le même `ps_main` les portait pour le fond, +// l'ombre et la vidéo, qui couvrent toute la sortie (cf. le haut de `shaders.hlsl`). #include using namespace metal; @@ -42,7 +48,7 @@ using namespace metal; // ================================================================================= // // Le moteur côté CPU upload ce buffer via `setVertexBytes` (vertex stage) et -// `setFragmentBytes` (fragment stage) avant chaque draw — la copie est de 176 octets, +// `setFragmentBytes` (fragment stage) avant chaque draw — la copie est de 256 octets, // ce qui est sous le seuil d'alignement 4K de Metal pour le mode « immediate ». struct Layer @@ -60,6 +66,9 @@ struct Layer float4 trail_a; // mode 8 : coins TL, TR du plan à la frame précédente (px locaux, comme fx) ; mode 18 incliné : en fractions de sortie float4 trail_b; // mode 8 : coins BR, BL du plan à la frame précédente (comme src_prev) ; mode 18 incliné : en fractions de sortie float4 trail_mb; // mode 8 : x = taps, y = force du flou de mouvement (ceux du mode 0) ; 0 ailleurs + float4 cover; // x = desk-view cover 0..1, y = blur radius (quad px), z = dim, w unused + float4 persp[3]; // mode 0: rows of the homography quad point (0..1 in dst) -> camera uv (0..1 of the valid frame), xyz; read when layer_fx.y = 1 + float4 layer_fx; // x = transparency 0..1 (0 = opaque), every mode; y = 1 when persp applies; z, w unused }; // Mode 15 (curseur modélisé) : le détail des emplacements est dans `frame_geometry.rs`, en tête // de la section « Curseur modélisé » (`cursor_model_cb`). Mode 17 (appareil modelé) : en tête de @@ -170,9 +179,13 @@ inline float sd_segment(float2 p, float2 a, float2 b) inline float sd_round_rect(float2 p, float2 halfsz, float r) { float hmin = min(halfsz.x, halfsz.y); - if (r <= 0.0 || hmin <= 0.0) + float2 q0 = abs(p) - halfsz; + // Hors des coins, la distance est celle du rectangle vif quel que soit l'exposant : un seul + // axe déborde de l'étendue E, qui s'y ajoute puis s'en retranche. E ne dépasse pas 1,42 r + // (n = 3) : sous 1,5 r du bord sur un axe, on rend la boîte sans calculer `n` ni E, deux + // transcendantes de moins sur presque tous les pixels d'un calque arrondi. + if (r <= 0.0 || hmin <= 0.0 || min(q0.x, q0.y) <= -min(1.5 * r, hmin)) { - float2 q0 = abs(p) - halfsz; return length(max(q0, 0.0)) + min(max(q0.x, q0.y), 0.0); } float n = 3.0 - smoothstep(0.5, 1.0, r / hmin); @@ -414,12 +427,17 @@ constant float3 VOGEL_TAPS[21] = { float3(-0.633036, -0.758588, 0.087119) }; -inline float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 local_px, - texture2d texY, - texture2d texUV) +// Vogel-disc blur of the camera texture at a radius in quad pixels. `valid` is the part of the +// texture the picture fills (fx.xy): decoders allocate aligned textures (a 1080-line camera in a +// 1088-line texture), so each tap is clamped half a chroma texel inside it, never into padding. +inline float3 blur_webcam_radius(float2 uv, float max_r_px, float2 qpx, float2 local_px, + float2 valid, + texture2d texY, + texture2d texUV) { - float max_r_px = max(intensity, 0.0) * 22.0 + 1.5; float2 step = max_r_px / max(qpx, float2(1.0)); + float2 chroma = max(float2(float(texUV.get_width()), float(texUV.get_height())), float2(1.0)); + float2 hi = max(valid - 0.5 / chroma, float2(0.0)); float noise = fract(52.9829189 * fract(0.06711056 * local_px.x + 0.00583715 * local_px.y)); float angle = noise * 6.2831853; float s = sin(angle); @@ -431,12 +449,34 @@ inline float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 loca float2 p = VOGEL_TAPS[k].xy; float w = VOGEL_TAPS[k].z; float2 rot_p = float2(p.x * c - p.y * s, p.x * s + p.y * c); - sum += sample_yuv(saturate(uv + rot_p * step), texY, texUV) * w; + sum += sample_yuv(clamp(uv + rot_p * step, float2(0.0), hi), texY, texUV) * w; total += w; } return sum / max(total, 1e-4); } +// Camera homography (`persp`, read when `layer_fx.y` = 1): the camera point (0..1 of its valid +// frame) seen at `local` (0..1 in the layer's dst quad), and in z whether there is one -- 0 when +// the point is behind the projection (q.z <= 0) or outside the camera frame, where the layer is +// transparent. Mirror of the HLSL `persp_camera`. +inline float3 persp_camera(constant Layer &layer, float2 local) +{ + float3 p = float3(local, 1.0); + float3 q = float3(dot(layer.persp[0].xyz, p), dot(layer.persp[1].xyz, p), dot(layer.persp[2].xyz, p)); + float2 cam = q.xy / max(q.z, 1e-6); + bool inside = q.z > 0.0 && all(cam >= 0.0) && all(cam <= 1.0); + return float3(cam, inside ? 1.0 : 0.0); +} + +inline float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 local_px, + float2 valid, + texture2d texY, + texture2d texUV) +{ + return blur_webcam_radius(uv, max(intensity, 0.0) * 22.0 + 1.5, qpx, local_px, valid, texY, + texUV); +} + // Hash 2D -> [0,1) sans sin(). Miroir de `hash12` côté HLSL. inline float hash12(float2 p) { @@ -545,7 +585,9 @@ inline float3 image_motion(float2 q, float time, float motion, float aspect) // ============ Curseur MODÉLISÉ (mode 15) ============ // Port ligne pour ligne de `cursor_model` (HLSL), dont les commentaires font foi ; seules // différences : `layer` et les textures arrivent en paramètres (le sprite en texture(2), son -// champ R16F en texture(4)), `lerp` s'écrit `mix`, `SampleLevel` s'écrit `sample(…, level(0.0))`, +// champ R16F en texture(4), la copie de l'image composée en texture(5) pour le cristal de Prism +// Glow), +// `lerp` s'écrit `mix`, `SampleLevel` s'écrit `sample(…, level(0.0))`, un `out` s'écrit `thread &`, // et `pow` veut un exposant du type de sa base. // Constantes : miroir exact de `frame_geometry.rs` (MODEL_*) et de `sculpt.rs` (SCULPT_*). constant float MODEL_BEVEL = 0.045; @@ -590,8 +632,6 @@ inline int sculpt_id(constant Layer &layer) constant float SCULPT_SCALE = 0.85; constant float SCULPT_HOVER = 0.05; constant float SCULPT_VOX = 0.0625; -constant float SCULPT_AR_ROUND = 0.03; -constant float SCULPT_HAND_ZC = 0.185; constant float SCULPT_ZREF_ARROW = 0.2; constant float SCULPT_ZREF_HAND = 0.185; constant float SCULPT_LAMP_DIST = 1.9; @@ -613,13 +653,6 @@ inline float2 s_opu(float2 a, float2 b) return a.x < b.x ? a : b; } -static float s_capsule(float3 p, float3 a, float3 b, float r) -{ - float3 pa = p - a, ba = b - a; - float h = saturate(dot(pa, ba) / dot(ba, ba)); - return length(pa - ba * h) - r; -} - static float s_round_box(float3 p, float3 b, float r) { float3 q = abs(p) - b + r; @@ -646,24 +679,76 @@ inline float2 s_rot(float2 v, float a) return float2(c * v.x - s * v.y, s * v.x + c * v.y); } -constant float2 SCULPT_ARROW[7] = { - float2(0.0, 0.0), float2(0.0, -0.86), float2(0.215, -0.665), float2(0.37, -1.0), - float2(0.53, -0.93), float2(0.38, -0.60), float2(0.64, -0.60) +static float s_star5(float2 p, float r, float rf) +{ + const float2 k1 = float2(0.809016994375, -0.587785252292); + const float2 k2 = float2(-0.809016994375, -0.587785252292); + p.x = abs(p.x); + p -= 2.0 * max(dot(k1, p), 0.0) * k1; + p -= 2.0 * max(dot(k2, p), 0.0) * k2; + p.x = abs(p.x); + p.y -= r; + float2 ba = rf * float2(-k1.y, k1.x) - float2(0.0, 1.0); + float h = clamp(dot(p, ba) / dot(ba, ba), 0.0, r); + return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); +} + +static float s_vesica(float2 p, float r, float d) +{ + p = abs(p); + float b = sqrt(r * r - d * d); + return (p.y - b) * d > p.x * b ? length(p - float2(0.0, b)) : length(p + float2(d, 0.0)) - r; +} + +// Curseurs cercles (cf. HLSL) : Studio Ink, Pop Coral, Star Sprout. +constant float2 RIM_POLY[28] = { + float2(0.0, -0.06), float2(0.0, -0.7712), float2(0.165, -0.6325), float2(0.313, -0.9318), + float2(0.4234, -0.8557), float2(0.2672, -0.5686), float2(0.456, -0.5393), + float2(-0.173, -0.5237), float2(-0.016, -0.75), float2(0.329, -0.75), float2(0.411, -0.592), + float2(0.411, -0.43), float2(0.19, -0.435), float2(-0.03, -0.44), + float2(-0.1825, -0.6215), float2(0.0037, -0.9195), float2(0.3462, -0.9195), float2(0.467, -0.6906), + float2(0.467, -0.49), float2(0.2, -0.49), float2(-0.03, -0.49), + float2(-0.1807, -0.5534), float2(0.0194, -0.872), float2(0.3485, -0.872), float2(0.47, -0.6795), + float2(0.47, -0.49), float2(0.2, -0.49), float2(-0.03, -0.49) +}; + +constant float4 RIM_GLOVE[27] = { + float4(0.0, -0.1077, 0.0, -0.6), float4(0.1493, -0.3279, 0.1493, -0.6), + float4(0.2863, -0.3654, 0.2863, -0.6), float4(0.4193, -0.4043, 0.4193, -0.6), + float4(-0.2, -0.478, -0.075, -0.625), float4(0.0788, -0.25, 0.0788, -0.3992), + float4(0.2188, -0.3, 0.2188, -0.4234), float4(0.3552, -0.33, 0.3552, -0.4534), + float4(-0.0845, -0.5386, 0.762, 0.648), + float4(0.0, -0.1186, 0.0, -0.65), float4(0.1614, -0.3838, 0.1614, -0.65), + float4(0.3177, -0.427, 0.3177, -0.65), float4(0.4645, -0.4848, 0.4645, -0.65), + float4(-0.215, -0.535, -0.1208, -0.6864), float4(0.0832, -0.3, 0.0832, -0.48), + float4(0.2414, -0.35, 0.2414, -0.482), float4(0.3959, -0.41, 0.3959, -0.533), + float4(-0.122, -0.5718, 0.867, 0.498), + float4(0.0, -0.122, 0.0, -0.65), float4(0.172, -0.387, 0.172, -0.65), + float4(0.3267, -0.427, 0.3267, -0.65), float4(0.4727, -0.4867, 0.4727, -0.65), + float4(-0.199, -0.483, -0.102, -0.645), float4(0.0927, -0.3, 0.0927, -0.476), + float4(0.2493, -0.35, 0.2493, -0.478), float4(0.402, -0.41, 0.402, -0.52), + float4(-0.0993, -0.5267, 0.858, 0.514) +}; + +constant float4 RIM_GLOVE_R[9] = { + float4(0.0552, 0.047, 0.045, 0.0375), float4(0.052, 0.0235, 0.0225, 0.0225), float4(0.0552, -0.548, 0.0525, 0.0), + float4(0.064, 0.0585, 0.0585, 0.0515), float4(0.06, 0.0197, 0.0178, 0.0172), float4(0.0648, -0.6, 0.058, 0.0), + float4(0.066, 0.0553, 0.0553, 0.0507), float4(0.063, 0.024, 0.022, 0.02), float4(0.0647, -0.555, 0.056, 0.0) }; -static float s_arrow2(float2 p) +static float s_rim_poly(float2 p, int base) { - float d = dot(p - SCULPT_ARROW[0], p - SCULPT_ARROW[0]); + float d = dot(p - RIM_POLY[base], p - RIM_POLY[base]); float s = 1.0; - int j = 6; - for (int i = 0; i < 7; i++) + int j = base + 6; + for (int i = base; i < base + 7; i++) { - float2 e = SCULPT_ARROW[j] - SCULPT_ARROW[i]; - float2 w = p - SCULPT_ARROW[i]; + float2 e = RIM_POLY[j] - RIM_POLY[i]; + float2 w = p - RIM_POLY[i]; float2 b = w - e * saturate(dot(w, e) / dot(e, e)); d = min(d, dot(b, b)); - bool c0 = p.y >= SCULPT_ARROW[i].y; - bool c1 = p.y < SCULPT_ARROW[j].y; + bool c0 = p.y >= RIM_POLY[i].y; + bool c1 = p.y < RIM_POLY[j].y; bool c2 = e.x * w.y > e.y * w.x; if ((c0 && c1 && c2) || (!c0 && !c1 && !c2)) { @@ -674,276 +759,1190 @@ static float s_arrow2(float2 p) return s * sqrt(d); } -static float s_star5(float2 p, float r, float rf) +static float2 s_rim_glove(float2 p, int g, float palm) { - const float2 k1 = float2(0.809016994375, -0.587785252292); - const float2 k2 = float2(-0.809016994375, -0.587785252292); - p.x = abs(p.x); - p -= 2.0 * max(dot(k1, p), 0.0) * k1; - p -= 2.0 * max(dot(k2, p), 0.0) * k2; - p.x = abs(p.x); - p.y -= r; - float2 ba = rf * float2(-k1.y, k1.x) - float2(0.0, 1.0); - float h = clamp(dot(p, ba) / dot(ba, ba), 0.0, r); - return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); + float4 r0 = RIM_GLOVE_R[3 * g]; + float4 r1 = RIM_GLOVE_R[3 * g + 1]; + float4 r2 = RIM_GLOVE_R[3 * g + 2]; + int i = 9 * g; + float f = min(sd_segment(p, RIM_GLOVE[i].xy, RIM_GLOVE[i].zw) - r0.x, + sd_segment(p, RIM_GLOVE[i + 1].xy, RIM_GLOVE[i + 1].zw) - r0.y); + f = min(f, sd_segment(p, RIM_GLOVE[i + 2].xy, RIM_GLOVE[i + 2].zw) - r0.z); + f = min(f, sd_segment(p, RIM_GLOVE[i + 3].xy, RIM_GLOVE[i + 3].zw) - r0.w); + f = min(f, sd_segment(p, RIM_GLOVE[i + 4].xy, RIM_GLOVE[i + 4].zw) - r1.x); + float grooves = min(sd_segment(p, RIM_GLOVE[i + 5].xy, RIM_GLOVE[i + 5].zw) - r1.y, + sd_segment(p, RIM_GLOVE[i + 6].xy, RIM_GLOVE[i + 6].zw) - r1.z); + grooves = min(grooves, sd_segment(p, RIM_GLOVE[i + 7].xy, RIM_GLOVE[i + 7].zw) - r1.w); + float sil = s_smin(palm, f, 0.03); + float4 v = RIM_GLOVE[i + 8]; + float wedge = max(max(-dot(p - v.xy, v.zw), p.x + r2.x), r2.y - p.y); + return float2(sil, max(sil, -min(grooves, wedge))); } -static float s_arrow_solid(float3 p, float h, float re, float dome) +static float2 s_piece(float d, float dc, float sil, float z, float w, float zt, float h, float bump, float mat) { - float d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - float hh = h + dome * smoothstep(0.0, 0.07, -d2); - return s_extrude(d2, p.z - (SCULPT_HOVER + h + dome), hh, re); + float tray = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + zt), 0.5 * (zt - SCULPT_HOVER), 0.012); + float bead = length(float2(d - 0.5 * w, z - zt)) - 0.5 * w; + float u = saturate(-dc / 0.045); + float cushion = 0.8 * s_extrude(dc, z - zt, h + 0.022 * u * (2.0 - u) + bump, 0.75 * h); + return s_opu(float2(min(tray, bead), 5.0), float2(cushion, mat)); } -static float s_piping(float3 p, float ztop) +static float2 s_paper(float d, float sil, float z, float w) { - float d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - return length(float2(d2 + 0.075, p.z - ztop)) - 0.017; + float navy = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + 0.15), 0.5 * (0.15 - SCULPT_HOVER), 0.006); + float sheet = s_extrude(d, z - 0.16, 0.01, 0.004); + return s_opu(float2(navy, 5.0), float2(sheet, 1.0)); } -static float s_glove(float3 p, float zc) +inline float s_bump(float2 p, float2 c, float r) { - float index = s_capsule(p, float3(0.0, -0.10, zc), float3(0.0, -0.52, zc), 0.098); - float palm = s_round_box(p - float3(0.185, -0.70, zc), float3(0.255, 0.19, 0.105), 0.1); - float f1 = s_capsule(p, float3(0.17, -0.57, zc + 0.012), float3(0.17, -0.41, zc + 0.045), 0.086); - float f2 = s_capsule(p, float3(0.31, -0.59, zc + 0.01), float3(0.31, -0.45, zc + 0.04), 0.08); - float f3 = s_capsule(p, float3(0.435, -0.625, zc + 0.005), float3(0.435, -0.52, zc + 0.03), 0.07); - float thumb = s_capsule(p, float3(0.03, -0.77, zc + 0.03), float3(-0.165, -0.60, zc + 0.065), 0.082); - float d = s_smin(palm, min(f1, min(f2, f3)), 0.035); - d = s_smin(d, index, 0.05); - return s_smin(d, thumb, 0.05); + float k = saturate(1.0 - dot(p - c, p - c) / (r * r)); + return k * k; } -static float s_cuff(float3 p, float zc) +static float s_dash(float2 p, float2 a, float2 b, float ra, float rb) { - float3 q = p - float3(0.185, -0.925, zc); - float2 ab = float2(0.272, 0.12); - float e = (length(q.xz / ab) - 1.0) * min(ab.x, ab.y); - return s_extrude(e, q.y, 0.07, 0.06); + p -= a; + b -= a; + float hb = dot(b, b); + float2 q = float2(abs(dot(p, float2(b.y, -b.x))), dot(p, b)) / hb; + float2 c = float2(sqrt(hb - (ra - rb) * (ra - rb)), ra - rb); + float k = c.x * q.y - c.y * q.x; + if (k < 0.0) + { + return sqrt(hb * dot(q, q)) - ra; + } + if (k > c.x) + { + return sqrt(hb * (dot(q, q) + 1.0 - 2.0 * q.y)) - rb; + } + return dot(c, q) - ra; } -static float2 s_sprout(float3 p, float3 c) +static float2 s_rimmed(float3 p, int theme, int shape) { - float3 q = p - c; - float st2 = s_star5(q.xy, 0.125, 0.52) - 0.022; - float2 r = float2(s_extrude(st2, q.z, 0.038, 0.034), 3.0); - float3 e = float3(abs(q.x) - 0.032, q.y + 0.005, q.z - 0.036); - r = s_opu(r, float2(length(e) - 0.0135, 5.0)); - float3 l1 = float3(s_rot(q.xy - float2(-0.04, 0.15), -0.6), q.z); - float3 l2 = float3(s_rot(q.xy - float2(0.045, 0.155), 0.7), q.z); - float leaves = min(s_ellipsoid(l1, float3(0.03, 0.058, 0.02)), s_ellipsoid(l2, float3(0.03, 0.058, 0.02))); - return s_opu(r, float2(leaves, 4.0)); -} - -// Pixel Candy : une ligne par entier, bit c = colonne c (cf. HLSL et `sculpt.rs`). -constant int SCULPT_ARROWPIX[16] = { 0, 1, 3, 7, 31, 63, 127, 255, 511, 63, 55, 115, 113, 224, 224, 64 }; -constant int SCULPT_ARROWBACK[18] = { - 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 255, 511, 511, 999, 995, 960, 192 + bool arrow = shape == 0; + int g = theme == 4 ? 0 : (theme == 0 ? 1 : 2); + float poly = s_rim_poly(p.xy, arrow ? 0 : 7 * g + 7); + float2 body = float2(poly, poly); + float w = 0.06, zt = 0.17, h = 0.03; + float bump = 0.022 * s_bump(p.xy, float2(0.15, -0.45), 0.3); + float back = 1e9; + if (!arrow) + { + w = RIM_GLOVE_R[3 * g + 2].z; + for (int i = 0; i < 2; i++) + { + float2 q = p.xy + (i == 1 ? float2(0.03, 0.04) : float2(0.0, 0.0)); + float2 v = s_rim_glove(q, g, (i == 1 ? s_rim_poly(q, 7 * g + 7) : poly) - 0.05); + if (i == 1) + { + back = v.x - w; + } + else + { + body = v; + if (theme != 2 || v.x - w <= 0.0) + { + break; + } + } + } + zt = 0.185; + h = 0.028; + bump = 0.03 * s_bump(p.xy, float2(0.19, -0.62), 0.28); + } + float dc = body.y; + if (arrow && theme == 0) + { + body -= 0.015; + w = 0.045; + dc = 1.0; + } + float2 r = theme == 2 ? s_paper(body.y, body.x, p.z, w) + : s_piece(body.y, dc, body.x, p.z, w, zt, h, bump, 1.0); + if (theme == 0) + { + if (arrow) + { + float band = s_extrude(abs(poly + 0.0075) - 0.0225, p.z - zt, 0.028, 0.012); + r = s_opu(r, float2(band, 2.0)); + } + return r; + } + if (theme == 2) + { + float dash; + if (arrow) + { + float back = s_rim_poly(p.xy + float2(0.025, 0.05), 0) - 0.06; + r = s_opu(r, float2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 2.0)); + dash = min(s_dash(p.xy, float2(-0.1179, -0.1374), float2(-0.1799, -0.0443), 0.028, 0.045), + s_dash(p.xy, float2(-0.1347, -0.2493), float2(-0.248, -0.2056), 0.026, 0.042)); + } + else + { + r = s_opu(r, float2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 3.0)); + dash = min(s_dash(p.xy, float2(0.216, -0.1695), float2(0.2593, -0.0685), 0.028, 0.042), + s_dash(p.xy, float2(0.3054, -0.2339), float2(0.3949, -0.1652), 0.0275, 0.041)); + } + return s_opu(r, float2(s_extrude(dash, p.z - 0.11, 0.06, 0.006), 3.0)); + } + if (!arrow) + { + float2 cq = p.xy - float2(0.1541, -0.9047); + float2 bq = abs(float2(cq.x, cq.y - 0.2066 * cq.x * cq.x)) - float2(0.214, 0.0361); + float cuff = length(max(bq, 0.0)) + min(max(bq.x, bq.y), 0.0) - 0.03; + r = s_opu(r, s_piece(cuff, cuff, cuff, p.z, 0.0477, 0.225, 0.025, 0.0, 2.0)); + } + float2 c = arrow ? float2(0.6118, -0.8079) : float2(0.15, -0.885); + float rs = arrow ? 0.1678 : 0.128; + float zs = arrow ? 0.23 : 0.285; + float2 q = s_rot(p.xy - c, arrow ? 0.2443 : 0.0) / rs; + float star = (s_star5(q, 0.82, 0.55) - 0.18) * rs; + float leaves = min(s_vesica(s_rot(q - float2(-0.33, 1.38), -0.925), 0.4296, 0.2626), + s_vesica(s_rot(q - float2(0.53, 1.37), 0.873), 0.4296, 0.2626)) * rs; + r = s_opu(r, s_piece(leaves, leaves, leaves, p.z, 0.038, zs - 0.02, 0.018, 0.0, 4.0)); + r = s_opu(r, s_piece(star, star, star, p.z, 0.038, zs, 0.022, 0.018 * s_bump(q, float2(0.0), 1.0), 3.0)); + float3 e = float3(abs(q.x) - 0.25, q.y - 0.08, (p.z - zs - 0.056) / rs); + return s_opu(r, float2(s_ellipsoid(e, float3(0.075, 0.13, 0.1)) * rs, 5.0)); +} + +// Pixel Candy : tables générées par scripts/generate-pixel-candy-voxels.mjs (cf. HLSL). +// +constant int PIX_BODY[32] = { 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 127, 247, 243, 480, 192, 48, 120, 120, 120, 504, 4088, 32760, 65534, 65535, 65535, 65534, 32766, 32764, 16380, 16376, 16376 }; +constant int PIX_LINE[32] = { 1, 3, 5, 9, 17, 33, 65, 129, 257, 513, 1985, 73, 149, 147, 288, 192, 48, 72, 72, 72, 456, 3656, 29256, 37454, 32777, 32769, 32770, 16386, 16388, 8196, 8200, 16376 }; +constant int PIX_HI[32] = { 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 18, 34, 32, 64, 0, 0, 16, 16, 16, 16, 16, 16, 16, 22, 18, 4, 4, 8, 8, 16, 0 }; +constant int PIX_SHADE[32] = { 0, 0, 0, 4, 8, 16, 32, 64, 128, 448, 40, 36, 64, 64, 128, 0, 0, 32, 32, 32, 32, 288, 2336, 18720, 16384, 16384, 16384, 8192, 8192, 4096, 8160, 0 }; +constant int PIX_RECT_N[3] = { 0, 17, 29 }; +constant int4 PIX_GRID[2] = { + int4(11, 16, 0, 0), + int4(16, 16, 16, 0) +}; +constant float2 PIX_ORIGIN[2] = { + float2(0.0, 0.0), + float2(-0.3125, 0.0) }; -constant int SCULPT_HANDPIX[15] = { 4, 14, 14, 14, 110, 878, 7022, 7022, 8190, 8191, 8191, 8191, 8190, 4092, 4092 }; -constant int SCULPT_HANDBACK[17] = { - 28, 62, 62, 62, 510, 4094, 32766, 32766, 32766, 32767, 32767, 32767, 32767, 32767, 32766, 16380, 16380 +constant float4 PIX_BOX[2] = { + float4(0.3438, -0.5, 0.3438, 0.5), + float4(0.1875, -0.5, 0.5, 0.5) }; -constant float2 SCULPT_HANDSPAN[17] = { - float2(2.0, 4.0), float2(1.0, 5.0), float2(1.0, 5.0), float2(1.0, 5.0), float2(1.0, 8.0), - float2(1.0, 11.0), float2(1.0, 14.0), float2(1.0, 14.0), float2(1.0, 14.0), float2(0.0, 14.0), - float2(0.0, 14.0), float2(0.0, 14.0), float2(0.0, 14.0), float2(0.0, 14.0), float2(1.0, 14.0), - float2(2.0, 13.0), float2(2.0, 13.0) +constant float4 PIX_RECT[29] = { + float4(0.0313, -0.0313, 0.0313, 0.0313), + float4(0.0625, -0.0938, 0.0625, 0.0313), + float4(0.0938, -0.1563, 0.0938, 0.0313), + float4(0.125, -0.2188, 0.125, 0.0313), + float4(0.1563, -0.2813, 0.1563, 0.0313), + float4(0.1875, -0.3438, 0.1875, 0.0313), + float4(0.2188, -0.4063, 0.2188, 0.0313), + float4(0.25, -0.4688, 0.25, 0.0313), + float4(0.2813, -0.5313, 0.2813, 0.0313), + float4(0.3125, -0.5938, 0.3125, 0.0313), + float4(0.3438, -0.6563, 0.3438, 0.0313), + float4(0.2188, -0.7188, 0.2188, 0.0313), + float4(0.0938, -0.7813, 0.0938, 0.0313), + float4(0.375, -0.8125, 0.125, 0.0625), + float4(0.0625, -0.8438, 0.0625, 0.0313), + float4(0.4375, -0.9063, 0.125, 0.0313), + float4(0.4375, -0.9688, 0.0625, 0.0313), + float4(0.0, -0.0313, 0.0625, 0.0313), + float4(0.0, -0.1563, 0.125, 0.0938), + float4(0.0625, -0.2813, 0.1875, 0.0313), + float4(0.1563, -0.3438, 0.2813, 0.0313), + float4(0.25, -0.4063, 0.375, 0.0313), + float4(0.2188, -0.4688, 0.4688, 0.0313), + float4(0.1875, -0.5625, 0.5, 0.0625), + float4(0.2188, -0.6563, 0.4688, 0.0313), + float4(0.1875, -0.7188, 0.4375, 0.0313), + float4(0.2188, -0.7813, 0.4063, 0.0313), + float4(0.1875, -0.8438, 0.375, 0.0313), + float4(0.2188, -0.9375, 0.3438, 0.0625) }; +// -inline float2 s_grid_origin(int shape) +inline bool s_pix_bit(int m, int c) { - return shape == 0 ? float2(0.0, 0.0) : float2(-2.5 * SCULPT_VOX, 0.0); + return ((m >> clamp(c, 0, 31)) & 1) == 1; } -inline bool s_bit(int row, int r, int c, int rows, int cols) +static bool s_pix_body(int shape, int c, int r) { - return r >= 0 && r < rows && c >= 0 && c < cols && ((row >> clamp(c, 0, 31)) & 1) == 1; + int4 g = PIX_GRID[shape]; + return c >= 0 && c < g.x && r >= 0 && r < g.y && s_pix_bit(PIX_BODY[g.z + clamp(r, 0, g.y - 1)], c); } -static bool s_occ(float2 id, int shape) +inline int2 s_pix_cell(float2 p, int shape) { - int c = int(id.x); - int r = -int(id.y) - 1; - if (shape == 0) - { - return s_bit(SCULPT_ARROWPIX[clamp(r, 0, 15)], r, c, 16, 16); - } - return s_bit(SCULPT_HANDPIX[clamp(r, 0, 14)], r, c, 15, 13); -} - -static bool s_occ_back(float2 id, int shape) -{ - int c = int(id.x) + 1; - int r = -int(id.y); - if (shape == 0) - { - return s_bit(SCULPT_ARROWBACK[clamp(r, 0, 17)], r, c, 18, 17); - } - return s_bit(SCULPT_HANDBACK[clamp(r, 0, 16)], r, c, 17, 15); + float2 o = PIX_ORIGIN[shape]; + return int2(int(floor((p.x - o.x) / SCULPT_VOX)), int(floor((o.y - p.y) / SCULPT_VOX))); } static float2 s_voxels(float3 p, int shape) { - float2 o = s_grid_origin(shape); - float2 cell = floor((p.xy - o) / SCULPT_VOX); - float2 bc = shape == 0 ? float2(0.33, -0.5) : float2(0.25, -0.47); - float2 bh = shape == 0 ? float2(0.44, 0.61) : float2(0.48, 0.55); - float2 bq = max(abs(p.xy - bc) - bh, 0.0); + float4 b = PIX_BOX[shape]; + float2 bq = max(abs(p.xy - b.xy) - b.zw, 0.0); float bz = max(abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07, 0.0); - float far = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); - float dF = far; - float dB = far; - const float3 half_cell = float3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.04); + float d = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); + float2 o = PIX_ORIGIN[shape]; + int2 cell = s_pix_cell(p.xy, shape); + const float3 cube = float3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.07); for (int j = -1; j <= 1; j++) { for (int i = -1; i <= 1; i++) { - float2 id = cell + float2(float(i), float(j)); - float3 q = float3(p.xy - (o + (id + 0.5) * SCULPT_VOX), p.z); - if (s_occ(id, shape)) + int c = cell.x + i; + int r = cell.y + j; + if (s_pix_body(shape, c, r)) { - dF = min(dF, s_round_box(q - float3(0.0, 0.0, SCULPT_HOVER + 0.1), half_cell, 0.009)); - } - if (s_occ_back(id, shape)) - { - dB = min(dB, s_round_box(q - float3(0.0, 0.0, SCULPT_HOVER + 0.04), half_cell, 0.009)); + float3 q = float3(p.x - o.x - (float(c) + 0.5) * SCULPT_VOX, p.y - o.y + (float(r) + 0.5) * SCULPT_VOX, + p.z - SCULPT_HOVER - 0.07); + d = min(d, s_round_box(q, cube, 0.006)); } } } - return dF < dB ? float2(dF, 6.0) : float2(dB, 7.0); + return float2(d, 6.0); } -static float s_pixel_hand_dist(float2 p) +static float s_pixel_outline(float2 p, int shape) { float d = 1e9; - for (int r = 0; r < 17; r++) + for (int i = PIX_RECT_N[shape]; i < PIX_RECT_N[shape + 1]; i++) { - float2 x = (SCULPT_HANDSPAN[r] + float2(-3.5, -2.5)) * SCULPT_VOX; - float2 y = float2(-float(r), 1.0 - float(r)) * SCULPT_VOX; - float2 q = max(max(float2(x.x, y.x) - p, p - float2(x.y, y.y)), 0.0); - d = min(d, length(q)); + float2 q = abs(p - PIX_RECT[i].xy) - PIX_RECT[i].zw; + d = min(d, length(max(q, 0.0)) + min(max(q.x, q.y), 0.0)); } return d; } -static float s_gem_edge(float3 p, float2 a, float2 b, float z0) -{ - float2 e = b - a; - float len = length(e); - float2 d = e / len; - float2 q = p.xy - a; - float dist = dot(q, float2(-d.y, d.x)); - float along = abs(dot(q, d) - 0.5 * len); - float z = p.z - z0; - float girdle = (z - 0.035 - 2.2 * dist) * 0.4138; - float crown = (z - 0.07 - 0.6 * dist + 0.18 * along) * 0.8438; - return max(-dist, max(girdle, crown)); -} +// ---- Prism Glow : un cristal en MAILLAGE ---- (cf. HLSL) +// prism mesh: generated by design/cursors/prism-glow/model/export_compositor.py +constant int PRISM_TRI_START[2] = { 0, 31 }; +constant int PRISM_TRI_COUNT[2] = { 31, 130 }; +constant int PRISM_SIL_START[2] = { 0, 25 }; +constant int PRISM_SIL_COUNT[2] = { 25, 42 }; +constant int PRISM_OUT_START[2] = { 67, 76 }; +constant int PRISM_OUT_COUNT[2] = { 9, 34 }; +constant int PRISM_BOX_START[2] = { 0, 4 }; +constant int PRISM_BOX_COUNT[2] = { 4, 18 }; +constant float4 PRISM_TRIS[644] = { + float4(0.252301, 0.266258, -0.042945, 11.0), + float4(-0.221166, -0.203988, 0.0, 0.01033), + float4(-0.221166, -0.203988, 0.042945, 0.045182), + float4(0.677984, -0.735077, 0.0, 0.577584), + float4(0.252301, 0.266258, -0.042945, 3.0), + float4(-0.221166, -0.203988, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.677984, -0.735077, 0.0, 0.577584), + float4(0.031135, 0.06227, -0.042945, 7.0), + float4(0.004294, 0.419785, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-0.999948, 0.01023, 0.0, 0.577584), + float4(0.169632, 0.356442, 0.115951, 14.0), + float4(0.082669, -0.090184, -0.115951, 0.590616), + float4(-0.138497, -0.294172, -0.115951, 0.737911), + float4(0.466385, -0.505659, 0.725806, 0.973449), + float4(0.035429, 0.482055, 0.0, 14.0), + float4(0.134202, -0.125614, 0.115951, 0.000305), + float4(-0.004294, -0.419785, 0.0, 0.938689), + float4(-0.650191, 0.006652, 0.759742, 0.854994), + float4(0.031135, 0.06227, -0.042945, 9.0), + float4(0.004294, 0.419785, 0.0, 0.01033), + float4(0.004294, 0.419785, 0.042945, 0.045182), + float4(-0.999948, 0.01023, 0.0, 0.577584), + float4(0.169632, 0.356442, 0.115951, 14.0), + float4(0.159969, 0.080521, -0.045092, 0.381323), + float4(0.082669, -0.090184, -0.115951, 0.056123), + float4(0.461407, -0.510214, 0.725799, 0.973449), + float4(0.541104, 0.544325, -0.042945, 9.0), + float4(-0.288804, -0.278068, 0.0, 0.01033), + float4(-0.288804, -0.278068, 0.042945, 0.045182), + float4(0.69359, -0.72037, 0.0, 0.577584), + float4(0.329601, 0.436963, 0.070859, 14.0), + float4(0.211503, 0.107362, -0.070859, 0.545724), + float4(-0.077301, -0.170706, -0.070859, 0.064805), + float4(0.49567, -0.514808, 0.699488, 0.973449), + float4(0.268405, 0.533589, 0.081595, 14.0), + float4(0.061196, -0.096626, -0.010736, 0.000916), + float4(-0.098773, -0.177147, 0.034356, 0.046662), + float4(0.247834, 0.049458, 0.967539, 0.904662), + float4(0.541104, 0.544325, -0.042945, 7.0), + float4(-0.288804, -0.278068, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.69359, -0.72037, 0.0, 0.577584), + float4(0.199693, 0.595859, 0.0, 14.0), + float4(0.068712, -0.06227, 0.081595, 0.0), + float4(-0.030061, -0.239417, 0.115951, 0.01445), + float4(-0.504461, 0.426837, 0.750553, 0.254154), + float4(0.268405, 0.533589, 0.081595, 14.0), + float4(0.272699, 0.010736, -0.081595, 0.0), + float4(0.061196, -0.096626, -0.010736, 0.025193), + float4(0.283244, 0.073137, 0.956255, 0.527108), + float4(0.035429, 0.482055, 0.0, 14.0), + float4(0.0, 0.258742, 0.0, 0.0), + float4(0.134202, -0.125614, 0.115951, 0.473533), + float4(-0.653777, 0.0, 0.756687, 0.964691), + float4(0.359663, 0.559356, 0.0, 14.0), + float4(0.181442, -0.015031, 0.0, 0.006043), + float4(-0.091258, -0.025767, 0.081595, 0.076188), + float4(0.076465, 0.923048, 0.37701, 0.964691), + float4(0.035429, 0.482055, -0.042945, 3.0), + float4(0.0, 0.258742, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-1.0, 0.0, 0.0, 0.577584), + float4(0.035429, 0.482055, -0.042945, 11.0), + float4(0.0, 0.258742, 0.0, 0.01033), + float4(0.0, 0.258742, 0.042945, 0.045182), + float4(-1.0, 0.0, 0.0, 0.577584), + float4(0.035429, 0.740798, -0.042945, 7.0), + float4(0.164264, -0.144939, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.661622, 0.749838, 0.0, 0.577584), + float4(0.035429, 0.740798, 0.0, 14.0), + float4(0.164264, -0.144939, 0.0, 0.0), + float4(0.134202, -0.384356, 0.115951, 0.010956), + float4(0.332572, 0.376915, 0.864483, 0.215854), + float4(0.035429, 0.740798, -0.042945, 11.0), + float4(0.164264, -0.144939, 0.0, 0.01033), + float4(0.164264, -0.144939, 0.042945, 0.045182), + float4(0.661622, 0.749838, 0.0, 0.577584), + float4(0.199693, 0.595859, -0.042945, 7.0), + float4(0.105215, 0.22546, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-0.906183, 0.422886, 0.0, 0.577584), + float4(0.199693, 0.595859, 0.0, 14.0), + float4(0.105215, 0.22546, 0.0, 0.0), + float4(0.068712, -0.06227, 0.081595, 0.015991), + float4(-0.613882, 0.286478, 0.735581, 0.473533), + float4(0.199693, 0.595859, -0.042945, 11.0), + float4(0.105215, 0.22546, 0.0, 0.01033), + float4(0.105215, 0.22546, 0.042945, 0.045182), + float4(-0.906183, 0.422886, 0.0, 0.577584), + float4(0.304908, 0.821319, 0.0, 14.0), + float4(0.128834, -0.067638, 0.0, 0.0), + float4(-0.036503, -0.28773, 0.081595, 0.254154), + float4(0.133686, 0.254641, 0.957751, 0.973449), + float4(0.304908, 0.821319, -0.042945, 7.0), + float4(0.128834, -0.067638, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.464834, 0.885398, 0.0, 0.577584), + float4(0.433742, 0.753681, 0.0, 14.0), + float4(-0.07408, -0.194325, 0.0, 0.0), + float4(-0.165337, -0.220092, 0.081595, 0.964691), + float4(0.683362, -0.260508, 0.682021, 0.964691), + float4(0.433742, 0.753681, -0.042945, 11.0), + float4(-0.07408, -0.194325, 0.0, 0.01033), + float4(-0.07408, -0.194325, 0.042945, 0.045182), + float4(0.934406, -0.35621, 0.0, 0.577584), + float4(0.304908, 0.821319, -0.042945, 11.0), + float4(0.128834, -0.067638, 0.0, 0.01033), + float4(0.128834, -0.067638, 0.042945, 0.045182), + float4(0.464834, 0.885398, 0.0, 0.577584), + float4(0.433742, 0.753681, -0.042945, 7.0), + float4(-0.07408, -0.194325, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.934406, -0.35621, 0.0, 0.577584), + float4(0.359663, 0.559356, -0.042945, 7.0), + float4(0.181442, -0.015031, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.082557, 0.996586, 0.0, 0.577584), + float4(0.359663, 0.559356, -0.042945, 11.0), + float4(0.181442, -0.015031, 0.0, 0.01033), + float4(0.181442, -0.015031, 0.042945, 0.045182), + float4(0.082557, 0.996586, 0.0, 0.577584), + float4(0.044345, 0.043263, -0.043263, 11.0), + float4(-0.043263, -0.005408, 0.0, 0.01033), + float4(-0.043263, -0.005408, 0.043263, 0.045182), + float4(0.124035, -0.992278, 0.0, 0.577584), + float4(0.044345, 0.043263, -0.043263, 7.0), + float4(-0.043263, -0.005408, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.124035, -0.992278, 0.0, 0.577584), + float4(0.001082, 0.037855, -0.043263, 7.0), + float4(-0.044345, 0.040019, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.669964, -0.742393, 0.0, 0.577584), + float4(0.044345, 0.043263, 0.0, 10.0), + float4(-0.043263, -0.005408, 0.0, 0.000305), + float4(-0.040019, 0.030284, 0.049753, 0.964691), + float4(0.101434, -0.811469, 0.575525, 0.973449), + float4(0.004326, 0.073548, 0.049753, 14.0), + float4(-0.003245, -0.035692, -0.049753, 0.0), + float4(-0.04759, 0.004326, -0.049753, 0.955978), + float4(-0.580475, -0.643229, 0.499304, 0.896269), + float4(0.001082, 0.037855, -0.043263, 11.0), + float4(-0.044345, 0.040019, 0.0, 0.01033), + float4(-0.044345, 0.040019, 0.043263, 0.045182), + float4(-0.669964, -0.742393, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.035692, -0.155748, -0.008653, 0.000305), + float4(-0.011897, -0.151422, -0.058405, 0.955978), + float4(-0.716533, -0.20131, 0.667874, 0.904662), + float4(-0.043263, 0.077874, -0.043263, 7.0), + float4(-0.001082, 0.154666, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999976, -0.006993, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.089771, -0.137361, -0.02055, 0.445197), + float4(0.035692, -0.155748, -0.008653, 0.973449), + float4(0.216369, -0.004654, 0.976301, 0.973449), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(-0.011897, -0.151422, -0.058405, 0.000305), + float4(-0.012979, 0.003245, -0.058405, 0.955978), + float4(-0.976245, -0.006827, 0.216564, 0.904662), + float4(-0.043263, 0.077874, -0.043263, 9.0), + float4(-0.001082, 0.154666, 0.0, 0.01033), + float4(-0.001082, 0.154666, 0.043263, 0.045182), + float4(-0.999976, -0.006993, 0.0, 0.577584), + float4(-0.045426, 0.564586, 0.052998, 14.0), + float4(0.014061, -0.335291, 0.005408, 0.520989), + float4(0.001082, -0.332046, -0.052998, 0.088655), + float4(-0.975941, -0.037462, 0.214794, 0.973449), + float4(-0.044345, 0.23254, 0.0, 3.0), + float4(0.0, 0.0, -0.043263, 0.01033), + float4(-0.001082, 0.332046, -0.043263, 0.045182), + float4(-0.999995, -0.003257, 0.0, 0.577584), + float4(-0.163319, 0.380717, -0.043263, 7.0), + float4(-0.060569, 0.02704, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.407651, -0.913138, 0.0, 0.577584), + float4(-0.094098, 0.431551, -0.043263, 11.0), + float4(-0.069221, -0.050834, 0.0, 0.01033), + float4(-0.069221, -0.050834, 0.043263, 0.045182), + float4(0.591909, -0.806004, 0.0, 0.577584), + float4(-0.163319, 0.380717, -0.043263, 11.0), + float4(-0.060569, 0.02704, 0.0, 0.01033), + float4(-0.060569, 0.02704, 0.043263, 0.045182), + float4(-0.407651, -0.913138, 0.0, 0.577584), + float4(0.075711, 0.08869, -0.043263, 11.0), + float4(-0.031366, -0.045426, 0.0, 0.01033), + float4(-0.031366, -0.045426, 0.043263, 0.045182), + float4(0.822897, -0.568191, 0.0, 0.577584), + float4(0.044345, 0.043263, 0.0, 6.0), + float4(-0.040019, 0.030284, 0.049753, 0.000305), + float4(0.014061, 0.048671, 0.037855, 0.964691), + float4(0.365607, -0.634949, 0.680566, 0.973449), + float4(0.075711, 0.08869, -0.043263, 7.0), + float4(-0.031366, -0.045426, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.822897, -0.568191, 0.0, 0.577584), + float4(0.058405, 0.091934, 0.037855, 14.0), + float4(0.017305, -0.003245, -0.037855, 0.0), + float4(-0.014061, -0.048671, -0.037855, 0.473533), + float4(0.757369, -0.522945, 0.39105, 0.904662), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.107077, -0.140606, -0.058405, 0.0), + float4(0.089771, -0.137361, -0.02055, 0.376257), + float4(0.812052, 0.481346, 0.329966, 0.830777), + float4(0.077874, 0.407756, -0.043263, 11.0), + float4(-0.002163, -0.319067, 0.0, 0.01033), + float4(-0.002163, -0.319067, 0.043263, 0.045182), + float4(0.999977, -0.006779, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.10924, 0.178461, -0.058405, 0.0), + float4(0.107077, -0.140606, -0.058405, 0.376257), + float4(0.475583, -0.003224, 0.879665, 0.830777), + float4(0.147095, 0.272559, -0.043263, 7.0), + float4(-0.035692, 0.034611, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.696146, -0.7179, 0.0, 0.577584), + float4(0.147095, 0.272559, -0.043263, 11.0), + float4(-0.035692, 0.034611, 0.0, 0.01033), + float4(-0.035692, 0.034611, 0.043263, 0.045182), + float4(-0.696146, -0.7179, 0.0, 0.577584), + float4(0.144932, 0.307169, 0.049753, 14.0), + float4(0.002163, -0.034611, -0.049753, 0.0), + float4(-0.033529, 0.0, -0.049753, 0.9131), + float4(-0.630237, -0.649931, 0.424725, 0.904662), + float4(0.183869, 0.317985, 0.037855, 6.0), + float4(-0.036774, -0.045426, -0.037855, 0.0), + float4(-0.038937, -0.010816, 0.011897, 0.806958), + float4(0.374433, -0.753477, 0.540438, 0.854994), + float4(0.077874, 0.407756, -0.043263, 7.0), + float4(-0.002163, -0.319067, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.999977, -0.006779, 0.0, 0.577584), + float4(0.111403, 0.307169, -0.043263, 7.0), + float4(-0.002163, 0.131953, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999866, -0.016391, 0.0, 0.577584), + float4(0.10924, 0.439122, 0.0, 6.0), + float4(0.035692, -0.131953, 0.049753, 0.18117), + float4(0.002163, -0.131953, 0.0, 0.955978), + float4(-0.82919, -0.013593, 0.558802, 0.964691), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(0.055161, -0.146014, -0.025958, 0.246204), + float4(0.016224, -0.156829, -0.014061, 0.964691), + float4(0.305397, -0.053644, 0.950713, 0.973449), + float4(0.111403, 0.307169, -0.043263, 11.0), + float4(-0.002163, 0.131953, 0.0, 0.01033), + float4(-0.002163, 0.131953, 0.043263, 0.045182), + float4(-0.999866, -0.016391, 0.0, 0.577584), + float4(0.189277, 0.28013, -0.043263, 11.0), + float4(-0.042182, -0.007571, 0.0, 0.01033), + float4(-0.042182, -0.007571, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.183869, 0.317985, 0.037855, 10.0), + float4(0.005408, -0.037855, -0.037855, 0.0), + float4(-0.036774, -0.045426, -0.037855, 0.806958), + float4(0.124328, -0.692684, 0.710445, 0.854994), + float4(0.189277, 0.28013, -0.043263, 7.0), + float4(-0.042182, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.183869, 0.317985, 0.037855, 14.0), + float4(0.02704, -0.004326, -0.037855, 0.0), + float4(0.005408, -0.037855, -0.037855, 0.514916), + float4(0.70062, -0.452013, 0.552101, 0.846876), + float4(0.210908, 0.313659, -0.043263, 11.0), + float4(-0.021632, -0.033529, 0.0, 0.01033), + float4(-0.021632, -0.033529, 0.043263, 0.045182), + float4(0.840297, -0.542127, 0.0, 0.577584), + float4(0.210908, 0.445612, 0.0, 10.0), + float4(0.0, -0.131953, 0.0, 0.0), + float4(-0.02704, -0.127627, 0.037855, 0.439658), + float4(0.813734, 0.0, 0.581238, 0.871366), + float4(0.210908, 0.313659, -0.043263, 7.0), + float4(-0.021632, -0.033529, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.840297, -0.542127, 0.0, 0.577584), + float4(0.210908, 0.445612, -0.043263, 11.0), + float4(0.0, -0.131953, 0.0, 0.01033), + float4(0.0, -0.131953, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.241193, 0.343943, -0.043263, 7.0), + float4(-0.002163, 0.107077, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999796, -0.020198, 0.0, 0.577584), + float4(0.276885, 0.311496, -0.043263, 11.0), + float4(-0.035692, 0.032447, 0.0, 0.01033), + float4(-0.035692, 0.032447, 0.043263, 0.045182), + float4(-0.672673, -0.73994, 0.0, 0.577584), + float4(0.23903, 0.45102, 0.0, 14.0), + float4(0.042182, -0.108158, 0.046508, 0.000305), + float4(0.002163, -0.107077, 0.0, 0.846876), + float4(-0.758098, -0.015315, 0.651961, 0.896269), + float4(0.276885, 0.311496, -0.043263, 7.0), + float4(-0.035692, 0.032447, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.672673, -0.73994, 0.0, 0.577584), + float4(0.281211, 0.342862, 0.046508, 14.0), + float4(-0.004326, -0.031366, -0.046508, 0.0), + float4(-0.040019, 0.001082, -0.046508, 0.752941), + float4(-0.58651, -0.645161, 0.489667, 0.846876), + float4(0.246601, 0.459672, 0.06165, 14.0), + float4(0.070303, -0.101669, -0.024876, 0.262257), + float4(0.034611, -0.116811, -0.015142, 0.973449), + float4(0.27928, -0.041605, 0.959308, 0.973449), + float4(0.319067, 0.319067, -0.043263, 11.0), + float4(-0.042182, -0.007571, 0.0, 0.01033), + float4(-0.042182, -0.007571, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.319067, 0.319067, 0.0, 10.0), + float4(-0.042182, -0.007571, 0.0, 0.0), + float4(-0.037855, 0.023795, 0.046508, 0.896269), + float4(0.1483, -0.826245, 0.54344, 0.896269), + float4(0.319067, 0.319067, -0.043263, 7.0), + float4(-0.042182, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.319067, 0.319067, 0.0, 6.0), + float4(-0.037855, 0.023795, 0.046508, 0.0), + float4(-0.002163, 0.038937, 0.036774, 0.896269), + float4(0.437906, -0.604311, 0.665618, 0.896269), + float4(0.316904, 0.358004, 0.036774, 14.0), + float4(0.024876, -0.004326, -0.036774, 0.0), + float4(0.002163, -0.038937, -0.036774, 0.617212), + float4(0.707338, -0.464191, 0.533104, 0.921584), + float4(0.34178, 0.353677, -0.043263, 11.0), + float4(-0.022713, -0.034611, 0.0, 0.01033), + float4(-0.022713, -0.034611, 0.043263, 0.045182), + float4(0.836048, -0.548657, 0.0, 0.577584), + float4(0.340698, 0.474815, 0.0, 14.0), + float4(0.001082, -0.121137, 0.0, 0.0), + float4(-0.023795, -0.116811, 0.036774, 0.473533), + float4(0.828664, 0.007399, 0.559697, 0.930114), + float4(0.34178, 0.353677, -0.043263, 7.0), + float4(-0.022713, -0.034611, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.836048, -0.548657, 0.0, 0.577584), + float4(0.340698, 0.474815, -0.043263, 11.0), + float4(0.001082, -0.121137, 0.0, 0.01033), + float4(0.001082, -0.121137, 0.043263, 0.045182), + float4(0.99996, 0.008928, 0.0, 0.577584), + float4(0.411001, 0.351514, -0.043263, 11.0), + float4(-0.034611, 0.030284, 0.0, 0.01033), + float4(-0.034611, 0.030284, 0.043263, 0.045182), + float4(-0.658505, -0.752577, 0.0, 0.577584), + float4(0.411001, 0.351514, -0.043263, 7.0), + float4(-0.034611, 0.030284, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.658505, -0.752577, 0.0, 0.577584), + float4(0.412083, 0.38288, 0.043263, 14.0), + float4(-0.001082, -0.031366, -0.043263, 0.0), + float4(-0.035692, -0.001082, -0.043263, 0.745403), + float4(-0.574039, -0.656045, 0.489984, 0.854994), + float4(0.45102, 0.359085, -0.043263, 11.0), + float4(-0.040019, -0.007571, 0.0, 0.01033), + float4(-0.040019, -0.007571, 0.043263, 0.045182), + float4(0.185892, -0.98257, 0.0, 0.577584), + float4(0.45102, 0.359085, 0.0, 10.0), + float4(-0.040019, -0.007571, 0.0, 0.000305), + float4(-0.038937, 0.023795, 0.043263, 0.879622), + float4(0.151736, -0.802035, 0.577682, 0.9131), + float4(0.45102, 0.359085, -0.043263, 7.0), + float4(-0.040019, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.185892, -0.98257, 0.0, 0.577584), + float4(0.45102, 0.359085, 0.0, 6.0), + float4(-0.038937, 0.023795, 0.043263, 0.000305), + float4(0.003245, 0.0411, 0.032447, 0.879622), + float4(0.417852, -0.583049, 0.696744, 0.9131), + float4(0.473733, 0.391533, -0.043263, 11.0), + float4(-0.022713, -0.032447, 0.0, 0.01033), + float4(-0.022713, -0.032447, 0.043263, 0.045182), + float4(0.819232, -0.573462, 0.0, 0.577584), + float4(0.454264, 0.400185, 0.032447, 14.0), + float4(0.019468, -0.008653, -0.032447, 0.0), + float4(-0.003245, -0.0411, -0.032447, 0.527108), + float4(0.688617, -0.482032, 0.541712, 0.887922), + float4(0.473733, 0.391533, -0.043263, 7.0), + float4(-0.022713, -0.032447, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.819232, -0.573462, 0.0, 0.577584), + float4(-0.223888, 0.407756, -0.043263, 7.0), + float4(0.002163, 0.06814, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999496, 0.03173, 0.0, 0.577584), + float4(-0.223888, 0.407756, -0.043263, 11.0), + float4(0.002163, 0.06814, 0.0, 0.01033), + float4(0.002163, 0.06814, 0.043263, 0.045182), + float4(-0.999496, 0.03173, 0.0, 0.577584), + float4(-0.167645, 0.416409, 0.046508, 14.0), + float4(-0.056242, -0.008653, -0.046508, 0.000305), + float4(-0.054079, 0.059487, -0.046508, 0.955978), + float4(-0.638985, 0.020285, 0.768952, 0.973449), + float4(-0.167645, 0.416409, 0.046508, 14.0), + float4(0.004326, -0.035692, -0.046508, 0.000305), + float4(-0.056242, -0.008653, -0.046508, 0.955978), + float4(-0.339782, -0.761112, 0.552501, 0.973449), + float4(-0.221724, 0.475896, 0.0, 14.0), + float4(0.139524, 0.055161, 0.043263, 0.520989), + float4(0.054079, -0.059487, 0.046508, 0.973449), + float4(-0.39307, 0.317373, 0.863001, 0.973449), + float4(-0.167645, 0.416409, 0.046508, 6.0), + float4(0.073548, 0.015142, -0.046508, 0.0), + float4(0.004326, -0.035692, -0.046508, 0.768154), + float4(0.490917, -0.668482, 0.558688, 0.871366), + float4(-0.094098, 0.431551, -0.043263, 7.0), + float4(-0.069221, -0.050834, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.591909, -0.806004, 0.0, 0.577584), + float4(-0.167645, 0.416409, 0.046508, 10.0), + float4(0.085445, 0.114648, -0.003245, 0.0), + float4(0.073548, 0.015142, -0.046508, 0.768154), + float4(0.548356, -0.38771, 0.740936, 0.871366), + float4(-0.0822, 0.531057, -0.043263, 11.0), + float4(-0.011897, -0.099506, 0.0, 0.01033), + float4(-0.011897, -0.099506, 0.043263, 0.045182), + float4(0.992928, -0.11872, 0.0, 0.577584), + float4(-0.044345, 0.23254, 0.0, 7.0), + float4(-0.001082, 0.332046, -0.043263, 0.01033), + float4(-0.001082, 0.332046, 0.052998, 0.045182), + float4(-0.999995, -0.003257, 0.0, 0.577584), + float4(-0.045426, 0.564586, 0.052998, 14.0), + float4(0.1233, -0.156829, -0.052998, 0.520989), + float4(0.014061, -0.335291, 0.005408, 0.088655), + float4(0.429355, 0.032562, 0.902549, 0.973449), + float4(0.10924, 0.439122, 0.0, 14.0), + float4(-0.031366, -0.031366, 0.0, 0.132876), + float4(-0.154666, 0.125464, 0.052998, 0.028428), + float4(0.182761, -0.182761, 0.966021, 0.955978), + float4(0.10924, 0.439122, -0.043263, 11.0), + float4(-0.031366, -0.031366, 0.0, 0.01033), + float4(-0.031366, -0.031366, 0.043263, 0.045182), + float4(0.707107, -0.707107, 0.0, 0.577584), + float4(0.10924, 0.439122, -0.043263, 7.0), + float4(-0.031366, -0.031366, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.707107, -0.707107, 0.0, 0.577584), + float4(0.10924, 0.439122, 0.0, 10.0), + float4(0.019468, 0.024876, 0.063813, 0.18117), + float4(0.035692, -0.131953, 0.049753, 0.955978), + float4(-0.933936, -0.126584, 0.334276, 0.964691), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(0.050834, 0.002163, -0.005408, 0.0), + float4(0.0822, -0.018387, -0.063813, 0.439658), + float4(0.07859, -0.92643, 0.368172, 0.871366), + float4(-0.221724, 0.475896, -0.043263, 7.0), + float4(0.122219, 0.207664, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.861819, 0.507216, 0.0, 0.577584), + float4(-0.221724, 0.475896, -0.043263, 9.0), + float4(0.122219, 0.207664, 0.0, 0.01033), + float4(0.122219, 0.207664, 0.043263, 0.045182), + float4(-0.861819, 0.507216, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.017305, -0.152503, 0.043263, 0.0), + float4(-0.122219, -0.207664, 0.0, 0.723049), + float4(-0.365879, 0.215335, 0.905408, 0.854994), + float4(-0.0822, 0.531057, -0.043263, 7.0), + float4(-0.011897, -0.099506, 0.043263, 0.01033), + float4(0.0, 0.0, 0.086527, 0.045182), + float4(0.992928, -0.11872, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.054079, -0.118974, 0.052998, 0.0), + float4(0.017305, -0.152503, 0.043263, 0.514916), + float4(-0.41958, 0.203352, 0.884647, 0.863157), + float4(-0.099506, 0.68356, -0.043263, 3.0), + float4(0.086527, 0.149258, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.86514, 0.501531, 0.0, 0.577584), + float4(-0.045426, 0.564586, -0.043263, 11.0), + float4(-0.036774, -0.033529, 0.0, 0.01033), + float4(-0.036774, -0.033529, 0.086527, 0.045182), + float4(0.673754, -0.738956, 0.0, 0.577584), + float4(-0.045426, 0.564586, -0.043263, 7.0), + float4(-0.036774, -0.033529, 0.086527, 0.01033), + float4(0.0, 0.0, 0.096261, 0.045182), + float4(0.673754, -0.738956, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.086527, 0.149258, 0.0, 0.0), + float4(0.054079, -0.118974, 0.052998, 0.262257), + float4(-0.385563, 0.223515, 0.8952, 0.9131), + float4(-0.099506, 0.68356, -0.043263, 11.0), + float4(0.086527, 0.149258, 0.0, 0.01033), + float4(0.086527, 0.149258, 0.043263, 0.045182), + float4(-0.86514, 0.501531, 0.0, 0.577584), + float4(-0.012979, 0.832818, 0.0, 14.0), + float4(0.214153, -0.194685, 0.0822, 0.000305), + float4(-0.032447, -0.268232, 0.052998, 0.930114), + float4(-0.176863, 0.211325, 0.961281, 0.973449), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(-0.019468, -0.024876, -0.063813, 0.168276), + float4(-0.174135, 0.100587, -0.010816, 0.038209), + float4(-0.469238, -0.764881, 0.441331, 0.955978), + float4(0.201174, 0.638133, 0.0822, 14.0), + float4(-0.072466, -0.174135, -0.018387, 0.18117), + float4(-0.246601, -0.073548, -0.029203, 0.082277), + float4(-0.098561, -0.063844, 0.993081, 0.973449), + float4(-0.012979, 0.832818, -0.043263, 7.0), + float4(0.339617, -0.002163, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.006369, 0.99998, 0.0, 0.577584), + float4(0.179543, 0.466162, 0.058405, 10.0), + float4(-0.050834, -0.002163, 0.005408, 0.000305), + float4(0.021632, 0.171972, 0.023795, 0.022171), + float4(0.110896, -0.149888, 0.982464, 0.381323), + float4(-0.012979, 0.832818, 0.0, 14.0), + float4(0.339617, -0.002163, 0.0, 0.0), + float4(0.214153, -0.194685, 0.0822, 0.027314), + float4(0.002492, 0.391296, 0.920261, 0.49102), + float4(0.183869, 0.317985, 0.037855, 10.0), + float4(-0.055161, 0.146014, 0.025958, 0.0), + float4(0.02704, 0.127627, -0.037855, 0.439658), + float4(0.623841, 0.097823, 0.775405, 0.871366), + float4(0.210908, 0.445612, -0.043263, 7.0), + float4(0.0, -0.131953, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.210908, 0.445612, 0.0, 10.0), + float4(-0.031366, 0.02055, 0.058405, 0.000305), + float4(0.035692, 0.014061, 0.06165, 0.022171), + float4(-0.105864, -0.954447, 0.278971, 0.381323), + float4(0.23903, 0.45102, -0.043263, 11.0), + float4(-0.028121, -0.005408, 0.0, 0.01033), + float4(-0.028121, -0.005408, 0.043263, 0.045182), + float4(0.188847, -0.982006, 0.0, 0.577584), + float4(0.23903, 0.45102, -0.043263, 7.0), + float4(-0.028121, -0.005408, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.188847, -0.982006, 0.0, 0.577584), + float4(0.246601, 0.459672, 0.06165, 11.0), + float4(-0.007571, -0.008653, -0.06165, 0.000305), + float4(-0.035692, -0.014061, -0.06165, 0.022171), + float4(0.187618, -0.975617, 0.113888, 0.381323), + float4(0.241193, 0.343943, -0.043263, 11.0), + float4(-0.002163, 0.107077, 0.0, 0.01033), + float4(-0.002163, 0.107077, 0.043263, 0.045182), + float4(-0.999796, -0.020198, 0.0, 0.577584), + float4(0.23903, 0.45102, 0.0, 15.0), + float4(0.007571, 0.008653, 0.06165, 0.000305), + float4(0.042182, -0.108158, 0.046508, 0.846876), + float4(-0.94107, -0.299262, 0.157572, 0.896269), + float4(0.316904, 0.358004, 0.036774, 14.0), + float4(-0.070303, 0.101669, 0.024876, 0.0), + float4(0.023795, 0.116811, -0.036774, 0.473533), + float4(0.523424, 0.157027, 0.837478, 0.930114), + float4(0.340698, 0.474815, -0.043263, 7.0), + float4(0.001082, -0.121137, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.99996, 0.008928, 0.0, 0.577584), + float4(0.376391, 0.381799, -0.043263, 11.0), + float4(-0.001082, 0.107077, 0.0, 0.01033), + float4(-0.001082, 0.107077, 0.043263, 0.045182), + float4(-0.999949, -0.0101, 0.0, 0.577584), + float4(0.376391, 0.381799, -0.043263, 7.0), + float4(-0.001082, 0.107077, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999949, -0.0101, 0.0, 0.577584), + float4(0.375309, 0.488875, 0.0, 14.0), + float4(0.036774, -0.105995, 0.043263, 0.000305), + float4(0.001082, -0.107077, 0.0, 0.879622), + float4(-0.771254, -0.00779, 0.63648, 0.896269), + float4(0.375309, 0.488875, 0.0, 14.0), + float4(0.078955, -0.08869, 0.032447, 0.223224), + float4(0.036774, -0.105995, 0.043263, 0.964691), + float4(0.071225, 0.39802, 0.914608, 0.973449), + float4(0.473733, 0.603523, 0.0, 14.0), + float4(0.0, -0.21199, 0.0, 0.158953), + float4(-0.019468, -0.203337, 0.032447, 0.030716), + float4(0.857493, 0.0, 0.514496, 0.964691), + float4(0.473733, 0.603523, -0.043263, 11.0), + float4(0.0, -0.21199, 0.0, 0.01033), + float4(0.0, -0.21199, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.276885, 0.484549, 0.058405, 10.0), + float4(-0.030284, -0.024876, 0.003245, 0.000305), + float4(-0.097342, -0.018387, 0.0, 0.022171), + float4(-0.03153, 0.166923, 0.985466, 0.381323), + float4(0.246601, 0.459672, 0.06165, 14.0), + float4(0.030284, 0.024876, -0.003245, 0.0), + float4(0.094098, 0.015142, -0.06165, 0.473533), + float4(0.518869, -0.545855, 0.657888, 0.930114), + float4(0.375309, 0.488875, -0.043263, 11.0), + float4(-0.034611, -0.014061, 0.0, 0.01033), + float4(-0.034611, -0.014061, 0.043263, 0.045182), + float4(0.376377, -0.926467, 0.0, 0.577584), + float4(0.340698, 0.474815, 0.0, 14.0), + float4(-0.063813, 0.009734, 0.058405, 0.000305), + float4(0.034611, 0.014061, 0.0, 0.022171), + float4(0.3276, -0.806402, 0.492334, 0.381323), + float4(0.375309, 0.488875, -0.043263, 7.0), + float4(-0.034611, -0.014061, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.376377, -0.926467, 0.0, 0.577584), + float4(0.454264, 0.400185, 0.032447, 12.0), + float4(-0.078955, 0.08869, -0.032447, 0.158953), + float4(0.019468, 0.203337, -0.032447, 0.030716), + float4(-0.201684, 0.173144, 0.964025, 0.964691), + float4(0.201174, 0.638133, 0.0822, 8.0), + float4(0.075711, -0.153585, -0.023795, 0.000305), + float4(-0.021632, -0.171972, -0.023795, 0.022171), + float4(0.026497, -0.140281, 0.989757, 0.381323), + float4(0.473733, 0.603523, -0.043263, 7.0), + float4(0.0, -0.21199, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.276885, 0.484549, 0.058405, 14.0), + float4(-0.075711, 0.153585, 0.023795, 0.000305), + float4(0.098424, 0.004326, -0.058405, 0.022171), + float4(0.50316, 0.115345, 0.856461, 0.381323), + float4(0.375309, 0.488875, 0.0, 10.0), + float4(0.090853, 0.12979, 0.0, 0.158953), + float4(0.098424, 0.114648, 0.0, 0.030716), + float4(0.0, 0.0, 1.0, 0.964691), + float4(0.466162, 0.618665, -0.043263, 11.0), + float4(0.007571, -0.015142, 0.0, 0.01033), + float4(0.007571, -0.015142, 0.043263, 0.045182), + float4(0.894427, 0.447214, 0.0, 0.577584), + float4(0.466162, 0.618665, -0.043263, 3.0), + float4(0.007571, -0.015142, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.894427, 0.447214, 0.0, 0.577584), + float4(0.326638, 0.830655, 0.0, 14.0), + float4(0.139524, -0.21199, 0.0, 0.0), + float4(0.048671, -0.34178, 0.0, 0.015213), + float4(0.0, 0.0, 1.0, 0.428687), + float4(0.326638, 0.830655, 0.0, 14.0), + float4(0.048671, -0.34178, 0.0, 0.007492), + float4(-0.125464, -0.192522, 0.0822, 0.054475), + float4(0.472493, 0.067285, 0.878762, 0.964691), + float4(0.326638, 0.830655, -0.043263, 9.0), + float4(0.139524, -0.21199, 0.0, 0.01033), + float4(0.139524, -0.21199, 0.043263, 0.045182), + float4(0.835314, 0.549773, 0.0, 0.577584), + float4(0.326638, 0.830655, -0.043263, 7.0), + float4(0.139524, -0.21199, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.835314, 0.549773, 0.0, 0.577584), + float4(-0.012979, 0.832818, -0.043263, 11.0), + float4(0.339617, -0.002163, 0.0, 0.01033), + float4(0.339617, -0.002163, 0.043263, 0.045182), + float4(0.006369, 0.99998, 0.0, 0.577584) +}; +constant float4 PRISM_BOXES[44] = { + float4(0.030135, 0.06127, -0.043945, 0.0), + float4(0.330601, 0.483055, 0.116951, 7.0), + float4(0.034429, 0.265258, -0.043945, 7.0), + float4(0.542104, 0.741798, 0.116951, 8.0), + float4(0.034429, 0.355442, -0.043945, 15.0), + float4(0.305908, 0.822319, 0.116951, 8.0), + float4(0.267405, 0.532589, -0.043945, 23.0), + float4(0.542104, 0.822319, 0.082595, 8.0), + float4(-0.045345, 0.036855, -0.044263, 31.0), + float4(0.045345, 0.23354, 0.059405, 8.0), + float4(-0.224888, 0.072548, -0.044263, 39.0), + float4(0.059405, 0.565586, 0.059405, 8.0), + float4(-0.032366, 0.042263, -0.044263, 47.0), + float4(0.148095, 0.408756, 0.059405, 8.0), + float4(0.074711, 0.08769, -0.044263, 55.0), + float4(0.184869, 0.464999, 0.064813, 8.0), + float4(0.146095, 0.271559, -0.044263, 63.0), + float4(0.211908, 0.446612, 0.038855, 8.0), + float4(0.23803, 0.310496, -0.044263, 71.0), + float4(0.320067, 0.460672, 0.06265, 8.0), + float4(0.275885, 0.310496, -0.044263, 79.0), + float4(0.412001, 0.475815, 0.047508, 8.0), + float4(0.375391, 0.350514, -0.044263, 87.0), + float4(0.45202, 0.38388, 0.044263, 4.0), + float4(0.410001, 0.350514, -0.044263, 91.0), + float4(0.474733, 0.401185, 0.044263, 5.0), + float4(-0.224888, 0.379717, -0.044263, 96.0), + float4(-0.0812, 0.532057, 0.047508, 8.0), + float4(-0.095098, 0.228295, -0.044263, 104.0), + float4(0.211908, 0.565586, 0.064813, 8.0), + float4(-0.222724, 0.430551, -0.044263, 112.0), + float4(-0.011979, 0.833818, 0.053998, 8.0), + float4(-0.100506, 0.438122, -0.044263, 120.0), + float4(0.327638, 0.833818, 0.0832, 8.0), + float4(0.127708, 0.312659, -0.044263, 128.0), + float4(0.282211, 0.467162, 0.064813, 8.0), + float4(0.245601, 0.352677, -0.044263, 136.0), + float4(0.474733, 0.604523, 0.06265, 8.0), + float4(0.178543, 0.390533, -0.044263, 144.0), + float4(0.474733, 0.639133, 0.0832, 8.0), + float4(0.200174, 0.483549, -0.044263, 152.0), + float4(0.474733, 0.639133, 0.0832, 4.0), + float4(-0.013979, 0.487875, -0.044263, 156.0), + float4(0.467162, 0.833818, 0.0832, 5.0) +}; +constant float2 PRISM_POLY[110] = { + float2(0.006442, -0.002147), + float2(-0.001074, 0.001074), + float2(-0.004294, 0.007515), + float2(-0.004294, 0.806288), + float2(-0.001074, 0.809509), + float2(0.004294, 0.809509), + float2(0.022546, 0.796626), + float2(0.185736, 0.655982), + float2(0.281288, 0.86319), + float2(0.286656, 0.870706), + float2(0.293098, 0.871779), + float2(0.465951, 0.781595), + float2(0.478834, 0.773006), + float2(0.478834, 0.763344), + float2(0.407975, 0.592638), + float2(0.45092, 0.586196), + float2(0.601227, 0.573313), + float2(0.606595, 0.569018), + float2(0.606595, 0.564724), + float2(0.601227, 0.556135), + float2(0.50138, 0.453067), + float2(0.425153, 0.37684), + float2(0.289877, 0.246933), + float2(0.081595, 0.057975), + float2(0.013957, 0.0), + float2(-0.011897, 0.0), + float2(-0.055161, 0.037855), + float2(-0.081119, 0.063813), + float2(-0.083282, 0.06814), + float2(-0.084363, 0.38288), + float2(-0.089771, 0.38288), + float2(-0.158993, 0.33529), + float2(-0.166564, 0.336372), + float2(-0.250927, 0.379635), + float2(-0.261743, 0.388288), + float2(-0.263906, 0.461836), + float2(-0.262824, 0.486712), + float2(-0.173053, 0.638133), + float2(-0.044345, 0.860939), + float2(-0.036774, 0.867429), + float2(-0.024876, 0.871755), + float2(0.33529, 0.871755), + float2(0.349351, 0.867429), + float2(0.363412, 0.853368), + float2(0.513752, 0.619747), + float2(0.513752, 0.379635), + float2(0.466162, 0.322311), + float2(0.461836, 0.320148), + float2(0.399104, 0.311496), + float2(0.393696, 0.313659), + float2(0.374227, 0.332046), + float2(0.343943, 0.288782), + float2(0.336372, 0.282293), + float2(0.269314, 0.271477), + float2(0.263906, 0.27364), + float2(0.243356, 0.292027), + float2(0.21199, 0.244438), + float2(0.207664, 0.241193), + float2(0.149258, 0.231459), + float2(0.138443, 0.231459), + float2(0.118974, 0.248764), + float2(0.116811, 0.222806), + float2(0.117892, 0.162237), + float2(0.115729, 0.076792), + float2(0.06814, 0.011897), + float2(0.062732, 0.007571), + float2(0.005408, 0.0), + float2(0.031135, 0.06227), + float2(0.252301, 0.266258), + float2(0.541104, 0.544325), + float2(0.359663, 0.559356), + float2(0.433742, 0.753681), + float2(0.304908, 0.821319), + float2(0.199693, 0.595859), + float2(0.035429, 0.740798), + float2(0.035429, 0.482055), + float2(-0.043263, 0.077874), + float2(0.001082, 0.037855), + float2(0.044345, 0.043263), + float2(0.075711, 0.08869), + float2(0.077874, 0.407756), + float2(0.10924, 0.439122), + float2(0.111403, 0.307169), + float2(0.147095, 0.272559), + float2(0.189277, 0.28013), + float2(0.210909, 0.313659), + float2(0.210909, 0.445612), + float2(0.23903, 0.45102), + float2(0.241193, 0.343943), + float2(0.276885, 0.311496), + float2(0.319067, 0.319067), + float2(0.34178, 0.353677), + float2(0.340698, 0.474815), + float2(0.375309, 0.488875), + float2(0.376391, 0.381799), + float2(0.411001, 0.351514), + float2(0.45102, 0.359085), + float2(0.473733, 0.391533), + float2(0.473733, 0.603523), + float2(0.466162, 0.618665), + float2(0.326638, 0.830655), + float2(-0.012979, 0.832818), + float2(-0.099506, 0.68356), + float2(-0.221724, 0.475896), + float2(-0.223888, 0.407756), + float2(-0.163319, 0.380717), + float2(-0.094098, 0.431551), + float2(-0.0822, 0.531057), + float2(-0.045426, 0.564586), + float2(-0.044345, 0.23254) +}; +constant float PRISM_BEVEL = 0.012883; +// end of the prism mesh -static float s_gem_arrow(float3 p) -{ - float z0 = SCULPT_HOVER + 0.03; - float slab = max(p.z - z0 - 0.2, z0 - 0.03 - p.z); - float head = max(slab, max(s_gem_edge(p, float2(-0.02, 0.03), float2(-0.02, -0.88), z0), - max(s_gem_edge(p, float2(-0.02, -0.88), float2(0.66, -0.61), z0), - s_gem_edge(p, float2(0.66, -0.61), float2(-0.02, 0.03), z0)))); - float tail = max(slab, max(max(s_gem_edge(p, float2(0.215, -0.665), float2(0.37, -1.0), z0), - s_gem_edge(p, float2(0.37, -1.0), float2(0.53, -0.93), z0)), - max(s_gem_edge(p, float2(0.53, -0.93), float2(0.38, -0.60), z0), - s_gem_edge(p, float2(0.38, -0.60), float2(0.215, -0.665), z0)))); - return min(head, tail); -} +constant float PRISM_IOR = 1.61; +constant float PRISM_DISPERSION = 0.035; +constant float PRISM_GLOW = 0.45; +constant float PRISM_FOLD = 0.35; +constant int PRISM_BOUNCES = 6; +constant float PRISM_MAT = 9.0; -static float s_facet_capsule(float3 p, float3 a, float3 b, float r, float spin) +inline int prism_shape(constant Layer &layer) { - float3 u = normalize(b - a); - float3 v = normalize(cross(u, float3(0.0, 0.0, 1.0))); - float3 w = cross(u, v); - float3 q = p - b; - float h = dot(q, u); - float2 rad = float2(dot(q, v), dot(q, w)); - float d = max(h - r, -dot(p - a, u) - r); - for (int k = 0; k < 6; k++) - { - float an = spin + float(k) * 1.0471976; - float s = dot(rad, float2(cos(an), sin(an))); - d = max(d, s - r); - d = max(d, dot(rad, float2(cos(an + 0.5236), sin(an + 0.5236))) * 0.8660 + h * 0.5 - r); - d = max(d, s * 0.5 + h * 0.8660 - r); - } - return d; + int id = sculpt_id(layer); + return id == 3 || id == 4 ? id - 3 : -1; } -constant float3 SCULPT_FACETS[10] = { - float3(1.0, 0.0, 0.0), float3(0.0, 1.0, 0.0), float3(0.0, 0.0, 1.0), - float3(0.7071, 0.7071, 0.0), float3(0.7071, 0.0, 0.7071), float3(0.0, 0.7071, 0.7071), - float3(0.5774, 0.5774, 0.5774), float3(0.4472, 0.0, 0.8944), float3(0.0, 0.4472, 0.8944), - float3(0.3015, 0.3015, 0.9045) -}; - -static float s_facet_ellipsoid(float3 p, float3 r) +static float prism_poly_dist(float2 p, int i0, int n) { - float3 q = abs(p); - float d = -1e9; - for (int i = 0; i < 10; i++) + float2 vj = PRISM_POLY[i0 + n - 1]; + float d = 1e9; + float sgn = 1.0; + for (int i = 0; i < n; i++) { - d = max(d, dot(q, SCULPT_FACETS[i]) - length(r * SCULPT_FACETS[i])); + float2 vi = PRISM_POLY[i0 + i]; + float2 e = vj - vi; + float2 w = p - vi; + float2 b = w - e * saturate(dot(w, e) / dot(e, e)); + d = min(d, dot(b, b)); + bool c0 = p.y >= vi.y; + bool c1 = p.y < vj.y; + bool c2 = e.x * w.y > e.y * w.x; + if ((c0 && c1 && c2) || (!c0 && !c1 && !c2)) + { + sgn = -sgn; + } + vj = vi; } - return d; + return sgn * sqrt(d); } -static float s_crystal_hand(float3 p) +static float prism_rim(float3 p, int shape, constant Layer &layer) { - float zc = SCULPT_HAND_ZC; - float d = s_facet_ellipsoid(p - float3(0.185, -0.70, zc), float3(0.3, 0.235, 0.125)); - d = min(d, s_facet_capsule(p, float3(0.0, -0.52, zc), float3(0.0, -0.10, zc), 0.098, 0.3)); - d = min(d, s_facet_capsule(p, float3(0.17, -0.57, zc + 0.012), float3(0.17, -0.41, zc + 0.045), 0.086, 0.1)); - d = min(d, s_facet_capsule(p, float3(0.31, -0.59, zc + 0.01), float3(0.31, -0.45, zc + 0.04), 0.08, 0.5)); - d = min(d, s_facet_capsule(p, float3(0.435, -0.625, zc + 0.005), float3(0.435, -0.52, zc + 0.03), 0.07, 0.2)); - return min(d, s_facet_capsule(p, float3(0.03, -0.77, zc + 0.03), float3(-0.165, -0.60, zc + 0.065), 0.082, 0.7)); + float half_t = model_thick(layer) * 0.5; + float sil = prism_poly_dist(p.xy, PRISM_SIL_START[shape], PRISM_SIL_COUNT[shape]); + float2 w = float2(sil + PRISM_BEVEL, abs(p.z + half_t) - (half_t - PRISM_BEVEL)); + return min(max(w.x, w.y), 0.0) + length(max(w, 0.0)) - PRISM_BEVEL; } -static float2 sculpt_proto(float3 p, int theme, int shape) +static float prism_tri(float3 ro, float3 rd, int k) { - if (theme == 3) + float3 v0 = PRISM_TRIS[4 * k].xyz; + float3 e1 = PRISM_TRIS[4 * k + 1].xyz; + float3 e2 = PRISM_TRIS[4 * k + 2].xyz; + float3 pv = cross(rd, e2); + float det = dot(e1, pv); + if (abs(det) < 1e-10) { - return s_voxels(p, shape); + return -1.0; } - if (theme == 1) + float inv = 1.0 / det; + float3 sv = ro - v0; + float u = dot(sv, pv) * inv; + float3 qv = cross(sv, e1); + float v = dot(rd, qv) * inv; + if (u < -2e-4 || v < -2e-4 || u + v > 1.0004) { - return float2(shape == 0 ? s_gem_arrow(p) : s_crystal_hand(p), 8.0); + return -1.0; } - float2 body; - float3 star; - if (shape == 0) - { - float h = 0.075, re = 0.03, dome = 0.0; - if (theme == 2) - { - h = 0.07; - re = 0.06; - dome = 0.035; - } - if (theme == 4) + return dot(e2, qv) * inv; +} + +static float prism_trace(float3 ro, float3 rd, int shape, thread int &hit) +{ + float3 inv = 1.0 / select(float3(1e-6), rd, abs(rd) > 1e-6); + int b0 = PRISM_BOX_START[shape]; + int nb = PRISM_BOX_COUNT[shape]; + float best = 1e9; + hit = -1; + for (int j = b0; j < b0 + nb; j++) + { + float4 lo = PRISM_BOXES[2 * j]; + float4 hi = PRISM_BOXES[2 * j + 1]; + float3 t0 = (lo.xyz - ro) * inv; + float3 t1 = (hi.xyz - ro) * inv; + float3 tn = min(t0, t1); + float3 tf = max(t0, t1); + float tmin = max(max(tn.x, tn.y), tn.z); + float tmax = min(min(tf.x, tf.y), tf.z); + if (tmax < max(tmin, 1e-5) || tmin > best) { - h = 0.07; - re = 0.055; - dome = 0.025; + continue; } - body = float2(s_arrow_solid(p, h, re, dome), 1.0); - if (theme == 0) + int k0 = int(lo.w); + int n = int(hi.w); + for (int i = k0; i < k0 + n; i++) { - body = s_opu(body, float2(s_piping(p, SCULPT_HOVER + 2.0 * h - 0.004), 2.0)); + float t = prism_tri(ro, rd, i); + if (t > 1e-5 && t < best) + { + best = t; + hit = i; + } } - star = float3(0.57, -0.86, SCULPT_HOVER + 2.0 * h + dome); } - else + return best; +} + +static float2 sculpt_proto(float3 p, int theme, int shape) +{ + if (theme == 3) { - body = s_opu(float2(s_glove(p, SCULPT_HAND_ZC), 1.0), float2(s_cuff(p, SCULPT_HAND_ZC), 2.0)); - star = float3(0.185, -0.93, SCULPT_HAND_ZC + 0.15); + return s_voxels(p, shape); } - return theme == 4 ? s_opu(body, s_sprout(p, star)) : body; + return s_rimmed(p, theme, shape); } inline float3 sculpt_point(float3 q, constant Layer &layer) @@ -961,17 +1960,21 @@ static float2 sculpt_eval(float3 q, bool occ, constant Layer &layer) { int id = sculpt_id(layer) - 1; int theme = id / 2, shape = id % 2; + if (theme == 1) + { + return float2(prism_rim(q, shape, layer), 8.0); + } float3 p = sculpt_point(q, layer); float2 r; if (occ && theme == 3) { - float d2 = shape == 0 ? s_arrow2(p.xy) - 1.2 * SCULPT_VOX : s_pixel_hand_dist(p.xy); + float d2 = s_pixel_outline(p.xy, shape); float dz = abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07; r = float2(length(max(float2(d2, dz), 0.0)) + min(max(d2, dz), 0.0), 7.0); } else { - r = sculpt_proto(p, occ && theme == 1 ? 2 : theme, shape); + r = sculpt_proto(p, theme, shape); } return float2(r.x * sculpt_units(layer), r.y); } @@ -1076,16 +2079,13 @@ inline SculptMat s_mat(float3 alb, float rough, float spec, float sss, float ref static float3 s_pixel_colour(float3 p, int shape) { - float2 id = floor((p.xy - s_grid_origin(shape)) / SCULPT_VOX); - if (!s_occ(id + float2(-1.0, 0.0), shape) || !s_occ(id + float2(0.0, -1.0), shape)) - { - return s_lin(0.52, 0.91, 0.77); - } - if (!s_occ(id + float2(1.0, 0.0), shape) || !s_occ(id + float2(0.0, 1.0), shape)) - { - return s_lin(1.0, 0.78, 0.87); - } - return s_lin(1.0, 0.50, 0.71); + int2 cell = s_pix_cell(p.xy, shape); + int4 g = PIX_GRID[shape]; + int row = g.z + clamp(cell.y, 0, g.y - 1); + if (s_pix_bit(PIX_LINE[row], cell.x)) return s_lin(0.29, 0.12, 0.36); + if (s_pix_bit(PIX_HI[row], cell.x)) return s_lin(1.0, 0.78, 0.87); + if (s_pix_bit(PIX_SHADE[row], cell.x)) return s_lin(0.87, 0.27, 0.51); + return s_lin(1.0, 0.435, 0.66); } static SculptMat sculpt_material(float mat, float3 p, int theme, int shape) @@ -1093,28 +2093,26 @@ static SculptMat sculpt_material(float mat, float3 p, int theme, int shape) bool primary = mat < 1.5; if (theme == 0) { - if (shape == 0 && primary) return s_mat(s_lin(0.10, 0.10, 0.115), 0.3, 0.2, 0.0, 0.9); - if (shape == 0) return s_mat(s_lin(0.94, 0.91, 0.84), 0.45, 0.4, 0.2, 0.3); - if (primary) return s_mat(s_lin(0.95, 0.92, 0.85), 0.55, 0.35, 0.35, 0.25); - return s_mat(s_lin(0.17, 0.18, 0.22), 0.35, 0.6, 0.0, 0.6); + if (mat < 2.5) return s_mat(s_lin(0.95, 0.92, 0.85), 0.3, 0.6, 0.35, 0.25); + return s_mat(s_lin(0.1, 0.1, 0.11), 0.55, 0.05, 0.0, 0.1); } + if (theme == 1) return s_mat(s_lin(0.02, 0.05, 0.33), 0.35, 0.5, 0.05, 0.3); if (theme == 2) { - if (shape == 0) return s_mat(s_lin(1.0, 0.40, 0.30), 0.5, 0.45, 0.4, 0.25); - if (primary) return s_mat(s_lin(1.0, 0.79, 0.16), 0.5, 0.45, 0.4, 0.25); - return s_mat(s_lin(0.18, 0.20, 0.29), 0.4, 0.5, 0.0, 0.4); + if ((primary && shape == 0) || (mat > 2.5 && mat < 3.5 && shape == 1)) return s_mat(s_lin(1.0, 0.40, 0.30), 0.85, 0.08, 0.15, 0.03); + if (mat < 3.5) return s_mat(s_lin(1.0, 0.80, 0.10), 0.85, 0.08, 0.15, 0.03); + return s_mat(s_lin(0.09, 0.13, 0.45), 0.9, 0.05, 0.05, 0.02); } if (theme == 4) { - if (primary && shape == 0) return s_mat(s_lin(0.62, 0.91, 0.78), 0.22, 0.8, 0.25, 0.6); - if (primary) return s_mat(s_lin(0.96, 0.94, 0.88), 0.5, 0.35, 0.35, 0.25); - if (mat < 2.5) return s_mat(s_lin(0.62, 0.91, 0.78), 0.25, 0.7, 0.25, 0.5); - if (mat < 3.5) return s_mat(s_lin(1.0, 0.80, 0.20), 0.3, 0.6, 0.3, 0.4); - if (mat < 4.5) return s_mat(s_lin(0.38, 0.80, 0.55), 0.35, 0.5, 0.35, 0.3); - return s_mat(s_lin(0.16, 0.12, 0.10), 0.2, 0.8, 0.0, 0.5); + if (primary && shape == 0) return s_mat(s_lin(0.68, 0.93, 0.80), 0.25, 0.7, 0.3, 0.3); + if (primary) return s_mat(s_lin(0.97, 0.95, 0.90), 0.3, 0.6, 0.35, 0.25); + if (mat < 2.5) return s_mat(s_lin(0.52, 0.87, 0.78), 0.25, 0.7, 0.3, 0.3); + if (mat < 3.5) return s_mat(s_lin(1.0, 0.75, 0.25), 0.25, 0.7, 0.3, 0.3); + if (mat < 4.5) return s_mat(s_lin(0.62, 0.92, 0.72), 0.3, 0.6, 0.3, 0.35); + return s_mat(s_lin(0.07, 0.15, 0.33), 0.7, 0.15, 0.05, 0.08); } - if (mat < 6.5) return s_mat(s_pixel_colour(p, shape), 0.45, 0.35, 0.15, 0.2); - return s_mat(s_lin(0.36, 0.18, 0.54), 0.45, 0.35, 0.1, 0.2); + return s_mat(s_pixel_colour(p, shape), 0.65, 0.1, 0.12, 0.04); } static float3 model_env(float3 d, float rough, float3 l, float3 fill) @@ -1127,34 +2125,6 @@ static float3 model_env(float3 d, float rough, float3 l, float3 fill) return col; } -static float3 s_gem(float k) -{ - k = saturate(k) * 3.0; - float3 a = s_lin(0.20, 0.95, 1.0); - float3 b = s_lin(0.15, 0.42, 1.0); - float3 c = s_lin(0.45, 0.25, 0.95); - float3 d = s_lin(0.88, 0.50, 1.0); - if (k < 1.0) return mix(a, b, k); - if (k < 2.0) return mix(b, c, k - 1.0); - return mix(c, d, k - 2.0); -} - -static float3 model_shade_crystal(float3 n, float3 rd, float3 L, float fall, float3 l, float3 fill) -{ - float cosi = saturate(dot(-rd, n)); - float F = 0.04 + 0.96 * pow(1.0 - cosi, 5.0); - float3 t = refract(rd, n, 1.0 / 1.6); - float k = 0.42 + 0.9 * dot(float2(n.x, -n.y), float2(0.7557, -0.6549)) - + 0.6 * dot(float2(t.x, -t.y), float2(0.6, -0.8)); - float3 body = float3(s_gem(k - 0.08).r, s_gem(k).g, s_gem(k + 0.08).b); - float3 col = body * (0.1 + 1.8 * fall * pow(max(dot(n, L), 0.0), 2.5)); - col += body * model_env(reflect(t, float3(0.0, 0.0, 1.0)) * float3(1.0, 1.0, -1.0), 0.15, l, fill) * 0.5; - col += pow(max(dot(reflect(rd, n), L), 0.0), 30.0) * 2.0; - col += model_env(reflect(rd, n), 0.05, l, fill) * F; - col += s_lin(0.5, 0.9, 1.0) * pow(1.0 - cosi, 4.0) * 0.6; - return col; -} - static float3 model_tonemap(float3 c) { const float start = 0.76; @@ -1178,16 +2148,16 @@ static float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, f texture2d texImg) { int id = sculpt_id(layer); - if (id > 0 && (id - 1) / 2 == 1) - { - return model_tonemap(model_shade_crystal(n, rd, L, fall, l, fill)); - } SculptMat m; float gloss = 1.0; if (id > 0) { float3 p = sculpt_point(q, layer) - float3(n.x, -n.y, n.z) * 0.01; m = sculpt_material(mat, p, (id - 1) / 2, (id - 1) % 2); + if (id == 3 || id == 4) + { + gloss = 1.0 - smoothstep(0.97, 0.995, abs(n.z)); + } } else { @@ -1211,13 +2181,168 @@ static float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, f return model_tonemap(col); } +// ---- Le cristal de Prism Glow ---- (cf. HLSL) + +inline float prism_fresnel(float c, float ior) +{ + float f0 = (ior - 1.0) / (ior + 1.0); + f0 *= f0; + return f0 + (1.0 - f0) * pow(1.0 - c, 5.0); +} + +static float prism_primary(float3 ro, float3 rd, thread int &hc, constant Layer &layer) +{ + hc = -1; + int shape = prism_shape(layer); + if (shape < 0) + { + return 1e9; + } + float t = prism_trace(ro, rd, shape, hc); + if (hc >= 0 && fmod(PRISM_TRIS[4 * hc].w, 2.0) > 0.5 && ro.z + rd.z * t < 0.0) + { + hc = -1; + return 1e9; + } + return t; +} + +inline float3 prism_normal(int h, float3 rd) +{ + float3 n = PRISM_TRIS[4 * h + 3].xyz; + return dot(n, rd) > 0.0 ? -n : n; +} + +inline float prism_seg(float3 p, float3 a, float3 b) +{ + float3 pa = p - a, ba = b - a; + return length(pa - ba * saturate(dot(pa, ba) / dot(ba, ba))); +} + +static float prism_edge_px(float3 p, int h, float px) +{ + float4 r0 = PRISM_TRIS[4 * h]; + float3 a = r0.xyz; + float3 b = a + PRISM_TRIS[4 * h + 1].xyz; + float3 c = a + PRISM_TRIS[4 * h + 2].xyz; + int bits = int(r0.w * 0.5); + float d = 1e9; + if ((bits & 1) != 0) d = min(d, prism_seg(p, b, c)); + if ((bits & 2) != 0) d = min(d, prism_seg(p, c, a)); + if ((bits & 4) != 0) d = min(d, prism_seg(p, a, b)); + return d / px; +} + +static float3 plane_to_world(float3 v, ModelFrame f) +{ + float x = v.x * f.c.z - v.y * f.s.z; + float y = v.x * f.s.z + v.y * f.c.z; + float z = -x * f.s.y + v.z * f.c.y; + return float3(x * f.c.y + v.z * f.s.y, y * f.c.x - z * f.s.x, y * f.s.x + z * f.c.x); +} + +static float3 prism_screen(float3 q, float3 d, float3 nz, float hz, float3 tip, float unit, ModelFrame f, + constant Layer &layer, texture2d texFrame) +{ + float denom = dot(d, nz); + if (denom > -1e-4) + { + return SCULPT_SCREEN * 0.2; + } + float3 g = q + d * ((hz - dot(q, nz)) / denom); + float3 w = plane_to_world(tip + unit * model_to_plane(g, f), f); + float k = 1.0 - w.z / layer.src.z; + if (k < 1e-3) + { + return SCULPT_SCREEN * 0.2; + } + float2 uv = layer.dst.xy + ((w.xy + layer.mb.zw) / k - layer.src.xy) / layer.quad_px * layer.dst.zw; + if (any(uv < 0.0) || any(uv > 1.0)) + { + return SCULPT_SCREEN * 0.2; + } + return pow(texFrame.sample(samp, uv, level(0.0)).rgb, float3(2.2)); +} + +static float3 prism_shade(float3 p, float3 n, float3 rd, int h, float px, float3 l, float3 fill, float3 nz, + float hz, float3 tip, float unit, ModelFrame f, constant Layer &layer, + texture2d texFrame) +{ + int shape = prism_shape(layer); + float4 r3 = PRISM_TRIS[4 * h + 3]; + float3 facet = float3(PRISM_TRIS[4 * h + 1].w, PRISM_TRIS[4 * h + 2].w, r3.w); + float F = prism_fresnel(saturate(-dot(rd, n)), PRISM_IOR); + float zb = -model_thick(layer); + float3 trans = float3(0.0); + for (int ch = 0; ch < 3; ch++) + { + float3 mask = float3(float(ch == 0), float(ch == 1), float(ch == 2)); + float ior = PRISM_IOR + PRISM_DISPERSION * float(ch - 1); + float3 d = refract(rd, n, 1.0 / ior); + float3 pos = p; + float thr = 1.0; + float acc = 0.0; + for (int b = 0; b < PRISM_BOUNCES; b++) + { + int hh; + float th = prism_trace(pos, d, shape, hh); + float tb = d.z < -1e-6 ? (zb - pos.z) / d.z : 1e9; + if (tb <= th) + { + float3 qb = pos + d * tb; + float3 d2 = refract(d, float3(0.0, 0.0, 1.0), ior); + if (dot(d2, d2) < 1e-8) + { + pos = qb; + d = reflect(d, float3(0.0, 0.0, 1.0)); + continue; + } + acc += thr * dot(prism_screen(qb, d2, nz, hz, tip, unit, f, layer, texFrame), mask); + thr = 0.0; + break; + } + if (hh < 0) + { + break; + } + pos += d * th; + float4 rn = PRISM_TRIS[4 * hh + 3]; + if (fmod(PRISM_TRIS[4 * hh].w, 2.0) > 0.5 && pos.z < 0.0) + { + acc += thr * dot(s_lin(0.03, 0.06, 0.37) * 0.35, mask); + thr = 0.0; + break; + } + float3 dout = refract(d, -rn.xyz, ior); + if (dot(dout, dout) < 1e-8) + { + d = reflect(d, -rn.xyz); + continue; + } + float fi = prism_fresnel(saturate(dot(dout, rn.xyz)), ior); + acc += thr * (1.0 - fi) * dot(model_env(dout, 0.05, l, fill), mask); + thr *= fi; + d = reflect(d, -rn.xyz); + } + trans += acc * mask; + } + float3 col = F * model_env(reflect(rd, n), 0.05, l, fill) + (1.0 - F) * trans + facet * PRISM_GLOW; + float fold = 1.0 - smoothstep(0.2, 1.0, prism_edge_px(p, h, px)); + col = mix(col, s_lin(0.96, 0.99, 1.0) * 1.3, PRISM_FOLD * fold * (0.35 + 0.65 * saturate(dot(n, l)))); + return model_tonemap(col); +} + // Les passes de la boucle unique de `cursor_model` (cf. HLSL : un seul appel de `model_eval`). constant int STAGE_MARCH = 0; constant int STAGE_NORMAL = 1; constant int STAGE_AO = 2; constant int STAGE_SELF = 3; constant int STAGE_PLANE = 4; -constant int STAGE_DONE = 5; +constant int STAGE_EDGE = 5; +constant int STAGE_SHADE = 6; +constant int STAGE_DONE = 7; + +constant float2 MODEL_SUBPIXEL[3] = { float2(0.35, 0.2), float2(-0.35, 0.2), float2(0.0, -0.4) }; inline float3 model_tetra(int k) { @@ -1226,7 +2351,8 @@ inline float3 model_tetra(int k) static float4 cursor_model(float2 local, constant Layer &layer, texture2d texSdf, - texture2d texImg) + texture2d texImg, + texture2d texFrame) { ModelFrame f; f.c = cos(layer.fx.xyz); @@ -1253,15 +2379,21 @@ static float4 cursor_model(float2 local, constant Layer &layer, float stride = sculpt_id(layer) > 0 ? 0.85 : 1.0; float3 lamp = float3(layer.color.rg + sprite_size(layer) * 0.5, 0.0) + l * SCULPT_LAMP_DIST; - float2 tb = ray_box(ro, rd, lo - 0.02, hi + 0.02); + float3 ray = rd; + float2 tb = ray_box(ro, ray, lo - 0.02, hi + 0.02); int stage = tb.x < tb.y && tb.y > 0.0 ? STAGE_MARCH : STAGE_DONE; bool plane_next = stage == STAGE_DONE; + bool need_tc = stage == STAGE_MARCH; + float tc = 1e9; + int hc = -1; int k = 0; float t = max(tb.x, 0.0); float best = 1e9; float t_best = t; + float d_best = 0.0; float mat = 0.0; float cov = 0.0; + float cov_s = 0.0; float3 q = float3(0.0); float3 n = float3(0.0); float3 L = float3(0.0); @@ -1273,12 +2405,20 @@ static float4 cursor_model(float2 local, constant Layer &layer, float inside = 0.0; float contact = 0.0; float dropped = 0.0; - for (int it = 0; it < 180; it++) + int sub = 0; + int shaded = 0; + float4 acc = float4(0.0); + for (int it = 0; it < 500; it++) { + if (need_tc) + { + need_tc = false; + tc = prism_primary(ro, ray, hc, layer); + } if (plane_next) { plane_next = false; - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; float denom = dot(rd, nz); if (cov < 1.0 && denom < -1e-4) { @@ -1293,18 +2433,64 @@ static float4 cursor_model(float2 local, constant Layer &layer, } } } + if (stage == STAGE_SHADE) + { + if (cov_s > 0.0) + { + float3 c; + if (mat > PRISM_MAT - 0.5) + { + c = prism_shade(q, n, ray, hc, t_best / dlen, l, fill, nz, hz, tip, unit, f, layer, texFrame); + } + else + { + float3 tl = lamp - q; + float fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); + c = model_shade(q, n, ray, normalize(tl), fall, sh, ao, mat, l, fill, layer, texSdf, texImg); + } + acc += float4(c * cov_s, cov_s); + } + shaded++; + stage = STAGE_DONE; + if (sub > 0) + { + float3 dws = float3(local + MODEL_SUBPIXEL[3 - sub] + layer.src.xy, -persp); + sub--; + dlen = length(dws); + ray = plane_to_model(world_to_plane(dws / dlen, f), f); + tb = ray_box(ro, ray, lo - 0.02, hi + 0.02); + cov_s = 0.0; + stage = STAGE_SHADE; + if (tb.x < tb.y && tb.y > 0.0) + { + stage = STAGE_MARCH; + k = 0; + t = max(tb.x, 0.0); + best = 1e9; + t_best = t; + need_tc = true; + } + } + continue; + } float3 pos; if (stage == STAGE_MARCH) { - pos = ro + rd * t; + pos = ro + ray * t; } else if (stage == STAGE_NORMAL) { pos = q + 0.002 * model_tetra(k); } + else if (stage == STAGE_EDGE) + { + float3 side = normalize(cross(n, ray)); + float3 e = (k >= 2 ? cross(n, side) : side) * (k % 2 == 1 ? -1.0 : 1.0); + pos = q + e * 0.5 * t_best / dlen; + } else if (stage == STAGE_AO) { - pos = q + (0.01 + 0.0175 * float(k)) * SCULPT_SCALE * n; + pos = q + (0.01 + 0.0175 * k) * SCULPT_SCALE * n; } else if (stage == STAGE_SELF) { @@ -1328,22 +2514,64 @@ static float4 cursor_model(float2 local, constant Layer &layer, { best = hit ? 0.0 : d / fp; t_best = t; + d_best = d; mat = m.y; } t += d * stride; k++; + if (!hit && t >= tc) + { + hit = true; + best = 0.0; + t_best = tc; + d_best = 0.0; + mat = PRISM_MAT; + } if (hit || t > tb.y || k == 96) { - cov = saturate(1.0 - best); - if (cov > 0.0) + cov_s = saturate(1.0 - best); + if (shaded > 0) { - q = ro + rd * t_best; - stage = STAGE_NORMAL; - k = 0; + stage = STAGE_SHADE; + if (cov_s > 0.0) + { + q = ro + ray * t_best; + n = float3(0.0); + stage = STAGE_NORMAL; + k = 0; + } } else { - plane_next = true; + cov = cov_s; + if (cov > 0.0) + { + q = ro + ray * t_best; + stage = STAGE_NORMAL; + k = 0; + } + else + { + plane_next = true; + } + } + if (stage == STAGE_NORMAL && mat > PRISM_MAT - 0.5) + { + n = prism_normal(hc, ray); + k = 0; + if (shaded > 0) + { + stage = STAGE_SHADE; + } + else + { + if (prism_edge_px(q, hc, t_best / dlen) < 0.75) + { + sub = 3; + } + stage = STAGE_DONE; + plane_next = true; + } } } } @@ -1354,6 +2582,31 @@ static float4 cursor_model(float2 local, constant Layer &layer, if (k == 4) { n = normalize(n); + k = 0; + stage = shaded > 0 ? STAGE_SHADE : STAGE_EDGE; + } + } + else if (stage == STAGE_EDGE) + { + if (m.y != mat || abs(d - d_best) > 0.02 * t_best / dlen) + { + sub = 3; + } + if (k == 0 && mat > 7.5) + { + int ps = prism_shape(layer); + if (ps >= 0) + { + float dout = prism_poly_dist(q.xy, PRISM_OUT_START[ps], PRISM_OUT_COUNT[ps]); + if (abs(dout) < 0.75 * t_best / dlen) + { + sub = 3; + } + } + } + k++; + if (k == 4) + { stage = STAGE_AO; k = 0; ao = 0.0; @@ -1361,7 +2614,7 @@ static float4 cursor_model(float2 local, constant Layer &layer, } else if (stage == STAGE_AO) { - ao += ((0.01 + 0.0175 * float(k)) * SCULPT_SCALE - d) * pow(0.85, float(k)); + ao += ((0.01 + 0.0175 * k) * SCULPT_SCALE - d) * pow(0.85, float(k)); k++; if (k == 5) { @@ -1402,7 +2655,7 @@ static float4 cursor_model(float2 local, constant Layer &layer, k = 1; if (!(tbp.x < tbp.y && tbp.y > 0.0)) { - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; } } else @@ -1414,21 +2667,15 @@ static float4 cursor_model(float2 local, constant Layer &layer, { res = saturate(res); dropped = 1.0 - res * res * (3.0 - 2.0 * res); - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; } } } - float3 rgb = float3(0.0); - if (cov > 0.0) - { - float3 tl = lamp - q; - float fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); - rgb = model_shade(q, n, rd, normalize(tl), fall, sh, ao, mat, l, fill, layer, texSdf, texImg); - } + float w = 1.0 / max(shaded, 1); float shadow = inside * max(dropped * MODEL_SHADOW_ALPHA, contact * MODEL_CONTACT_ALPHA); - float a = cov * layer.color.a; - return float4(rgb * a, a + (1.0 - a) * shadow * layer.color.a); // prémultiplié, ombre noire + float a = acc.a * w * layer.color.a; + return float4(acc.rgb * w * layer.color.a, a + (1.0 - a) * shadow * layer.color.a); // prémultiplié, ombre noire } // ============ Impact du clic (mode 16) ============ @@ -2009,22 +3256,16 @@ static float4 device_frame(float2 local, constant Layer &layer) return float4(rgb * a, a); // prémultiplié } -fragment float4 ps_main(VSOut i [[stage_in]], - constant Layer &layer [[buffer(0)]], - texture2d texY [[texture(0)]], - texture2d texUV [[texture(1)]], - texture2d texImg [[texture(2)]], - // Masque de segmentation du sujet webcam. Non lie tant qu'aucun - // masque n'existe : Metal rend alors 0, ce qui est sans effet - // puisque la branche n'est prise que si layer.fx.z > 0.5. - texture2d texMask [[texture(3)]], - // Champ de distance du sprite de curseur (mode 15 seulement), R16F, cf. - // `cursor_sdf.rs`. Le sprite lui-même est en texture(2), comme aux - // modes 7 et 13. - texture2d texSdf [[texture(4)]], - // Pyramide de profondeur de champ (`tilted_sample`), lue par le mode 8 et - // par le repli du mode 18, qui garde texture(2) pour son rendu isolé. - texture2d texDof [[texture(5)]]) +// The body of `ps_main`, which only adds the layer's transparency on top of it. Same resources +// as `ps_main`, without their bindings (MSL only allows those on an entry point). +inline float4 ps_layer(VSOut i, + constant Layer &layer, + texture2d texY, + texture2d texUV, + texture2d texImg, + texture2d texMask, + texture2d texSdf, + texture2d texDof) { // mode 18 : l'écran CADRÉ (ombre, cadre, métrage, appareil) flouté comme UN objet rigide // (`FrameGeometry::screen_trail`), port 1:1 du HLSL. texImg = son rendu isolé, prémultiplié, @@ -2082,6 +3323,7 @@ fragment float4 ps_main(VSOut i [[stage_in]], return acc / max(n, 1.0); } +#ifdef LAYER_MODELS // mode 17 : CADRE D'APPAREIL MODELÉ (`device_frame`). Testé en premier, comme le 16. if (layer.mode > 16.5) { @@ -2103,8 +3345,16 @@ fragment float4 ps_main(VSOut i [[stage_in]], { return float4(0.0, 0.0, 0.0, 0.0); } - return cursor_model(i.local, layer, texSdf, texImg); + return cursor_model(i.local, layer, texSdf, texImg, texDof); + } +#else + // Les modèles 3D n'existent que dans la variante `LAYER_MODELS` ; un draw qui les enverrait + // ici ne peint rien plutôt qu'une ombre faite de leurs emplacements. + if (layer.mode > 14.5) + { + return float4(0.0, 0.0, 0.0, 0.0); } +#endif // mode 14 : CADRE DE FENÊTRE autour de l'écran, dessiné SOUS lui. Cf. commentaires HLSL. // Testé avant le mode 13, dont la branche n'a pas de borne haute. @@ -2248,8 +3498,12 @@ fragment float4 ps_main(VSOut i [[stage_in]], float2 plane_px = layer.dst_prev.xy; float2 p = float2(r.x, r.y) * plane_px - plane_px * 0.5; float rad = max(layer.radius_px, 0.0); - float d = (layer.dst_prev.z > 0.5) ? sd_screen_under_bar(p, plane_px * 0.5, rad, layer.color.z) - : sd_round_rect(p, plane_px * 0.5, rad); + // Une branche et non `?:`, pour ne jamais évaluer les DEUX distances (trois SDF par pixel). + float d; + if (layer.dst_prev.z > 0.5) + d = sd_screen_under_bar(p, plane_px * 0.5, rad, layer.color.z); + else + d = sd_round_rect(p, plane_px * 0.5, rad); float tilt_a = 1.0 - smoothstep(0.0, 1.5, d); // Slot d'un layout en bloc (`color.w` = rayon de ses coins, px ; 0 ailleurs, sans effet) : // `dst` EST le slot, dont le rect arrondi rogne le plan (`ScreenMask`, cf. HLSL). @@ -2502,12 +3756,25 @@ fragment float4 ps_main(VSOut i [[stage_in]], float3 rgb; // 1 sauf en detourage, ou il porte le masque du sujet. Cf. la branche fx.z plus bas. float alpha_mask = 1.0; + // 0 where a camera homography finds no camera point (`persp_camera`), 1 everywhere else. + float persp_keep = 1.0; if (layer.mode < 0.5) { // flou de mouvement par vélocité (§8) float2 uv_now = i.uv; float2 localp = (i.pout - layer.dst_prev.xy) / layer.dst_prev.zw; float2 uv_prev = layer.src_prev.xy + localp * (layer.src_prev.zw - layer.src_prev.xy); + if (layer.layer_fx.y > 0.5) + { + // Camera homography: the texture uv comes from the fragment's place in the quad + // (`src` is not used), scaled to the valid part of the decoder texture. The previous + // frame's uv is the same map at the quad's previous place. + float3 cam = persp_camera(layer, (i.pout - layer.dst.xy) / layer.dst.zw); + persp_keep = cam.z; + uv_now = cam.xy * layer.fx.xy; + float3 cam_prev = persp_camera(layer, localp); + uv_prev = (cam_prev.z > 0.5) ? cam_prev.xy * layer.fx.xy : uv_now; + } float2 duv = uv_now - uv_prev; float mb_scale = saturate(layer.mb.y); float2 duv_blur = duv * mb_scale; @@ -2542,33 +3809,68 @@ fragment float4 ps_main(VSOut i [[stage_in]], } else if (effect > 1.5) { - rgb = mix(blur_webcam_bg(uv_now, layer.fx.w, layer.quad_px, i.local, texY, texUV), rgb, person); + rgb = mix(blur_webcam_bg(uv_now, layer.fx.w, layer.quad_px, i.local, layer.fx.xy, texY, texUV), rgb, person); } else { alpha_mask = person; } } + + // Desk-view cover: the camera is being tilted, so the whole picture is blurred and + // dimmed (cover.x = strength, cover.y = radius in quad px, cover.z = dim at full cover). + if (layer.cover.x > 0.001) + { + float3 hidden = blur_webcam_radius(uv_now, layer.cover.y, layer.quad_px, i.local, layer.fx.xy, + texY, texUV); + rgb = mix(rgb, hidden, layer.cover.x) * (1.0 - layer.cover.z * layer.cover.x); + } } else { rgb = layer.color.rgb; } - float alpha = layer.color.a * alpha_mask; + float alpha = layer.color.a * alpha_mask * persp_keep; if (layer.radius_px > 0.0) { // mb.w = 1 : écran sous le chrome de fenêtre, coins HAUTS carrés et rognés par l'arc du // cadre (`sd_screen_under_bar`, `mb.z` = la remontée du contour intérieur, px). float2 halfsz = layer.quad_px * 0.5; float2 p = i.local - layer.quad_px * 0.5; - float d = (layer.mb.w > 0.5) ? sd_screen_under_bar(p, halfsz, layer.radius_px, layer.mb.z) - : sd_round_rect(p, halfsz, layer.radius_px); + // Une branche et non `?:`, pour ne jamais évaluer les DEUX distances : trois SDF par pixel + // de l'écran au lieu d'une, 0,6 ms par frame 1080p sur une Radeon 610M (mesuré en WGSL). + float d; + if (layer.mb.w > 0.5) + d = sd_screen_under_bar(p, halfsz, layer.radius_px, layer.mb.z); + else + d = sd_round_rect(p, halfsz, layer.radius_px); alpha *= 1.0 - smoothstep(0.0, 1.5, d); } return float4(rgb * alpha, alpha); } +fragment float4 ps_main(VSOut i [[stage_in]], + constant Layer &layer [[buffer(0)]], + texture2d texY [[texture(0)]], + texture2d texUV [[texture(1)]], + texture2d texImg [[texture(2)]], + // Masque de segmentation du sujet webcam. Non lie tant qu'aucun + // masque n'existe : Metal rend alors 0, ce qui est sans effet + // puisque la branche n'est prise que si layer.fx.z > 0.5. + texture2d texMask [[texture(3)]], + // Champ de distance du sprite de curseur (mode 15 seulement), R16F, cf. + // `cursor_sdf.rs`. Le sprite lui-même est en texture(2), comme aux + // modes 7 et 13. + texture2d texSdf [[texture(4)]], + // Pyramide de profondeur de champ (`tilted_sample`), lue par le mode 8 et + // par le repli du mode 18, qui garde texture(2) pour son rendu isolé. + texture2d texDof [[texture(5)]]) +{ + // Premultiplied, so the transparency scales all four channels; 0 leaves them as they are. + return ps_layer(i, layer, texY, texUV, texImg, texMask, texSdf, texDof) * (1.0 - layer.layer_fx.x); +} + // ================================================================================= // Fullscreen pass : RGB -> NV12. Mêmes shaders que la passe équivalente HLSL. // ================================================================================= diff --git a/crates/compositor/src/shared_frames.rs b/crates/compositor/src/shared_frames.rs new file mode 100644 index 000000000..a6b4f8415 --- /dev/null +++ b/crates/compositor/src/shared_frames.rs @@ -0,0 +1,338 @@ +//! Livraison de la preview par textures partagées : le thread de rendu copie chaque frame +//! composée dans l'une des textures d'un petit anneau, et Electron l'importe côté GPU +//! (`sharedTexture.importSharedTexture`) au lieu d'en recevoir les pixels. Plus de `Map` +//! qui attend le GPU puis recopie l'image en RAM, plus de `Vec` de plusieurs Mo par frame +//! à travers l'IPC : mesuré, ce transport coûtait à lui seul 37 à 55 % du thread principal +//! du renderer à 30 images/s. +//! +//! Deux moitiés : +//! - `SlotBook`, portable et sans GPU : quelle case porte la frame prête, lesquelles +//! Chromium tient encore. Partagé entre le thread de rendu (`claim`, `publish`) et le +//! thread Node (`take`, `release`). +//! - `SharedRing` (Windows, backend matériel) : les textures D3D11 et leurs handles NT. +//! macOS (IOSurface) et Linux (dmabuf) restent sur le readback. + +use crate::frame_geometry::FootageQuad; +use crate::live::FramePosition; +use std::time::{Duration, Instant}; + +/// Cases de l'anneau : une prête, une en transit vers le renderer (tenue jusqu'à ce que +/// Chromium la relâche), une en écriture, et une de marge pour la latence de la libération. +pub const RING_SLOTS: u32 = 4; + +/// Une case tenue plus longtemps que ça a perdu sa libération (renderer rechargé, import +/// échoué en route) : Chromium relâche les autres dans la milliseconde. Même ordre que le +/// délai d'Electron sur `sendSharedTexture` (1 s). +pub const LOST_SLOT_AFTER: Duration = Duration::from_secs(1); + +/// Une frame composée posée dans une case de l'anneau, telle que JS la reçoit. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct SharedFrame { + /// Même compteur que les frames lues en RAM : le consommateur ne voit qu'une suite de + /// générations, quel que soit le transport. + pub gen: u64, + pub slot: u32, + /// Valeur du handle NT de la texture de la case, valable dans CE processus. Electron le + /// duplique à l'import : la case garde le sien. + pub handle: u64, + pub width: u32, + pub height: u32, + pub footage: Option, + pub position: FramePosition, +} + +/// Une case prise par JS : la génération qu'elle porte et quand elle est partie. +#[derive(Debug, Clone, Copy)] +struct Held { + slot: u32, + gen: u64, + since: Instant, +} + +/// Qui tient quelle case de l'anneau. +#[derive(Debug, Default)] +pub struct SlotBook { + /// La dernière frame publiée, que JS n'a pas encore prise. + ready: Option, + /// Les cases prises par JS, que Chromium n'a pas encore relâchées, plus anciennes d'abord. + held: Vec, +} + +impl SlotBook { + /// La case où écrire la prochaine frame. Une case libre d'abord ; à défaut, celle de la + /// frame prête que personne n'a prise, retirée puisqu'elle est périmée. `None` quand + /// Chromium tient toutes les cases : une case tenue n'est jamais réécrite, même perdue + /// (`lost`), puisqu'il peut encore la lire. + pub fn claim(&mut self, slots: u32) -> Option { + let ready = self.ready.map(|frame| frame.slot); + let is_held = |slot: u32| self.held.iter().any(|held| held.slot == slot); + if let Some(free) = (0..slots).find(|slot| !is_held(*slot) && Some(*slot) != ready) { + return Some(free); + } + if let Some(stale) = ready { + self.ready = None; + return Some(stale); + } + None + } + + /// Une case est tenue depuis `LOST_SLOT_AFTER` : sa libération ne viendra plus. + pub fn lost(&self, now: Instant) -> bool { + self.held.first().is_some_and(|held| now.duration_since(held.since) >= LOST_SLOT_AFTER) + } + + pub fn publish(&mut self, frame: SharedFrame) { + self.ready = Some(frame); + } + + /// La frame prête, si elle est plus récente que `since_gen`. Sa case reste tenue jusqu'à + /// `release` : le thread de rendu n'y écrira plus d'ici là. + pub fn take(&mut self, since_gen: u64, now: Instant) -> Option { + let frame = self.ready.filter(|frame| frame.gen > since_gen)?; + self.ready = None; + self.held.push(Held { slot: frame.slot, gen: frame.gen, since: now }); + Some(frame) + } + + /// Chromium a relâché la frame `gen` de la case `slot`. La génération évite qu'une + /// libération en double ne libère la frame suivante posée dans la même case. + pub fn release(&mut self, slot: u32, gen: u64) { + self.held.retain(|held| held.slot != slot || held.gen != gen); + } + + /// Oublie la frame prête : la vue repasse au readback, personne ne viendra la prendre. + pub fn clear_ready(&mut self) { + self.ready = None; + } +} + +#[cfg(windows)] +pub use ring::SharedRing; + +#[cfg(windows)] +mod ring { + use super::RING_SLOTS; + use crate::d3d::Gpu; + use anyhow::{bail, Result}; + use std::time::{Duration, Instant}; + use windows::core::{Interface, PCWSTR}; + use windows::Win32::Foundation::{CloseHandle, BOOL, HANDLE}; + use windows::Win32::Graphics::Direct3D11::{ + ID3D11Device, ID3D11DeviceContext, ID3D11Query, ID3D11Texture2D, + D3D11_ASYNC_GETDATA_DONOTFLUSH, D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, + D3D11_QUERY_DESC, D3D11_QUERY_EVENT, D3D11_RESOURCE_MISC_SHARED, + D3D11_RESOURCE_MISC_SHARED_NTHANDLE, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, + }; + use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_SAMPLE_DESC}; + use windows::Win32::Graphics::Dxgi::{ + IDXGIResource1, DXGI_SHARED_RESOURCE_READ, DXGI_SHARED_RESOURCE_WRITE, + }; + + /// Au-delà, le GPU est bloqué ou perdu : on rend une erreur plutôt que de geler la vue. + const COPY_TIMEOUT: Duration = Duration::from_millis(500); + + struct SlotTexture { + tex: ID3D11Texture2D, + handle: HANDLE, + width: u32, + height: u32, + } + + impl Drop for SlotTexture { + fn drop(&mut self) { + // Electron importe un duplicata du handle : fermer le nôtre ne retire rien à une + // frame que Chromium affiche encore. + unsafe { + let _ = CloseHandle(self.handle); + } + } + } + + /// Les textures partagées d'une vue live, créées à la demande à la taille du rendu. + pub struct SharedRing { + device: ID3D11Device, + context: ID3D11DeviceContext, + copied: ID3D11Query, + slots: Vec>, + } + + impl SharedRing { + pub fn new(gpu: &Gpu) -> Result { + let desc = D3D11_QUERY_DESC { Query: D3D11_QUERY_EVENT, MiscFlags: 0 }; + let mut copied = None; + unsafe { gpu.device.CreateQuery(&desc, Some(&mut copied))? }; + Ok(SharedRing { + device: gpu.device.clone(), + context: gpu.context.clone(), + copied: copied.expect("CreateQuery a réussi sans requête"), + slots: (0..RING_SLOTS).map(|_| None).collect(), + }) + } + + /// Copie `src` (`width`×`height`, RGBA8) dans la case `slot` et rend le handle de sa + /// texture une fois le GPU arrivé au bout de la copie. Chromium la lit depuis son propre + /// device, sans keyed mutex : quand le handle part, plus rien ne doit y écrire. L'attente + /// est celle que faisait déjà le `Map` du readback, la recopie en RAM en moins. + /// + /// La case ne doit être ni prête ni tenue (`SlotBook::claim`) : sa texture peut être + /// recréée ici quand la taille du rendu a changé. + pub unsafe fn write( + &mut self, + slot: u32, + src: &ID3D11Texture2D, + width: u32, + height: u32, + ) -> Result { + let entry = &mut self.slots[slot as usize]; + if !matches!(entry, Some(texture) if texture.width == width && texture.height == height) { + *entry = Some(create_slot(&self.device, width, height)?); + } + let target = entry.as_ref().expect("case créée juste au-dessus"); + self.context.CopyResource(&target.tex, src); + self.context.End(&self.copied); + self.context.Flush(); + let deadline = Instant::now() + COPY_TIMEOUT; + loop { + let mut done = BOOL(0); + self.context.GetData( + &self.copied, + Some(&mut done as *mut BOOL as *mut core::ffi::c_void), + std::mem::size_of::() as u32, + D3D11_ASYNC_GETDATA_DONOTFLUSH.0 as u32, + )?; + if done.as_bool() { + break; + } + if Instant::now() > deadline { + bail!("le GPU n'a pas fini de copier la frame en {COPY_TIMEOUT:?}"); + } + std::thread::yield_now(); + } + Ok(target.handle.0 as u64) + } + } + + unsafe fn create_slot(device: &ID3D11Device, width: u32, height: u32) -> Result { + let desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + // Le format du RT composé : la copie est un `CopyResource`, sans conversion. + Format: DXGI_FORMAT_R8G8B8A8_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32, + CPUAccessFlags: 0, + // Handle NT, le seul qu'Electron importe, sans keyed mutex : `write` attend la fin + // de la copie avant de le livrer. + MiscFlags: (D3D11_RESOURCE_MISC_SHARED.0 | D3D11_RESOURCE_MISC_SHARED_NTHANDLE.0) as u32, + }; + let mut tex = None; + device.CreateTexture2D(&desc, None, Some(&mut tex))?; + let tex = tex.expect("CreateTexture2D a réussi sans texture"); + let resource: IDXGIResource1 = tex.cast()?; + let handle = resource.CreateSharedHandle( + None, + DXGI_SHARED_RESOURCE_READ.0 | DXGI_SHARED_RESOURCE_WRITE.0, + PCWSTR::null(), + )?; + Ok(SlotTexture { tex, handle, width, height }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame(gen: u64, slot: u32) -> SharedFrame { + SharedFrame { + gen, + slot, + handle: 0, + width: 2, + height: 2, + footage: None, + position: FramePosition { clip_index: 0, source_time_sec: 0.0 }, + } + } + + /// Toutes les cases prises par JS, la case `n` portant la génération `n + 1`. + fn all_held(now: Instant) -> SlotBook { + let mut book = SlotBook::default(); + for slot in 0..RING_SLOTS { + book.publish(frame(u64::from(slot) + 1, slot)); + assert!(book.take(u64::from(slot), now).is_some()); + } + book + } + + #[test] + fn claims_a_free_slot_before_a_ready_one() { + let now = Instant::now(); + let mut book = SlotBook::default(); + book.publish(frame(1, 0)); + assert_eq!(book.claim(RING_SLOTS), Some(1)); + // La frame prête n'a pas bougé : JS peut toujours la prendre. + assert_eq!(book.take(0, now), Some(frame(1, 0))); + } + + #[test] + fn a_taken_slot_is_never_written_until_released() { + let now = Instant::now(); + let mut book = all_held(now); + assert_eq!(book.claim(RING_SLOTS), None, "Chromium tient toutes les cases"); + book.release(2, 3); + assert_eq!(book.claim(RING_SLOTS), Some(2)); + } + + #[test] + fn a_frame_nobody_took_gives_its_slot_back_when_nothing_else_is_free() { + let now = Instant::now(); + let mut book = SlotBook::default(); + for slot in 0..RING_SLOTS - 1 { + book.publish(frame(u64::from(slot) + 1, slot)); + book.take(u64::from(slot), now); + } + book.publish(frame(9, 3)); + assert_eq!(book.claim(RING_SLOTS), Some(3)); + // Retirée : la livrer maintenant, pendant qu'on la réécrit, donnerait une image déchirée. + assert_eq!(book.take(0, now), None); + } + + #[test] + fn take_hands_out_only_newer_generations() { + let now = Instant::now(); + let mut book = SlotBook::default(); + book.publish(frame(5, 1)); + assert_eq!(book.take(5, now), None); + assert_eq!(book.take(4, now), Some(frame(5, 1))); + assert_eq!(book.take(0, now), None, "déjà prise"); + } + + #[test] + fn a_release_only_frees_the_generation_it_names() { + let now = Instant::now(); + let mut book = SlotBook::default(); + book.publish(frame(1, 0)); + book.take(0, now); + book.release(3, 1); + book.release(0, 7); + assert_eq!(book.claim(1), None, "ni la case 3 ni la génération 7 ne sont tenues"); + book.release(0, 1); + assert_eq!(book.claim(1), Some(0)); + } + + #[test] + fn a_slot_whose_release_never_came_is_reported_lost_but_never_rewritten() { + let start = Instant::now(); + let mut book = all_held(start); + assert!(!book.lost(start + LOST_SLOT_AFTER / 2)); + assert!(book.lost(start + LOST_SLOT_AFTER)); + // Chromium peut encore lire une case perdue : la réécrire déchirerait son image. + assert_eq!(book.claim(RING_SLOTS), None); + // Si sa libération finit par venir, la case sert de nouveau. + book.release(0, 1); + assert_eq!(book.claim(RING_SLOTS), Some(0)); + } +} diff --git a/crates/compositor/src/text_anim.rs b/crates/compositor/src/text_anim.rs index d6d3bade1..ae3887f98 100644 --- a/crates/compositor/src/text_anim.rs +++ b/crates/compositor/src/text_anim.rs @@ -4,7 +4,8 @@ //! amplitudes. Les sept animations étaient déjà nommées dans le schéma, traduites dans les treize //! langues et transportées jusqu'ici par la scène — mais rien ne les jouait. Reprendre les //! constantes du TS plutôt que d'en réinventer garantit qu'un projet fait à l'époque de l'aperçu -//! DOM s'anime toujours pareil. +//! DOM s'anime toujours pareil. Exception : l'étiquette de la vue bureau (`deskCover`), que seul +//! le compositeur joue et qui n'a pas de courbe propre (cf. `annotation_text_state`). /// Les décalages ci-dessous sont exprimés en px À CETTE HAUTEUR : l'appelant les met à l'échelle /// de la sortie, exactement comme la taille de police (cf. `annotationScale.ts`). En pixels @@ -17,6 +18,10 @@ pub const TEXT_ANIMATION_DURATION_MS: f32 = 700.0; /// fin de l'annotation est une coupe sèche alors que son arrivée est animée. pub const TEXT_EXIT_DURATION_MS: f32 = 300.0; +/// The desk-view label's animation. The app generates it for a turned Full Camera section and +/// never stores it in a document, so it exists only here, not in `annotationTextAnimation.ts`. +pub const DESK_COVER_ANIMATION: &str = "deskCover"; + #[derive(Debug, Clone, Copy, PartialEq)] pub struct TextAnimationState { pub opacity: f32, @@ -37,6 +42,11 @@ fn clamp01(v: f32) -> f32 { v.clamp(0.0, 1.0) } +fn smoothstep01(v: f32) -> f32 { + let t = clamp01(v); + t * t * (3.0 - 2.0 * t) +} + fn ease_out_cubic(v: f32) -> f32 { let t = clamp01(v); 1.0 - (1.0 - t).powi(3) @@ -106,6 +116,30 @@ pub fn text_animation_state( } } +/// `true` for the desk-view label, whose opacity is the camera cover (`annotation_text_state`). +pub fn is_desk_cover(animation: Option<&str>) -> bool { + animation == Some(DESK_COVER_ANIMATION) +} + +/// What the compositors draw a text annotation with. Ordinary animations run on +/// `text_animation_state` (source time, unchanged). The desk-view label has no timing of its +/// own: its opacity IS `cover`, the webcam's cover strength this frame +/// (`camera_fullscreen_cover_at`, already on the screen clock). A second fade derived from the +/// label's own window drifted from the cover on short sections and inside speed regions; one +/// value read twice cannot. The app spans the label over its whole section, so the cover alone +/// decides when it shows. +pub fn annotation_text_state( + animation: Option<&str>, + elapsed_ms: f32, + region_ms: f32, + cover: f32, +) -> TextAnimationState { + if is_desk_cover(animation) { + return TextAnimationState { opacity: clamp01(cover), ..TextAnimationState::IDLE }; + } + text_animation_state(animation, elapsed_ms, region_ms) +} + #[cfg(test)] mod tests { use super::*; @@ -245,4 +279,28 @@ mod tests { fn an_empty_region_does_not_divide_by_zero() { assert!(text_animation_state(Some("fade"), 0.0, 0.0).opacity.is_finite()); } + + #[test] + fn the_desk_label_takes_the_cover_and_nothing_else() { + // Whatever the label's own window says, only the cover counts; the motion fields stay idle. + for cover in [0.0, 0.21, 0.45, 1.0] { + for (elapsed, region) in [(0.0, 2600.0), (900.0, 1150.0), (5000.0, LONG)] { + let s = annotation_text_state(Some(DESK_COVER_ANIMATION), elapsed, region, cover); + assert_eq!(s, TextAnimationState { opacity: cover, ..TextAnimationState::IDLE }); + } + } + } + + #[test] + fn ordinary_animations_ignore_the_cover() { + for name in [None, Some("fade"), Some("rise"), Some("pop"), Some("typewriter"), Some("pulse")] { + for at in [0.0, 200.0, 650.0, 9_900.0] { + assert_eq!( + annotation_text_state(name, at, LONG, 0.37), + text_animation_state(name, at, LONG), + "{name:?} at {at}" + ); + } + } + } } diff --git a/crates/compositor/src/timeline_walk.rs b/crates/compositor/src/timeline_walk.rs index a16271c8b..5a1205ae0 100644 --- a/crates/compositor/src/timeline_walk.rs +++ b/crates/compositor/src/timeline_walk.rs @@ -17,13 +17,17 @@ use crate::compositor::Compositor; use crate::config::Cfg; use crate::cursor::CursorTrack; use crate::d3d::Gpu; +use crate::extra_cameras::{ + camera_source_time, cameras_in_regions, extra_camera_active, extra_camera_keys, open_once, + step_extra_camera, +}; use crate::ffi::AVFrame; use crate::frame_geometry::webcam_is_real; use crate::pipeline::{ClipSource, Decoder}; use crate::regions::{speed_segments_for_window, SpeedSegment}; -use crate::scene::Scene; +use crate::scene::{Scene, SceneCameraLayoutRegion}; use anyhow::Result; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; /// Ce que le décodeur sait de la PROCHAINE frame, sans l'adopter. /// @@ -142,9 +146,24 @@ pub(crate) unsafe fn advance_decoder_to( /// N" defined exactly once: a GIF driven by its own loop is how the slow-motion /// truncation bug happened. /// +/// The extra cameras (k >= 1) a clip's layout regions show are opened once per path into +/// `extra_decs`, decoded only near those regions, and handed to the compositor with +/// `set_extra_camera_frames`. Unlike camera 0 they never shorten a clip, and a file that will +/// not open (or stops decoding) is skipped with a warning for the rest of the export. +/// /// `on_frame` runs after `compose_frame` with the running output index; /// `on_clip_end` runs once per clip with its clamped source window, the frames /// it produced, and the speed segments used (MP4 needs those for audio). +/// Runs its closure when dropped, so on every exit path of a scope, early `?` returns +/// included. +struct OnExit(F); + +impl Drop for OnExit { + fn drop(&mut self) { + (self.0)(); + } +} + #[allow(clippy::too_many_arguments)] pub(crate) unsafe fn walk_composited_timeline( clips: &[ClipSource], @@ -155,6 +174,7 @@ pub(crate) unsafe fn walk_composited_timeline( scene: &Option, screen_decs: &mut HashMap, webcam_decs: &mut HashMap, + extra_decs: &mut HashMap, on_frame: &mut dyn FnMut(u64) -> Result<()>, on_clip_end: &mut dyn FnMut(usize, f64, u64, &[SpeedSegment]) -> Result<()>, ) -> Result { @@ -168,6 +188,14 @@ pub(crate) unsafe fn walk_composited_timeline( let mut cursor_active_path: Option = None; let mut frames: u64 = 0; + let mut unreadable_extras: HashSet = HashSet::new(); + // The extra decoders outlive this call in the caller's map, but the compositor must not + // keep pointers into their frames once the walk is over — also when it fails half-way + // (a decode or encode error returns early through `?`). + let _forget_extra_frames = OnExit(|| { + // SAFETY: an empty list only resets the compositor's slots; no frame is read. + unsafe { comp.set_extra_camera_frames(&[]) } + }); // L'export doit être reproductible : deux rendus du même projet, les mêmes pixels. Cette // boucle avance aussi vite que la machine décode, sans rapport avec le temps réel, alors que @@ -230,17 +258,12 @@ pub(crate) unsafe fn walk_composited_timeline( clip.source_end_sec, ); } - // Les bornes de clip sont en temps écran. La disponibilité webcam est donc translatée - // par le même offset que le seek (`webcam_time = screen_time - offset`). - let webcam_available_screen_end = - webcam_available_duration.map(|duration| duration + clip.webcam_offset_sec); - let mut source_end_sec = clip.source_end_sec; - if let Some(duration) = screen_available_duration { - source_end_sec = source_end_sec.min(duration); - } - if let Some(duration) = webcam_available_screen_end { - source_end_sec = source_end_sec.min(duration); - } + let source_end_sec = available_clip_end( + clip.source_end_sec, + screen_available_duration, + webcam_available_duration, + clip.webcam_offset_sec, + ); if source_end_sec + 1e-6 < clip.source_end_sec { eprintln!( "[pipeline] warning: clip #{} raccourci de {:.3}s (fin demandée {:.3}s, fin disponible {:.3}s; screen=\"{}\", webcam=\"{}\")", @@ -268,6 +291,27 @@ pub(crate) unsafe fn walk_composited_timeline( source_end_sec, out_fps as f64, ); + // Only the extra cameras this clip's own regions show, and only those it has a file + // for. `None` slots stay null in `set_extra_camera_frames`. + let camera_regions = clip_scene + .as_ref() + .map(|s| s.camera_layout_regions.clone()) + .unwrap_or_default(); + let mut extras: Vec> = + extra_camera_keys(&cameras_in_regions(&camera_regions), &clip.additional_cameras) + .into_iter() + .map(|key| { + let key = key?; + open_once(extra_decs, &mut unreadable_extras, &key.path, |path| { + Decoder::open_for_export(path, gpu) + }) + .then_some(ExportExtraCamera { + path: key.path, + offset_sec: key.offset_sec, + ended_at: None, + }) + }) + .collect(); if clip_scene.is_some() { comp.set_scene(clip_scene); } @@ -328,6 +372,24 @@ pub(crate) unsafe fn walk_composited_timeline( break 'clip_frames; } + let (extra_frames, closed) = { + let _p = crate::export_probe::scope(crate::export_probe::Stage::DecodeWebcam); + step_extra_cameras( + &mut extras, + extra_decs, + &mut unreadable_extras, + &camera_regions, + target_source_time, + ) + }; + if closed { + // A dropped decoder's textures may still sit in the SRV cache; a later + // decoder could reuse their addresses (same contract as the preview's + // `hand_extra_frames`). + comp.clear_srv_cache(); + } + comp.set_extra_camera_frames(&extra_frames); + comp.set_timeline_time(Some(target_source_time as f32)); // Le temps de SORTIE, lui, ne saute ni aux coupes ni aux clips. Même // arithmétique (f64 puis f32) que `ProgrammeClock::at` côté preview. @@ -361,9 +423,104 @@ pub(crate) unsafe fn walk_composited_timeline( Ok(frames) } +/// The screen-time end of a clip: the requested end, clamped to what the screen and camera 0 +/// can deliver. Clip bounds are in screen time, so camera 0's duration is moved by the same +/// offset as its seek (`webcam_time = screen_time - offset`). The extra cameras do not enter +/// here: one that ends early only loses its layer, it never shortens the clip. +fn available_clip_end( + requested_end_sec: f64, + screen_duration: Option, + webcam_duration: Option, + webcam_offset_sec: f64, +) -> f64 { + let mut end = requested_end_sec; + if let Some(duration) = screen_duration { + end = end.min(duration); + } + if let Some(duration) = webcam_duration { + end = end.min(duration + webcam_offset_sec); + } + end +} + +/// An extra camera of the clip being exported; its decoder lives in `extra_decs` under `path`. +struct ExportExtraCamera { + path: String, + offset_sec: f64, + /// See `step_extra_camera`. + ended_at: Option, +} + +/// The frames for `set_extra_camera_frames` at screen source time `t`: each camera that is +/// near one of its regions steps toward its own source time, the others give null. A camera +/// that stops decoding is dropped with a warning and not reopened for the rest of the export; +/// the `bool` says one was dropped, so the caller clears the compositor's texture cache. +unsafe fn step_extra_cameras( + extras: &mut [Option], + extra_decs: &mut HashMap, + unreadable: &mut HashSet, + regions: &[SceneCameraLayoutRegion], + t: f64, +) -> (Vec<*const AVFrame>, bool) { + let mut closed = false; + let mut frames = Vec::with_capacity(extras.len()); + for (k, slot) in extras.iter_mut().enumerate() { + let camera = k + 1; + let mut frame: *const AVFrame = std::ptr::null(); + if let Some(cam) = slot.as_mut().filter(|_| extra_camera_active(regions, camera, t)) { + if let Some(dec) = extra_decs.get_mut(&cam.path) { + let target = camera_source_time(t, cam.offset_sec); + match step_extra_camera(dec, &mut cam.ended_at, target) { + Ok(f) => frame = f, + Err(e) => { + eprintln!( + "WARNING: extra camera {camera} stopped decoding ({}): {e:#}. Its layer will not be drawn.", + cam.path + ); + extra_decs.remove(&cam.path); + unreadable.insert(cam.path.clone()); + *slot = None; + closed = true; + } + } + } + } + frames.push(frame); + } + (frames, closed) +} + #[cfg(test)] mod tests { - use super::{frame_step, FrameStep, NextFrameTime}; + use super::{available_clip_end, frame_step, FrameStep, NextFrameTime, OnExit}; + + /// The guard that forgets the extra camera frames runs on an early `?` return as well as + /// at the end of the walk. + #[test] + fn the_exit_guard_runs_on_every_exit_path() { + fn walk(fail: bool, ran: &std::cell::Cell) -> anyhow::Result<()> { + let _guard = OnExit(|| ran.set(ran.get() + 1)); + if fail { + Err(anyhow::anyhow!("decode error"))?; + } + Ok(()) + } + let ran = std::cell::Cell::new(0); + assert!(walk(true, &ran).is_err()); + assert_eq!(ran.get(), 1, "early return"); + assert!(walk(false, &ran).is_ok()); + assert_eq!(ran.get(), 2, "normal end"); + } + + #[test] + fn a_short_extra_camera_does_not_shorten_the_clip() { + // Screen 60 s, camera 0 50 s at offset 2 s: the clip ends where camera 0 does. + assert_eq!(available_clip_end(60.0, Some(60.0), Some(50.0), 2.0), 52.0); + // A 10 s extra camera in the same clip changes nothing: only the screen and camera 0 + // bound the clip, the extra camera's layer just disappears past its end. + assert_eq!(available_clip_end(30.0, Some(60.0), Some(50.0), 2.0), 30.0); + assert_eq!(available_clip_end(30.0, None, None, 0.0), 30.0); + } /// La cadence de lecture, en une phrase : à 24 fps, une seconde réelle doit adopter 24 /// frames et pas une de plus. Le bug d'origine (un pas fixe de 1/60 s, une frame par diff --git a/crates/compositor/src/vk_shaders/layer.wgsl b/crates/compositor/src/vk_shaders/layer.wgsl index 2eb46a9b3..6d745cd13 100644 --- a/crates/compositor/src/vk_shaders/layer.wgsl +++ b/crates/compositor/src/vk_shaders/layer.wgsl @@ -16,6 +16,15 @@ // // Le rendu est en alpha PRÉMULTIPLIÉ (cf. commentaire HLSL), convention qu'on // retrouve dans tous les autres modes du compositeur (texte, curseur, ombre). +// +// `LAYER_MODELS` n'est PAS declare ici : `compositor_linux.rs` (`layer_source`) le prefixe et +// compile ce source deux fois. Un shader unique alloue a CHAQUE calque les registres de sa +// branche la plus lourde, et ce sont les modeles 3D ray-traces (modes 15 a 17) : 168 VGPR sous +// ACO (Radeon 610M), donc 6 vagues par SIMD au lieu de 32 pour le fond, l'ombre et la video, qui +// couvrent toute la sortie : l'export Linux du banc etait passe de 31 a 57 s. Sans ces trois modes, +// 48 VGPR et 20 vagues. Le pipeline plat (`false`) dessine donc tout le reste, et seuls le +// curseur modelise, l'impact de son clic, l'appareil et son ombre passent par l'autre : c'est +// `LayerCB::needs_models` qui en decide, au draw (`draw_layer`), d'apres le mode du calque. struct Layer { dst: vec4, // x,y,w,h sortie 0..1 (origine haut-gauche) @@ -31,19 +40,27 @@ struct Layer { trail_a: vec4, // mode 8 : coins TL, TR du plan a la frame precedente (px locaux, comme fx) ; mode 18 incline : en fractions de sortie trail_b: vec4, // mode 8 : coins BR, BL du plan a la frame precedente (comme src_prev) ; mode 18 incline : en fractions de sortie trail_mb: vec4, // mode 8 : x = taps, y = force du flou de mouvement (ceux du mode 0) ; mode 18 incline : le `mb` du mode 8 (profondeur de champ), et `color.xy` sa lampe ; 0 ailleurs + cover: vec4, // x = desk-view cover 0..1, y = blur radius (quad px), z = dim, w unused + persp: array, 3>, // mode 0: rows of the homography quad point (0..1 in dst) -> camera uv (0..1 of the valid frame), xyz; read when layer_fx.y = 1 + layer_fx: vec4, // x = transparency 0..1 (0 = opaque), every mode; y = 1 when persp applies; z, w unused } @group(0) @binding(0) var layer: Layer; -@group(0) @binding(1) var texY: texture_2d; // R8Unorm, sample .r ; modes 7, 13 et 15 : le sprite RGBA -@group(0) @binding(2) var texU: texture_2d; // R8Unorm, sample .r ; mode 15 : le champ R16F du sprite +@group(0) @binding(1) var texY: texture_2d; // R8Unorm, sample .r ; modes 7 et 13 : le sprite RGBA ; mode 15 : la copie de l'image composee +@group(0) @binding(2) var texU: texture_2d; // R8Unorm, sample .r @group(0) @binding(3) var samp: sampler; // Masque de segmentation du sujet webcam, R8. Une vue 1x1 est liee quand aucun masque // n'existe : la branche n'est de toute facon prise que si layer.fx.z > 0.5. -// Mode 18 : ce binding porte a la place le rendu isole de l'ecran cadre. +// Mode 18 : ce binding porte a la place le rendu isole de l'ecran cadre ; mode 15, le champ R16F +// du sprite. @group(0) @binding(4) var texMask: texture_2d; // Pyramide RGBA de profondeur de champ (`tilted_sample`), lue par le mode 8 et par le repli du -// mode 18, qui garde le binding 4 pour son rendu isole : d'ou un binding a elle. Vue 1x1 ailleurs. +// mode 18, qui garde le binding 4 pour son rendu isole : d'ou un binding a elle. Mode 15 : le +// sprite RGBA. Vue 1x1 ailleurs. @group(0) @binding(6) var texDof: texture_2d; +// Le maillage de Prism Glow (mode 15), rempli de `prism_mesh.rs` : un uniform et non des tables +// dans le shader, que lavapipe recopierait a chaque pixel de chaque calque (cf. `PrismMesh`). +@group(0) @binding(7) var prism: PrismMesh; // Plafond de la profondeur de champ du mode 8, en niveau de la pyramide demi-resolution. // Meme valeur que `DOF_MAX_LOD` du HLSL. @@ -170,8 +187,12 @@ fn screen_trail(pout: vec2) -> vec4 { // garde sa largeur tout autour du coin. r <= 0 : le rectangle vif d'avant, à l'identique. fn sd_round_rect(p: vec2, halfsz: vec2, r: f32) -> f32 { let hmin = min(halfsz.x, halfsz.y); - if (r <= 0.0 || hmin <= 0.0) { - let q0 = abs(p) - halfsz; + let q0 = abs(p) - halfsz; + // Hors des coins, la distance est celle du rectangle vif quel que soit l'exposant : un seul + // axe deborde de l'etendue E, qui s'y ajoute puis s'en retranche. E ne depasse pas 1,42 r + // (n = 3) : sous 1,5 r du bord sur un axe, on rend la boite sans calculer `n` ni E, deux + // transcendantes de moins sur presque tous les pixels d'un calque arrondi. + if (r <= 0.0 || hmin <= 0.0 || min(q0.x, q0.y) <= -min(1.5 * r, hmin)) { return length(max(q0, vec2(0.0))) + min(max(q0.x, q0.y), 0.0); } let n = 3.0 - smoothstep(0.5, 1.0, r / hmin); @@ -542,9 +563,13 @@ const VOGEL_TAPS = array, 21>( vec3(-0.633036, -0.758588, 0.087119) ); -fn blur_webcam_bg(uv: vec2, intensity: f32, qpx: vec2, local_px: vec2) -> vec3 { - let max_r_px = max(intensity, 0.0) * 22.0 + 1.5; +// Vogel-disc blur of the camera texture at a radius in quad pixels. `valid` is the part of the +// texture the picture fills (fx.xy): decoders allocate aligned textures (a 1080-line camera in a +// 1088-line texture), so each tap is clamped half a chroma texel inside it, never into padding. +fn blur_webcam_radius(uv: vec2, max_r_px: f32, qpx: vec2, local_px: vec2, valid: vec2) -> vec3 { let step = max_r_px / max(qpx, vec2(1.0)); + let chroma = max(vec2(textureDimensions(texU)), vec2(1.0)); + let hi = max(valid - 0.5 / chroma, vec2(0.0)); let noise = fract(52.9829189 * fract(0.06711056 * local_px.x + 0.00583715 * local_px.y)); let angle = noise * 6.2831853; let s = sin(angle); @@ -555,12 +580,28 @@ fn blur_webcam_bg(uv: vec2, intensity: f32, qpx: vec2, local_px: vec2< let p = VOGEL_TAPS[k].xy; let w = VOGEL_TAPS[k].z; let rot_p = vec2(p.x * c - p.y * s, p.x * s + p.y * c); - sum = sum + sample_yuv(clamp(uv + rot_p * step, vec2(0.0), vec2(1.0))) * w; + sum = sum + sample_yuv(clamp(uv + rot_p * step, vec2(0.0), hi)) * w; total = total + w; } return sum / max(total, 1e-4); } +fn blur_webcam_bg(uv: vec2, intensity: f32, qpx: vec2, local_px: vec2, valid: vec2) -> vec3 { + return blur_webcam_radius(uv, max(intensity, 0.0) * 22.0 + 1.5, qpx, local_px, valid); +} + +// Camera homography (`persp`, read when `layer_fx.y` = 1): the camera point (0..1 of its valid +// frame) seen at `local` (0..1 in the layer's dst quad), and in z whether there is one -- 0 when +// the point is behind the projection (q.z <= 0) or outside the camera frame, where the layer is +// transparent. Mirror of the HLSL and MSL `persp_camera`. +fn persp_camera(local: vec2) -> vec3 { + let p = vec3(local, 1.0); + let q = vec3(dot(layer.persp[0].xyz, p), dot(layer.persp[1].xyz, p), dot(layer.persp[2].xyz, p)); + let cam = q.xy / max(q.z, 1e-6); + let inside = q.z > 0.0 && all(cam >= vec2(0.0)) && all(cam <= vec2(1.0)); + return vec3(cam, select(0.0, 1.0, inside)); +} + // ---- Curseur MODELISE (mode 15) ---- // Port ligne pour ligne de `cursor_model` (HLSL), dont les commentaires font foi : un curseur // SCULPTE (`trail_a.x` > 0, la fleche ou la main d'un des cinq themes d'origine modelee en @@ -568,9 +609,10 @@ fn blur_webcam_bg(uv: vec2, intensity: f32, qpx: vec2, local_px: vec2< // propres et occlusion, et qui porte une ombre douce et une ombre de contact sur le plan de // l'ecran. Constantes : miroir exact de `frame_geometry.rs` (MODEL_*) et de `sculpt.rs` // (SCULPT_*), emplacements du cbuffer : `cursor_model_cb`. -// Textures : le sprite RGBA (alpha droit) au binding 1 (`texY`, comme aux modes 7 et 13), son -// champ R16F au binding 2 (`texU`), sur le meme rect ; `color.rg` = coin du sprite dans le repere -// du modele, `sprite_size()` = sa taille (w/h dans `radius_px`), `color.b` = l'ecrasement au clic. +// Textures : la copie de l'image composee au binding 1 (`texY`), que refracte le cristal de Prism +// Glow ; le sprite RGBA (alpha droit) est au binding 6 (`texDof`), son champ R16F au +// binding 4 (`texMask`), sur le meme rect ; `color.rg` = coin du sprite dans le repere du modele, +// `sprite_size()` = sa taille (w/h dans `radius_px`), `color.b` = l'ecrasement au clic. const MODEL_BEVEL: f32 = 0.045; const MODEL_LIGHT = vec3(-0.4194, -0.5792, 0.6990); const MODEL_FILL = vec3(0.7557, 0.2519, 0.6046); @@ -592,7 +634,7 @@ fn sprite_size() -> vec2 { // Un texel du sprite, en unites du modele : le champ est le sprite surechantillonne x4. const CURSOR_SDF_UPSAMPLE: f32 = 4.0; fn sprite_texel() -> f32 { - let d = vec2(textureDimensions(texU)); + let d = vec2(textureDimensions(texMask)); return CURSOR_SDF_UPSAMPLE / max(d.x, d.y); } @@ -610,8 +652,6 @@ fn sculpt_id() -> i32 { const SCULPT_SCALE: f32 = 0.85; const SCULPT_HOVER: f32 = 0.05; const SCULPT_VOX: f32 = 0.0625; -const SCULPT_AR_ROUND: f32 = 0.03; -const SCULPT_HAND_ZC: f32 = 0.185; const SCULPT_ZREF_ARROW: f32 = 0.2; const SCULPT_ZREF_HAND: f32 = 0.185; const SCULPT_LAMP_DIST: f32 = 1.9; @@ -630,13 +670,6 @@ fn s_opu(a: vec2, b: vec2) -> vec2 { return select(b, a, a.x < b.x); } -fn s_capsule(p: vec3, a: vec3, b: vec3, r: f32) -> f32 { - let pa = p - a; - let ba = b - a; - let h = saturate(dot(pa, ba) / dot(ba, ba)); - return length(pa - ba * h) - r; -} - fn s_round_box(p: vec3, b: vec3, r: f32) -> f32 { let q = abs(p) - b + vec3(r); return length(max(q, vec3(0.0))) + min(max(q.x, max(q.y, q.z)), 0.0) - r; @@ -660,31 +693,6 @@ fn s_rot(v: vec2, a: f32) -> vec2 { return vec2(c * v.x - s * v.y, s * v.x + c * v.y); } -const SCULPT_ARROW = array, 7>( - vec2(0.0, 0.0), vec2(0.0, -0.86), vec2(0.215, -0.665), vec2(0.37, -1.0), - vec2(0.53, -0.93), vec2(0.38, -0.60), vec2(0.64, -0.60) -); - -fn s_arrow2(p: vec2) -> f32 { - var d = dot(p - SCULPT_ARROW[0], p - SCULPT_ARROW[0]); - var s = 1.0; - var j = 6; - for (var i = 0; i < 7; i++) { - let e = SCULPT_ARROW[j] - SCULPT_ARROW[i]; - let w = p - SCULPT_ARROW[i]; - let b = w - e * saturate(dot(w, e) / dot(e, e)); - d = min(d, dot(b, b)); - let c0 = p.y >= SCULPT_ARROW[i].y; - let c1 = p.y < SCULPT_ARROW[j].y; - let c2 = e.x * w.y > e.y * w.x; - if (c0 && c1 && c2) || (!c0 && !c1 && !c2) { - s = -s; - } - j = i; - } - return s * sqrt(d); -} - fn s_star5(p0: vec2, r: f32, rf: f32) -> f32 { let k1 = vec2(0.809016994375, -0.587785252292); let k2 = vec2(-0.809016994375, -0.587785252292); @@ -698,233 +706,426 @@ fn s_star5(p0: vec2, r: f32, rf: f32) -> f32 { return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); } -fn s_arrow_solid(p: vec3, h: f32, re: f32, dome: f32) -> f32 { - let d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - let hh = h + dome * smoothstep(0.0, 0.07, -d2); - return s_extrude(d2, p.z - (SCULPT_HOVER + h + dome), hh, re); +fn s_vesica(p0: vec2, r: f32, d: f32) -> f32 { + let p = abs(p0); + let b = sqrt(r * r - d * d); + return select(length(p + vec2(d, 0.0)) - r, length(p - vec2(0.0, b)), (p.y - b) * d > p.x * b); } -fn s_piping(p: vec3, ztop: f32) -> f32 { - let d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - return length(vec2(d2 + 0.075, p.z - ztop)) - 0.017; -} +// Curseurs cercles (cf. HLSL) : Studio Ink, Pop Coral, Star Sprout. +const RIM_POLY = array, 28>( + vec2(0.0, -0.06), vec2(0.0, -0.7712), vec2(0.165, -0.6325), vec2(0.313, -0.9318), + vec2(0.4234, -0.8557), vec2(0.2672, -0.5686), vec2(0.456, -0.5393), + vec2(-0.173, -0.5237), vec2(-0.016, -0.75), vec2(0.329, -0.75), vec2(0.411, -0.592), + vec2(0.411, -0.43), vec2(0.19, -0.435), vec2(-0.03, -0.44), + vec2(-0.1825, -0.6215), vec2(0.0037, -0.9195), vec2(0.3462, -0.9195), vec2(0.467, -0.6906), + vec2(0.467, -0.49), vec2(0.2, -0.49), vec2(-0.03, -0.49), + vec2(-0.1807, -0.5534), vec2(0.0194, -0.872), vec2(0.3485, -0.872), vec2(0.47, -0.6795), + vec2(0.47, -0.49), vec2(0.2, -0.49), vec2(-0.03, -0.49) +); -fn s_glove(p: vec3, zc: f32) -> f32 { - let index = s_capsule(p, vec3(0.0, -0.10, zc), vec3(0.0, -0.52, zc), 0.098); - let palm = s_round_box(p - vec3(0.185, -0.70, zc), vec3(0.255, 0.19, 0.105), 0.1); - let f1 = s_capsule(p, vec3(0.17, -0.57, zc + 0.012), vec3(0.17, -0.41, zc + 0.045), 0.086); - let f2 = s_capsule(p, vec3(0.31, -0.59, zc + 0.01), vec3(0.31, -0.45, zc + 0.04), 0.08); - let f3 = s_capsule(p, vec3(0.435, -0.625, zc + 0.005), vec3(0.435, -0.52, zc + 0.03), 0.07); - let thumb = s_capsule(p, vec3(0.03, -0.77, zc + 0.03), vec3(-0.165, -0.60, zc + 0.065), 0.082); - var d = s_smin(palm, min(f1, min(f2, f3)), 0.035); - d = s_smin(d, index, 0.05); - return s_smin(d, thumb, 0.05); -} +const RIM_GLOVE = array, 27>( + vec4(0.0, -0.1077, 0.0, -0.6), vec4(0.1493, -0.3279, 0.1493, -0.6), + vec4(0.2863, -0.3654, 0.2863, -0.6), vec4(0.4193, -0.4043, 0.4193, -0.6), + vec4(-0.2, -0.478, -0.075, -0.625), vec4(0.0788, -0.25, 0.0788, -0.3992), + vec4(0.2188, -0.3, 0.2188, -0.4234), vec4(0.3552, -0.33, 0.3552, -0.4534), + vec4(-0.0845, -0.5386, 0.762, 0.648), + vec4(0.0, -0.1186, 0.0, -0.65), vec4(0.1614, -0.3838, 0.1614, -0.65), + vec4(0.3177, -0.427, 0.3177, -0.65), vec4(0.4645, -0.4848, 0.4645, -0.65), + vec4(-0.215, -0.535, -0.1208, -0.6864), vec4(0.0832, -0.3, 0.0832, -0.48), + vec4(0.2414, -0.35, 0.2414, -0.482), vec4(0.3959, -0.41, 0.3959, -0.533), + vec4(-0.122, -0.5718, 0.867, 0.498), + vec4(0.0, -0.122, 0.0, -0.65), vec4(0.172, -0.387, 0.172, -0.65), + vec4(0.3267, -0.427, 0.3267, -0.65), vec4(0.4727, -0.4867, 0.4727, -0.65), + vec4(-0.199, -0.483, -0.102, -0.645), vec4(0.0927, -0.3, 0.0927, -0.476), + vec4(0.2493, -0.35, 0.2493, -0.478), vec4(0.402, -0.41, 0.402, -0.52), + vec4(-0.0993, -0.5267, 0.858, 0.514) +); -fn s_cuff(p: vec3, zc: f32) -> f32 { - let q = p - vec3(0.185, -0.925, zc); - let ab = vec2(0.272, 0.12); - let e = (length(q.xz / ab) - 1.0) * min(ab.x, ab.y); - return s_extrude(e, q.y, 0.07, 0.06); -} +const RIM_GLOVE_R = array, 9>( + vec4(0.0552, 0.047, 0.045, 0.0375), vec4(0.052, 0.0235, 0.0225, 0.0225), vec4(0.0552, -0.548, 0.0525, 0.0), + vec4(0.064, 0.0585, 0.0585, 0.0515), vec4(0.06, 0.0197, 0.0178, 0.0172), vec4(0.0648, -0.6, 0.058, 0.0), + vec4(0.066, 0.0553, 0.0553, 0.0507), vec4(0.063, 0.024, 0.022, 0.02), vec4(0.0647, -0.555, 0.056, 0.0) +); -fn s_sprout(p: vec3, c: vec3) -> vec2 { - let q = p - c; - let st2 = s_star5(q.xy, 0.125, 0.52) - 0.022; - var r = vec2(s_extrude(st2, q.z, 0.038, 0.034), 3.0); - let e = vec3(abs(q.x) - 0.032, q.y + 0.005, q.z - 0.036); - r = s_opu(r, vec2(length(e) - 0.0135, 5.0)); - let l1 = vec3(s_rot(q.xy - vec2(-0.04, 0.15), -0.6), q.z); - let l2 = vec3(s_rot(q.xy - vec2(0.045, 0.155), 0.7), q.z); - let leaves = min(s_ellipsoid(l1, vec3(0.03, 0.058, 0.02)), s_ellipsoid(l2, vec3(0.03, 0.058, 0.02))); - return s_opu(r, vec2(leaves, 4.0)); +fn s_rim_poly(p: vec2, base: i32) -> f32 { + var d = dot(p - RIM_POLY[base], p - RIM_POLY[base]); + var s = 1.0; + var j = base + 6; + for (var i = base; i < base + 7; i++) { + let e = RIM_POLY[j] - RIM_POLY[i]; + let w = p - RIM_POLY[i]; + let b = w - e * saturate(dot(w, e) / dot(e, e)); + d = min(d, dot(b, b)); + let c0 = p.y >= RIM_POLY[i].y; + let c1 = p.y < RIM_POLY[j].y; + let c2 = e.x * w.y > e.y * w.x; + if (c0 && c1 && c2) || (!c0 && !c1 && !c2) { + s = -s; + } + j = i; + } + return s * sqrt(d); } -// Pixel Candy : une ligne par entier, bit c = colonne c (cf. HLSL et `sculpt.rs`). -const SCULPT_ARROWPIX = array(0, 1, 3, 7, 31, 63, 127, 255, 511, 63, 55, 115, 113, 224, 224, 64); -const SCULPT_ARROWBACK = array( - 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 255, 511, 511, 999, 995, 960, 192 +fn s_rim_glove(p: vec2, g: i32, palm: f32) -> vec2 { + let r0 = RIM_GLOVE_R[3 * g]; + let r1 = RIM_GLOVE_R[3 * g + 1]; + let r2 = RIM_GLOVE_R[3 * g + 2]; + let i = 9 * g; + var f = min(sd_segment(p, RIM_GLOVE[i].xy, RIM_GLOVE[i].zw) - r0.x, + sd_segment(p, RIM_GLOVE[i + 1].xy, RIM_GLOVE[i + 1].zw) - r0.y); + f = min(f, sd_segment(p, RIM_GLOVE[i + 2].xy, RIM_GLOVE[i + 2].zw) - r0.z); + f = min(f, sd_segment(p, RIM_GLOVE[i + 3].xy, RIM_GLOVE[i + 3].zw) - r0.w); + f = min(f, sd_segment(p, RIM_GLOVE[i + 4].xy, RIM_GLOVE[i + 4].zw) - r1.x); + var grooves = min(sd_segment(p, RIM_GLOVE[i + 5].xy, RIM_GLOVE[i + 5].zw) - r1.y, + sd_segment(p, RIM_GLOVE[i + 6].xy, RIM_GLOVE[i + 6].zw) - r1.z); + grooves = min(grooves, sd_segment(p, RIM_GLOVE[i + 7].xy, RIM_GLOVE[i + 7].zw) - r1.w); + let sil = s_smin(palm, f, 0.03); + let v = RIM_GLOVE[i + 8]; + let wedge = max(max(-dot(p - v.xy, v.zw), p.x + r2.x), r2.y - p.y); + return vec2(sil, max(sil, -min(grooves, wedge))); +} + +fn s_piece(d: f32, dc: f32, sil: f32, z: f32, w: f32, zt: f32, h: f32, bump: f32, mat: f32) -> vec2 { + let tray = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + zt), 0.5 * (zt - SCULPT_HOVER), 0.012); + let bead = length(vec2(d - 0.5 * w, z - zt)) - 0.5 * w; + let u = saturate(-dc / 0.045); + let cushion = 0.8 * s_extrude(dc, z - zt, h + 0.022 * u * (2.0 - u) + bump, 0.75 * h); + return s_opu(vec2(min(tray, bead), 5.0), vec2(cushion, mat)); +} + +fn s_paper(d: f32, sil: f32, z: f32, w: f32) -> vec2 { + let navy = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + 0.15), 0.5 * (0.15 - SCULPT_HOVER), 0.006); + let sheet = s_extrude(d, z - 0.16, 0.01, 0.004); + return s_opu(vec2(navy, 5.0), vec2(sheet, 1.0)); +} + +fn s_bump(p: vec2, c: vec2, r: f32) -> f32 { + let k = saturate(1.0 - dot(p - c, p - c) / (r * r)); + return k * k; +} + +fn s_dash(p: vec2, a: vec2, b0: vec2, ra: f32, rb: f32) -> f32 { + let q0 = p - a; + let b = b0 - a; + let hb = dot(b, b); + let q = vec2(abs(dot(q0, vec2(b.y, -b.x))), dot(q0, b)) / hb; + let c = vec2(sqrt(hb - (ra - rb) * (ra - rb)), ra - rb); + let k = c.x * q.y - c.y * q.x; + if k < 0.0 { + return sqrt(hb * dot(q, q)) - ra; + } + if k > c.x { + return sqrt(hb * (dot(q, q) + 1.0 - 2.0 * q.y)) - rb; + } + return dot(c, q) - ra; +} + +fn s_rimmed(p: vec3, theme: i32, shape: i32) -> vec2 { + let arrow = shape == 0; + let g = select(select(2, 1, theme == 0), 0, theme == 4); + let poly = s_rim_poly(p.xy, select(7 * g + 7, 0, arrow)); + var body = vec2(poly, poly); + var w = 0.06; + var zt = 0.17; + var h = 0.03; + var bump = 0.022 * s_bump(p.xy, vec2(0.15, -0.45), 0.3); + var back = 1e9; + if !arrow { + w = RIM_GLOVE_R[3 * g + 2].z; + for (var i = 0; i < 2; i++) { + let q = p.xy + select(vec2(0.0), vec2(0.03, 0.04), i == 1); + let v = s_rim_glove(q, g, select(poly, s_rim_poly(q, 7 * g + 7), i == 1) - 0.05); + if i == 1 { + back = v.x - w; + } else { + body = v; + if theme != 2 || v.x - w <= 0.0 { + break; + } + } + } + zt = 0.185; + h = 0.028; + bump = 0.03 * s_bump(p.xy, vec2(0.19, -0.62), 0.28); + } + var dc = body.y; + if arrow && theme == 0 { + body = body - vec2(0.015); + w = 0.045; + dc = 1.0; + } + var r: vec2; + if theme == 2 { + r = s_paper(body.y, body.x, p.z, w); + } else { + r = s_piece(body.y, dc, body.x, p.z, w, zt, h, bump, 1.0); + } + if theme == 0 { + if arrow { + let band = s_extrude(abs(poly + 0.0075) - 0.0225, p.z - zt, 0.028, 0.012); + r = s_opu(r, vec2(band, 2.0)); + } + return r; + } + if theme == 2 { + var dash: f32; + if arrow { + let back = s_rim_poly(p.xy + vec2(0.025, 0.05), 0) - 0.06; + r = s_opu(r, vec2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 2.0)); + dash = min(s_dash(p.xy, vec2(-0.1179, -0.1374), vec2(-0.1799, -0.0443), 0.028, 0.045), + s_dash(p.xy, vec2(-0.1347, -0.2493), vec2(-0.248, -0.2056), 0.026, 0.042)); + } else { + r = s_opu(r, vec2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 3.0)); + dash = min(s_dash(p.xy, vec2(0.216, -0.1695), vec2(0.2593, -0.0685), 0.028, 0.042), + s_dash(p.xy, vec2(0.3054, -0.2339), vec2(0.3949, -0.1652), 0.0275, 0.041)); + } + return s_opu(r, vec2(s_extrude(dash, p.z - 0.11, 0.06, 0.006), 3.0)); + } + if !arrow { + let cq = p.xy - vec2(0.1541, -0.9047); + let bq = abs(vec2(cq.x, cq.y - 0.2066 * cq.x * cq.x)) - vec2(0.214, 0.0361); + let cuff = length(max(bq, vec2(0.0))) + min(max(bq.x, bq.y), 0.0) - 0.03; + r = s_opu(r, s_piece(cuff, cuff, cuff, p.z, 0.0477, 0.225, 0.025, 0.0, 2.0)); + } + let c = select(vec2(0.15, -0.885), vec2(0.6118, -0.8079), arrow); + let rs = select(0.128, 0.1678, arrow); + let zs = select(0.285, 0.23, arrow); + let q = s_rot(p.xy - c, select(0.0, 0.2443, arrow)) / rs; + let star = (s_star5(q, 0.82, 0.55) - 0.18) * rs; + let l1 = s_rot(q - vec2(-0.33, 1.38), -0.925); + let l2 = s_rot(q - vec2(0.53, 1.37), 0.873); + let leaves = min(s_vesica(l1, 0.4296, 0.2626), s_vesica(l2, 0.4296, 0.2626)) * rs; + r = s_opu(r, s_piece(leaves, leaves, leaves, p.z, 0.038, zs - 0.02, 0.018, 0.0, 4.0)); + r = s_opu(r, s_piece(star, star, star, p.z, 0.038, zs, 0.022, 0.018 * s_bump(q, vec2(0.0), 1.0), 3.0)); + let e = vec3(abs(q.x) - 0.25, q.y - 0.08, (p.z - zs - 0.056) / rs); + return s_opu(r, vec2(s_ellipsoid(e, vec3(0.075, 0.13, 0.1)) * rs, 5.0)); +} + +// Pixel Candy : tables générées par scripts/generate-pixel-candy-voxels.mjs (cf. HLSL). +// +const PIX_BODY = array(1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 127, 247, 243, 480, 192, 48, 120, 120, 120, 504, 4088, 32760, 65534, 65535, 65535, 65534, 32766, 32764, 16380, 16376, 16376); +const PIX_LINE = array(1, 3, 5, 9, 17, 33, 65, 129, 257, 513, 1985, 73, 149, 147, 288, 192, 48, 72, 72, 72, 456, 3656, 29256, 37454, 32777, 32769, 32770, 16386, 16388, 8196, 8200, 16376); +const PIX_HI = array(0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 18, 34, 32, 64, 0, 0, 16, 16, 16, 16, 16, 16, 16, 22, 18, 4, 4, 8, 8, 16, 0); +const PIX_SHADE = array(0, 0, 0, 4, 8, 16, 32, 64, 128, 448, 40, 36, 64, 64, 128, 0, 0, 32, 32, 32, 32, 288, 2336, 18720, 16384, 16384, 16384, 8192, 8192, 4096, 8160, 0); +const PIX_RECT_N = array(0, 17, 29); +const PIX_GRID = array, 2>( + vec4(11, 16, 0, 0), + vec4(16, 16, 16, 0) ); -const SCULPT_HANDPIX = array(4, 14, 14, 14, 110, 878, 7022, 7022, 8190, 8191, 8191, 8191, 8190, 4092, 4092); -const SCULPT_HANDBACK = array( - 28, 62, 62, 62, 510, 4094, 32766, 32766, 32766, 32767, 32767, 32767, 32767, 32767, 32766, 16380, 16380 +const PIX_ORIGIN = array, 2>( + vec2(0.0, 0.0), + vec2(-0.3125, 0.0) ); -const SCULPT_HANDSPAN = array, 17>( - vec2(2.0, 4.0), vec2(1.0, 5.0), vec2(1.0, 5.0), vec2(1.0, 5.0), vec2(1.0, 8.0), - vec2(1.0, 11.0), vec2(1.0, 14.0), vec2(1.0, 14.0), vec2(1.0, 14.0), vec2(0.0, 14.0), - vec2(0.0, 14.0), vec2(0.0, 14.0), vec2(0.0, 14.0), vec2(0.0, 14.0), vec2(1.0, 14.0), - vec2(2.0, 13.0), vec2(2.0, 13.0) +const PIX_BOX = array, 2>( + vec4(0.3438, -0.5, 0.3438, 0.5), + vec4(0.1875, -0.5, 0.5, 0.5) ); +const PIX_RECT = array, 29>( + vec4(0.0313, -0.0313, 0.0313, 0.0313), + vec4(0.0625, -0.0938, 0.0625, 0.0313), + vec4(0.0938, -0.1563, 0.0938, 0.0313), + vec4(0.125, -0.2188, 0.125, 0.0313), + vec4(0.1563, -0.2813, 0.1563, 0.0313), + vec4(0.1875, -0.3438, 0.1875, 0.0313), + vec4(0.2188, -0.4063, 0.2188, 0.0313), + vec4(0.25, -0.4688, 0.25, 0.0313), + vec4(0.2813, -0.5313, 0.2813, 0.0313), + vec4(0.3125, -0.5938, 0.3125, 0.0313), + vec4(0.3438, -0.6563, 0.3438, 0.0313), + vec4(0.2188, -0.7188, 0.2188, 0.0313), + vec4(0.0938, -0.7813, 0.0938, 0.0313), + vec4(0.375, -0.8125, 0.125, 0.0625), + vec4(0.0625, -0.8438, 0.0625, 0.0313), + vec4(0.4375, -0.9063, 0.125, 0.0313), + vec4(0.4375, -0.9688, 0.0625, 0.0313), + vec4(0.0, -0.0313, 0.0625, 0.0313), + vec4(0.0, -0.1563, 0.125, 0.0938), + vec4(0.0625, -0.2813, 0.1875, 0.0313), + vec4(0.1563, -0.3438, 0.2813, 0.0313), + vec4(0.25, -0.4063, 0.375, 0.0313), + vec4(0.2188, -0.4688, 0.4688, 0.0313), + vec4(0.1875, -0.5625, 0.5, 0.0625), + vec4(0.2188, -0.6563, 0.4688, 0.0313), + vec4(0.1875, -0.7188, 0.4375, 0.0313), + vec4(0.2188, -0.7813, 0.4063, 0.0313), + vec4(0.1875, -0.8438, 0.375, 0.0313), + vec4(0.2188, -0.9375, 0.3438, 0.0625) +); +// -fn s_grid_origin(shape: i32) -> vec2 { - return select(vec2(-2.5 * SCULPT_VOX, 0.0), vec2(0.0), shape == 0); -} - -fn s_bit(row: i32, r: i32, c: i32, rows: i32, cols: i32) -> bool { - return r >= 0 && r < rows && c >= 0 && c < cols && ((row >> u32(clamp(c, 0, 31))) & 1) == 1; +fn s_pix_bit(m: i32, c: i32) -> bool { + return ((m >> u32(clamp(c, 0, 31))) & 1) == 1; } -fn s_occ(id: vec2, shape: i32) -> bool { - let c = i32(id.x); - let r = -i32(id.y) - 1; - if shape == 0 { - return s_bit(SCULPT_ARROWPIX[clamp(r, 0, 15)], r, c, 16, 16); - } - return s_bit(SCULPT_HANDPIX[clamp(r, 0, 14)], r, c, 15, 13); +fn s_pix_body(shape: i32, c: i32, r: i32) -> bool { + let g = PIX_GRID[shape]; + return c >= 0 && c < g.x && r >= 0 && r < g.y && s_pix_bit(PIX_BODY[g.z + clamp(r, 0, g.y - 1)], c); } -fn s_occ_back(id: vec2, shape: i32) -> bool { - let c = i32(id.x) + 1; - let r = -i32(id.y); - if shape == 0 { - return s_bit(SCULPT_ARROWBACK[clamp(r, 0, 17)], r, c, 18, 17); - } - return s_bit(SCULPT_HANDBACK[clamp(r, 0, 16)], r, c, 17, 15); +fn s_pix_cell(p: vec2, shape: i32) -> vec2 { + let o = PIX_ORIGIN[shape]; + return vec2(i32(floor((p.x - o.x) / SCULPT_VOX)), i32(floor((o.y - p.y) / SCULPT_VOX))); } fn s_voxels(p: vec3, shape: i32) -> vec2 { - let o = s_grid_origin(shape); - let cell = floor((p.xy - o) / SCULPT_VOX); - let bc = select(vec2(0.25, -0.47), vec2(0.33, -0.5), shape == 0); - let bh = select(vec2(0.48, 0.55), vec2(0.44, 0.61), shape == 0); - let bq = max(abs(p.xy - bc) - bh, vec2(0.0)); + let b = PIX_BOX[shape]; + let bq = max(abs(p.xy - b.xy) - b.zw, vec2(0.0)); let bz = max(abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07, 0.0); - let far = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); - var dF = far; - var dB = far; - let half_cell = vec3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.04); + var d = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); + let o = PIX_ORIGIN[shape]; + let cell = s_pix_cell(p.xy, shape); + let cube = vec3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.07); for (var j = -1; j <= 1; j++) { for (var i = -1; i <= 1; i++) { - let id = cell + vec2(f32(i), f32(j)); - let q = vec3(p.xy - (o + (id + vec2(0.5)) * SCULPT_VOX), p.z); - if s_occ(id, shape) { - dF = min(dF, s_round_box(q - vec3(0.0, 0.0, SCULPT_HOVER + 0.1), half_cell, 0.009)); - } - if s_occ_back(id, shape) { - dB = min(dB, s_round_box(q - vec3(0.0, 0.0, SCULPT_HOVER + 0.04), half_cell, 0.009)); + let c = cell.x + i; + let r = cell.y + j; + if s_pix_body(shape, c, r) { + let q = vec3(p.x - o.x - (f32(c) + 0.5) * SCULPT_VOX, p.y - o.y + (f32(r) + 0.5) * SCULPT_VOX, + p.z - SCULPT_HOVER - 0.07); + d = min(d, s_round_box(q, cube, 0.006)); } } } - return select(vec2(dB, 7.0), vec2(dF, 6.0), dF < dB); + return vec2(d, 6.0); } -fn s_pixel_hand_dist(p: vec2) -> f32 { +fn s_pixel_outline(p: vec2, shape: i32) -> f32 { var d = 1e9; - for (var r = 0; r < 17; r++) { - let x = (SCULPT_HANDSPAN[r] + vec2(-3.5, -2.5)) * SCULPT_VOX; - let y = vec2(-f32(r), 1.0 - f32(r)) * SCULPT_VOX; - let q = max(max(vec2(x.x, y.x) - p, p - vec2(x.y, y.y)), vec2(0.0)); - d = min(d, length(q)); + for (var i = PIX_RECT_N[shape]; i < PIX_RECT_N[shape + 1]; i++) { + let q = abs(p - PIX_RECT[i].xy) - PIX_RECT[i].zw; + d = min(d, length(max(q, vec2(0.0))) + min(max(q.x, q.y), 0.0)); } return d; } -fn s_gem_edge(p: vec3, a: vec2, b: vec2, z0: f32) -> f32 { - let e = b - a; - let len = length(e); - let d = e / len; - let q = p.xy - a; - let dist = dot(q, vec2(-d.y, d.x)); - let along = abs(dot(q, d) - 0.5 * len); - let z = p.z - z0; - let girdle = (z - 0.035 - 2.2 * dist) * 0.4138; - let crown = (z - 0.07 - 0.6 * dist + 0.18 * along) * 0.8438; - return max(-dist, max(girdle, crown)); -} - -fn s_gem_arrow(p: vec3) -> f32 { - let z0 = SCULPT_HOVER + 0.03; - let slab = max(p.z - z0 - 0.2, z0 - 0.03 - p.z); - let head = max(slab, max(s_gem_edge(p, vec2(-0.02, 0.03), vec2(-0.02, -0.88), z0), - max(s_gem_edge(p, vec2(-0.02, -0.88), vec2(0.66, -0.61), z0), - s_gem_edge(p, vec2(0.66, -0.61), vec2(-0.02, 0.03), z0)))); - let tail = max(slab, max(max(s_gem_edge(p, vec2(0.215, -0.665), vec2(0.37, -1.0), z0), - s_gem_edge(p, vec2(0.37, -1.0), vec2(0.53, -0.93), z0)), - max(s_gem_edge(p, vec2(0.53, -0.93), vec2(0.38, -0.60), z0), - s_gem_edge(p, vec2(0.38, -0.60), vec2(0.215, -0.665), z0)))); - return min(head, tail); -} - -fn s_facet_capsule(p: vec3, a: vec3, b: vec3, r: f32, spin: f32) -> f32 { - let u = normalize(b - a); - let v = normalize(cross(u, vec3(0.0, 0.0, 1.0))); - let w = cross(u, v); - let q = p - b; - let h = dot(q, u); - let rad = vec2(dot(q, v), dot(q, w)); - var d = max(h - r, -dot(p - a, u) - r); - for (var k = 0; k < 6; k++) { - let an = spin + f32(k) * 1.0471976; - let s = dot(rad, vec2(cos(an), sin(an))); - d = max(d, s - r); - d = max(d, dot(rad, vec2(cos(an + 0.5236), sin(an + 0.5236))) * 0.8660 + h * 0.5 - r); - d = max(d, s * 0.5 + h * 0.8660 - r); +// ---- Prism Glow : un cristal en MAILLAGE ---- (port du HLSL, dont les commentaires font foi) +// Les facettes tracees sur son dessin, en triangles dans le repere du modele ; le serti est la +// silhouette extrudee (champ de distance), le cristal est lance de rayons boite par boite. +// prism mesh: generated by design/cursors/prism-glow/model/export_compositor.py +var PRISM_TRI_START = array(0, 31); +var PRISM_TRI_COUNT = array(31, 130); +var PRISM_SIL_START = array(0, 25); +var PRISM_SIL_COUNT = array(25, 42); +var PRISM_OUT_START = array(67, 76); +var PRISM_OUT_COUNT = array(9, 34); +var PRISM_BOX_START = array(0, 4); +var PRISM_BOX_COUNT = array(4, 18); +struct PrismMesh { + tris: array, 644>, + poly: array, 110>, + boxes: array, 44>, +} +const PRISM_BEVEL: f32 = 0.012883; +// end of the prism mesh + +const PRISM_IOR: f32 = 1.61; +const PRISM_DISPERSION: f32 = 0.035; +const PRISM_GLOW: f32 = 0.45; +const PRISM_FOLD: f32 = 0.35; +const PRISM_BOUNCES: i32 = 6; +const PRISM_MAT: f32 = 9.0; + +fn prism_shape() -> i32 { + let id = sculpt_id(); + if id == 3 || id == 4 { + return id - 3; } - return d; + return -1; } -const SCULPT_FACETS = array, 10>( - vec3(1.0, 0.0, 0.0), vec3(0.0, 1.0, 0.0), vec3(0.0, 0.0, 1.0), - vec3(0.7071, 0.7071, 0.0), vec3(0.7071, 0.0, 0.7071), vec3(0.0, 0.7071, 0.7071), - vec3(0.5774, 0.5774, 0.5774), vec3(0.4472, 0.0, 0.8944), vec3(0.0, 0.4472, 0.8944), - vec3(0.3015, 0.3015, 0.9045) -); +// Drapeaux d'un triangle : paroi (bit 0), aretes reelles (bits 1 a 3). +fn prism_flags(k: i32) -> i32 { + return i32(prism.tris[4 * k].w + 0.5); +} -fn s_facet_ellipsoid(p: vec3, r: vec3) -> f32 { - let q = abs(p); - var d = -1e9; - for (var i = 0; i < 10; i++) { - d = max(d, dot(q, SCULPT_FACETS[i]) - length(r * SCULPT_FACETS[i])); +fn prism_poly_dist(p: vec2, i0: i32, n: i32) -> f32 { + var vj = prism.poly[i0 + n - 1].xy; + var d = 1e9; + var sgn = 1.0; + for (var i = 0; i < n; i++) { + let vi = prism.poly[i0 + i].xy; + let e = vj - vi; + let w = p - vi; + let b = w - e * saturate(dot(w, e) / dot(e, e)); + d = min(d, dot(b, b)); + let c0 = p.y >= vi.y; + let c1 = p.y < vj.y; + let c2 = e.x * w.y > e.y * w.x; + if (c0 && c1 && c2) || (!c0 && !c1 && !c2) { + sgn = -sgn; + } + vj = vi; } - return d; + return sgn * sqrt(d); } -fn s_crystal_hand(p: vec3) -> f32 { - let zc = SCULPT_HAND_ZC; - var d = s_facet_ellipsoid(p - vec3(0.185, -0.70, zc), vec3(0.3, 0.235, 0.125)); - d = min(d, s_facet_capsule(p, vec3(0.0, -0.52, zc), vec3(0.0, -0.10, zc), 0.098, 0.3)); - d = min(d, s_facet_capsule(p, vec3(0.17, -0.57, zc + 0.012), vec3(0.17, -0.41, zc + 0.045), 0.086, 0.1)); - d = min(d, s_facet_capsule(p, vec3(0.31, -0.59, zc + 0.01), vec3(0.31, -0.45, zc + 0.04), 0.08, 0.5)); - d = min(d, s_facet_capsule(p, vec3(0.435, -0.625, zc + 0.005), vec3(0.435, -0.52, zc + 0.03), 0.07, 0.2)); - return min(d, s_facet_capsule(p, vec3(0.03, -0.77, zc + 0.03), vec3(-0.165, -0.60, zc + 0.065), 0.082, 0.7)); +fn prism_rim(p: vec3, shape: i32) -> f32 { + let half_t = model_thick() * 0.5; + let sil = prism_poly_dist(p.xy, PRISM_SIL_START[shape], PRISM_SIL_COUNT[shape]); + let w = vec2(sil + PRISM_BEVEL, abs(p.z + half_t) - (half_t - PRISM_BEVEL)); + return min(max(w.x, w.y), 0.0) + length(max(w, vec2(0.0))) - PRISM_BEVEL; +} + +fn prism_tri(ro: vec3, rd: vec3, k: i32) -> f32 { + let v0 = prism.tris[4 * k].xyz; + let e1 = prism.tris[4 * k + 1].xyz; + let e2 = prism.tris[4 * k + 2].xyz; + let pv = cross(rd, e2); + let det = dot(e1, pv); + if abs(det) < 1e-10 { + return -1.0; + } + let inv = 1.0 / det; + let sv = ro - v0; + let u = dot(sv, pv) * inv; + let qv = cross(sv, e1); + let v = dot(rd, qv) * inv; + if u < -2e-4 || v < -2e-4 || u + v > 1.0004 { + return -1.0; + } + return dot(e2, qv) * inv; +} + +// (distance, indice) du triangle du cristal `shape` le plus proche devant (ro, rd) ; (1e9, -1) +// s'il n'y en a pas. Les triangles sont parcourus boîte par boîte (cf. HLSL). +fn prism_trace(ro: vec3, rd: vec3, shape: i32) -> vec2 { + let inv = vec3(1.0) / select(vec3(1e-6), rd, abs(rd) > vec3(1e-6)); + let b0 = PRISM_BOX_START[shape]; + let nb = PRISM_BOX_COUNT[shape]; + var best = 1e9; + var hit = -1; + for (var j = b0; j < b0 + nb; j++) { + let lo = prism.boxes[2 * j]; + let hi = prism.boxes[2 * j + 1]; + let t0 = (lo.xyz - ro) * inv; + let t1 = (hi.xyz - ro) * inv; + let tn = min(t0, t1); + let tf = max(t0, t1); + let tmin = max(max(tn.x, tn.y), tn.z); + let tmax = min(min(tf.x, tf.y), tf.z); + if tmax < max(tmin, 1e-5) || tmin > best { + continue; + } + let k0 = i32(lo.w + 0.5); + let n = i32(hi.w + 0.5); + for (var i = k0; i < k0 + n; i++) { + let t = prism_tri(ro, rd, i); + if t > 1e-5 && t < best { + best = t; + hit = i; + } + } + } + return vec2(best, f32(hit)); } fn sculpt_proto(p: vec3, theme: i32, shape: i32) -> vec2 { if theme == 3 { return s_voxels(p, shape); } - if theme == 1 { - return vec2(select(s_crystal_hand(p), s_gem_arrow(p), shape == 0), 8.0); - } - var body: vec2; - var star: vec3; - if shape == 0 { - var h = 0.075; - var re = 0.03; - var dome = 0.0; - if theme == 2 { - h = 0.07; - re = 0.06; - dome = 0.035; - } - if theme == 4 { - h = 0.07; - re = 0.055; - dome = 0.025; - } - body = vec2(s_arrow_solid(p, h, re, dome), 1.0); - if theme == 0 { - body = s_opu(body, vec2(s_piping(p, SCULPT_HOVER + 2.0 * h - 0.004), 2.0)); - } - star = vec3(0.57, -0.86, SCULPT_HOVER + 2.0 * h + dome); - } else { - body = s_opu(vec2(s_glove(p, SCULPT_HAND_ZC), 1.0), vec2(s_cuff(p, SCULPT_HAND_ZC), 2.0)); - star = vec3(0.185, -0.93, SCULPT_HAND_ZC + 0.15); - } - if theme == 4 { - return s_opu(body, s_sprout(p, star)); - } - return body; + return s_rimmed(p, theme, shape); } fn sculpt_point(q: vec3) -> vec3 { @@ -940,14 +1141,18 @@ fn sculpt_eval(q: vec3, occ: bool) -> vec2 { let id = sculpt_id() - 1; let theme = id / 2; let shape = id % 2; + if theme == 1 { + // Prism Glow : son serti seul, dans le repere du modele (le cristal est trace a part). + return vec2(prism_rim(q, shape), 8.0); + } let p = sculpt_point(q); var r: vec2; if occ && theme == 3 { - let d2 = select(s_pixel_hand_dist(p.xy), s_arrow2(p.xy) - 1.2 * SCULPT_VOX, shape == 0); + let d2 = s_pixel_outline(p.xy, shape); let dz = abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07; r = vec2(length(max(vec2(d2, dz), vec2(0.0))) + min(max(d2, dz), 0.0), 7.0); } else { - r = sculpt_proto(p, select(theme, 2, occ && theme == 1), shape); + r = sculpt_proto(p, theme, shape); } return vec2(r.x * sculpt_units(), r.y); } @@ -956,7 +1161,7 @@ fn sd_sprite2(p: vec2) -> f32 { let lo = layer.color.rg; let c = clamp(p, lo, lo + sprite_size()); // Niveau 0 explicite : la marche est une boucle a sortie anticipee (pas de derivees). - let d = textureSampleLevel(texU, samp, (c - lo) / sprite_size(), 0.0).r; + let d = textureSampleLevel(texMask, samp, (c - lo) / sprite_size(), 0.0).r; let o = p - c; let out2 = dot(o, o); let e = max(d, 0.0); @@ -1016,7 +1221,7 @@ fn model_albedo(p: vec2) -> vec3 { let g = vec2(sd_sprite2(p + vec2(e, 0.0)) - sd_sprite2(p - vec2(e, 0.0)), sd_sprite2(p + vec2(0.0, e)) - sd_sprite2(p - vec2(0.0, e))); let q = p - g / max(length(g), 1e-6) * max(sd_sprite2(p) + MODEL_RIM_INSET * texel, 0.0); - return textureSampleLevel(texY, samp, (q - layer.color.rg) / sprite_size(), 0.0).rgb; + return textureSampleLevel(texDof, samp, (q - layer.color.rg) / sprite_size(), 0.0).rgb; } struct SculptMat { @@ -1028,61 +1233,60 @@ struct SculptMat { } fn s_pixel_colour(p: vec3, shape: i32) -> vec3 { - let id = floor((p.xy - s_grid_origin(shape)) / SCULPT_VOX); - if !s_occ(id + vec2(-1.0, 0.0), shape) || !s_occ(id + vec2(0.0, -1.0), shape) { - return s_lin(0.52, 0.91, 0.77); + let cell = s_pix_cell(p.xy, shape); + let g = PIX_GRID[shape]; + let row = g.z + clamp(cell.y, 0, g.y - 1); + if s_pix_bit(PIX_LINE[row], cell.x) { + return s_lin(0.29, 0.12, 0.36); } - if !s_occ(id + vec2(1.0, 0.0), shape) || !s_occ(id + vec2(0.0, 1.0), shape) { + if s_pix_bit(PIX_HI[row], cell.x) { return s_lin(1.0, 0.78, 0.87); } - return s_lin(1.0, 0.50, 0.71); + if s_pix_bit(PIX_SHADE[row], cell.x) { + return s_lin(0.87, 0.27, 0.51); + } + return s_lin(1.0, 0.435, 0.66); } fn sculpt_material(mat: f32, p: vec3, theme: i32, shape: i32) -> SculptMat { let primary = mat < 1.5; if theme == 0 { - if shape == 0 && primary { - return SculptMat(s_lin(0.10, 0.10, 0.115), 0.3, 0.2, 0.0, 0.9); - } - if shape == 0 { - return SculptMat(s_lin(0.94, 0.91, 0.84), 0.45, 0.4, 0.2, 0.3); - } - if primary { - return SculptMat(s_lin(0.95, 0.92, 0.85), 0.55, 0.35, 0.35, 0.25); + if mat < 2.5 { + return SculptMat(s_lin(0.95, 0.92, 0.85), 0.3, 0.6, 0.35, 0.25); } - return SculptMat(s_lin(0.17, 0.18, 0.22), 0.35, 0.6, 0.0, 0.6); + return SculptMat(s_lin(0.1, 0.1, 0.11), 0.55, 0.05, 0.0, 0.1); + } + if theme == 1 { + return SculptMat(s_lin(0.02, 0.05, 0.33), 0.35, 0.5, 0.05, 0.3); } if theme == 2 { - if shape == 0 { - return SculptMat(s_lin(1.0, 0.40, 0.30), 0.5, 0.45, 0.4, 0.25); + if (primary && shape == 0) || (mat > 2.5 && mat < 3.5 && shape == 1) { + return SculptMat(s_lin(1.0, 0.40, 0.30), 0.85, 0.08, 0.15, 0.03); } - if primary { - return SculptMat(s_lin(1.0, 0.79, 0.16), 0.5, 0.45, 0.4, 0.25); + if mat < 3.5 { + return SculptMat(s_lin(1.0, 0.80, 0.10), 0.85, 0.08, 0.15, 0.03); } - return SculptMat(s_lin(0.18, 0.20, 0.29), 0.4, 0.5, 0.0, 0.4); + return SculptMat(s_lin(0.09, 0.13, 0.45), 0.9, 0.05, 0.05, 0.02); } if theme == 4 { if primary && shape == 0 { - return SculptMat(s_lin(0.62, 0.91, 0.78), 0.22, 0.8, 0.25, 0.6); + return SculptMat(s_lin(0.68, 0.93, 0.80), 0.25, 0.7, 0.3, 0.3); } if primary { - return SculptMat(s_lin(0.96, 0.94, 0.88), 0.5, 0.35, 0.35, 0.25); + return SculptMat(s_lin(0.97, 0.95, 0.90), 0.3, 0.6, 0.35, 0.25); } if mat < 2.5 { - return SculptMat(s_lin(0.62, 0.91, 0.78), 0.25, 0.7, 0.25, 0.5); + return SculptMat(s_lin(0.52, 0.87, 0.78), 0.25, 0.7, 0.3, 0.3); } if mat < 3.5 { - return SculptMat(s_lin(1.0, 0.80, 0.20), 0.3, 0.6, 0.3, 0.4); + return SculptMat(s_lin(1.0, 0.75, 0.25), 0.25, 0.7, 0.3, 0.3); } if mat < 4.5 { - return SculptMat(s_lin(0.38, 0.80, 0.55), 0.35, 0.5, 0.35, 0.3); + return SculptMat(s_lin(0.62, 0.92, 0.72), 0.3, 0.6, 0.3, 0.35); } - return SculptMat(s_lin(0.16, 0.12, 0.10), 0.2, 0.8, 0.0, 0.5); + return SculptMat(s_lin(0.07, 0.15, 0.33), 0.7, 0.15, 0.05, 0.08); } - if mat < 6.5 { - return SculptMat(s_pixel_colour(p, shape), 0.45, 0.35, 0.15, 0.2); - } - return SculptMat(s_lin(0.36, 0.18, 0.54), 0.45, 0.35, 0.1, 0.2); + return SculptMat(s_pixel_colour(p, shape), 0.65, 0.1, 0.12, 0.04); } fn model_env(d: vec3, rough: f32, l: vec3, fill: vec3) -> vec3 { @@ -1094,36 +1298,6 @@ fn model_env(d: vec3, rough: f32, l: vec3, fill: vec3) -> vec3 vec3 { - let k = saturate(k0) * 3.0; - let a = s_lin(0.20, 0.95, 1.0); - let b = s_lin(0.15, 0.42, 1.0); - let c = s_lin(0.45, 0.25, 0.95); - let d = s_lin(0.88, 0.50, 1.0); - if k < 1.0 { - return mix(a, b, k); - } - if k < 2.0 { - return mix(b, c, k - 1.0); - } - return mix(c, d, k - 2.0); -} - -fn model_shade_crystal(n: vec3, rd: vec3, L: vec3, fall: f32, l: vec3, fill: vec3) -> vec3 { - let cosi = saturate(dot(-rd, n)); - let F = 0.04 + 0.96 * pow(1.0 - cosi, 5.0); - let t = refract(rd, n, 1.0 / 1.6); - let k = 0.42 + 0.9 * dot(vec2(n.x, -n.y), vec2(0.7557, -0.6549)) - + 0.6 * dot(vec2(t.x, -t.y), vec2(0.6, -0.8)); - let body = vec3(s_gem(k - 0.08).r, s_gem(k).g, s_gem(k + 0.08).b); - var col = body * (0.1 + 1.8 * fall * pow(max(dot(n, L), 0.0), 2.5)); - col = col + body * model_env(reflect(t, vec3(0.0, 0.0, 1.0)) * vec3(1.0, 1.0, -1.0), 0.15, l, fill) * 0.5; - col = col + vec3(pow(max(dot(reflect(rd, n), L), 0.0), 30.0) * 2.0); - col = col + model_env(reflect(rd, n), 0.05, l, fill) * F; - col = col + s_lin(0.5, 0.9, 1.0) * pow(1.0 - cosi, 4.0) * 0.6; - return col; -} - fn model_tonemap(c0: vec3) -> vec3 { let start = 0.76; let x = min(c0.r, min(c0.g, c0.b)); @@ -1142,14 +1316,15 @@ fn model_tonemap(c0: vec3) -> vec3 { fn model_shade(q: vec3, n: vec3, rd: vec3, L: vec3, fall: f32, sh: f32, ao: f32, mat: f32, l: vec3, fill: vec3) -> vec3 { let id = sculpt_id(); - if id > 0 && (id - 1) / 2 == 1 { - return model_tonemap(model_shade_crystal(n, rd, L, fall, l, fill)); - } var m: SculptMat; var gloss = 1.0; if id > 0 { let p = sculpt_point(q) - vec3(n.x, -n.y, n.z) * 0.01; m = sculpt_material(mat, p, (id - 1) / 2, (id - 1) % 2); + if id == 3 || id == 4 { + // Le serti de Prism Glow, comme un sprite : brillant sur son arrondi seulement. + gloss = 1.0 - smoothstep(0.97, 0.995, abs(n.z)); + } } else { m = SculptMat(pow(model_albedo(q.xy), vec3(2.2)), 0.45, 0.35, 0.2, 0.3); gloss = 1.0 - smoothstep(0.97, 0.995, abs(n.z)); @@ -1171,13 +1346,160 @@ fn model_shade(q: vec3, n: vec3, rd: vec3, L: vec3, fall: f3 return model_tonemap(col); } +// ---- Le cristal de Prism Glow ---- + +fn prism_fresnel(c: f32, ior: f32) -> f32 { + var f0 = (ior - 1.0) / (ior + 1.0); + f0 = f0 * f0; + return f0 + (1.0 - f0) * pow(1.0 - c, 5.0); +} + +// (distance, facette) du cristal sur le rayon ; (1e9, -1) si rate, hors de Prism Glow, ou sur une +// paroi sous le dessus du serti (le rayon a traverse le serti avant). +fn prism_primary(ro: vec3, rd: vec3) -> vec2 { + let shape = prism_shape(); + if shape < 0 { + return vec2(1e9, -1.0); + } + let r = prism_trace(ro, rd, shape); + let h = i32(r.y); + if h >= 0 && (prism_flags(h) & 1) == 1 && ro.z + rd.z * r.x < 0.0 { + return vec2(1e9, -1.0); + } + return r; +} + +fn prism_normal(h: i32, rd: vec3) -> vec3 { + let n = prism.tris[4 * h + 3].xyz; + return select(n, -n, dot(n, rd) > 0.0); +} + +fn prism_seg(p: vec3, a: vec3, b: vec3) -> f32 { + let pa = p - a; + let ba = b - a; + return length(pa - ba * saturate(dot(pa, ba) / dot(ba, ba))); +} + +// Distance, en pixels, de `p` aux aretes reelles de la facette `h` (plis et bord du serti). +fn prism_edge_px(p: vec3, h: i32, px: f32) -> f32 { + let a = prism.tris[4 * h].xyz; + let b = a + prism.tris[4 * h + 1].xyz; + let c = a + prism.tris[4 * h + 2].xyz; + let bits = prism_flags(h) >> 1u; + var d = 1e9; + if (bits & 1) != 0 { + d = min(d, prism_seg(p, b, c)); + } + if (bits & 2) != 0 { + d = min(d, prism_seg(p, c, a)); + } + if (bits & 4) != 0 { + d = min(d, prism_seg(p, a, b)); + } + return d / px; +} + +// La rotation inverse de `world_to_plane` : du repere du plan a celui de la camera. +fn plane_to_world(v: vec3, f: ModelFrame) -> vec3 { + let x = v.x * f.c.z - v.y * f.s.z; + let y = v.x * f.s.z + v.y * f.c.z; + let z = -x * f.s.y + v.z * f.c.y; + return vec3(x * f.c.y + v.z * f.s.y, y * f.c.x - z * f.s.x, y * f.s.x + z * f.c.x); +} + +// L'image composee la ou le rayon retombe sur le plan (cf. HLSL) : sa copie au binding 1. +fn prism_screen(q: vec3, d: vec3, nz: vec3, hz: f32, tip: vec3, unit: f32, f: ModelFrame) -> vec3 { + let denom = dot(d, nz); + if denom > -1e-4 { + return SCULPT_SCREEN * 0.2; + } + let g = q + d * ((hz - dot(q, nz)) / denom); + let w = plane_to_world(tip + unit * model_to_plane(g, f), f); + let k = 1.0 - w.z / layer.src.z; + if k < 1e-3 { + return SCULPT_SCREEN * 0.2; + } + let uv = layer.dst.xy + ((w.xy + layer.mb.zw) / k - layer.src.xy) / layer.quad_px * layer.dst.zw; + if any(uv < vec2(0.0)) || any(uv > vec2(1.0)) { + return SCULPT_SCREEN * 0.2; + } + return pow(textureSampleLevel(texY, samp, uv, 0.0).rgb, vec3(2.2)); +} + +fn prism_shade(p: vec3, n: vec3, rd: vec3, h: i32, px: f32, l: vec3, fill: vec3, + nz: vec3, hz: f32, tip: vec3, unit: f32, f: ModelFrame) -> vec3 { + let shape = prism_shape(); + let facet = vec3(prism.tris[4 * h + 1].w, prism.tris[4 * h + 2].w, prism.tris[4 * h + 3].w); + let F = prism_fresnel(saturate(-dot(rd, n)), PRISM_IOR); + let zb = -model_thick(); + var trans = vec3(0.0); + for (var ch = 0; ch < 3; ch++) { + // Le rouge plie le moins, le bleu le plus ; le canal passe par un masque (comme au HLSL). + let mask = vec3(f32(ch == 0), f32(ch == 1), f32(ch == 2)); + let ior = PRISM_IOR + PRISM_DISPERSION * f32(ch - 1); + var d = refract(rd, n, 1.0 / ior); + var pos = p; + var thr = 1.0; + var acc = 0.0; + for (var b = 0; b < PRISM_BOUNCES; b++) { + let hit = prism_trace(pos, d, shape); + let hh = i32(hit.y); + let th = hit.x; + var tb = 1e9; + if d.z < -1e-6 { + tb = (zb - pos.z) / d.z; + } + if tb <= th { + let qb = pos + d * tb; + let d2 = refract(d, vec3(0.0, 0.0, 1.0), ior); + if dot(d2, d2) < 1e-8 { + pos = qb; + d = reflect(d, vec3(0.0, 0.0, 1.0)); + continue; + } + acc = acc + thr * dot(prism_screen(qb, d2, nz, hz, tip, unit, f), mask); + thr = 0.0; + break; + } + if hh < 0 { + break; + } + pos = pos + d * th; + let rn = prism.tris[4 * hh + 3].xyz; + if (prism_flags(hh) & 1) == 1 && pos.z < 0.0 { + acc = acc + thr * dot(s_lin(0.03, 0.06, 0.37) * 0.35, mask); + thr = 0.0; + break; + } + let dout = refract(d, -rn, ior); + if dot(dout, dout) < 1e-8 { + d = reflect(d, -rn); + continue; + } + let fi = prism_fresnel(saturate(dot(dout, rn)), ior); + acc = acc + thr * (1.0 - fi) * dot(model_env(dout, 0.05, l, fill), mask); + thr = thr * fi; + d = reflect(d, -rn); + } + trans = trans + acc * mask; + } + var col = F * model_env(reflect(rd, n), 0.05, l, fill) + (1.0 - F) * trans + facet * PRISM_GLOW; + let fold = 1.0 - smoothstep(0.2, 1.0, prism_edge_px(p, h, px)); + col = mix(col, s_lin(0.96, 0.99, 1.0) * 1.3, PRISM_FOLD * fold * (0.35 + 0.65 * saturate(dot(n, l)))); + return model_tonemap(col); +} + // Les passes de la boucle unique de `cursor_model` (cf. HLSL : un seul appel de `model_eval`). const STAGE_MARCH: i32 = 0; const STAGE_NORMAL: i32 = 1; const STAGE_AO: i32 = 2; const STAGE_SELF: i32 = 3; const STAGE_PLANE: i32 = 4; -const STAGE_DONE: i32 = 5; +const STAGE_EDGE: i32 = 5; +const STAGE_SHADE: i32 = 6; +const STAGE_DONE: i32 = 7; + +const MODEL_SUBPIXEL = array, 3>(vec2(0.35, 0.2), vec2(-0.35, 0.2), vec2(0.0, -0.4)); fn model_tetra(k: i32) -> vec3 { let b = vec3(f32(((k + 3) >> 1u) & 1), f32((k >> 1u) & 1), f32(k & 1)); @@ -1199,7 +1521,7 @@ fn cursor_model(local: vec2) -> vec4 { let hi = vec3(layer.color.rg + sprite_size(), layer.trail_a.z); let dw = vec3(local + layer.src.xy, -persp); - let dlen = length(dw); + var dlen = length(dw); // Plan translate de mb.zw dans le repere camera (camera reelle ; 0 sous un angle fixe). let ro = plane_to_model((world_to_plane(vec3(-layer.mb.z, -layer.mb.w, persp), f) - tip) / unit, f); let rd = plane_to_model(world_to_plane(dw / dlen, f), f); @@ -1210,15 +1532,22 @@ fn cursor_model(local: vec2) -> vec4 { let stride = select(1.0, 0.85, sculpt_id() > 0); let lamp = vec3(layer.color.rg + sprite_size() * 0.5, 0.0) + l * SCULPT_LAMP_DIST; - let tb = ray_box(ro, rd, lo - vec3(0.02), hi + vec3(0.02)); + var ray = rd; + var tb = ray_box(ro, ray, lo - vec3(0.02), hi + vec3(0.02)); var stage = select(STAGE_DONE, STAGE_MARCH, tb.x < tb.y && tb.y > 0.0); var plane_next = stage == STAGE_DONE; + // Prism Glow : le cristal, trace au depart de chaque rayon (`tc` son point, `hc` sa facette). + var need_tc = stage == STAGE_MARCH; + var tc = 1e9; + var hc = -1; var k = 0; var t = max(tb.x, 0.0); var best = 1e9; var t_best = t; + var d_best = 0.0; var mat = 0.0; var cov = 0.0; + var cov_s = 0.0; var q = vec3(0.0); var n = vec3(0.0); var L = vec3(0.0); @@ -1230,10 +1559,19 @@ fn cursor_model(local: vec2) -> vec4 { var inside = 0.0; var contact = 0.0; var dropped = 0.0; - for (var it = 0; it < 180; it++) { + var sub = 0; + var shaded = 0; + var acc = vec4(0.0); + for (var it = 0; it < 500; it++) { + if need_tc { + need_tc = false; + let pr = prism_primary(ro, ray); + tc = pr.x; + hc = i32(pr.y); + } if plane_next { plane_next = false; - stage = STAGE_DONE; + stage = select(STAGE_DONE, STAGE_SHADE, cov > 0.0); let denom = dot(rd, nz); if cov < 1.0 && denom < -1e-4 { g = ro + rd * ((hz - dot(ro, nz)) / denom); @@ -1246,11 +1584,48 @@ fn cursor_model(local: vec2) -> vec4 { } } } + if stage == STAGE_SHADE { + if cov_s > 0.0 { + var c: vec3; + if mat > PRISM_MAT - 0.5 { + c = prism_shade(q, n, ray, hc, t_best / dlen, l, fill, nz, hz, tip, unit, f); + } else { + let tl = lamp - q; + let fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); + c = model_shade(q, n, ray, normalize(tl), fall, sh, ao, mat, l, fill); + } + acc = acc + vec4(c * cov_s, cov_s); + } + shaded++; + stage = STAGE_DONE; + if sub > 0 { + let dws = vec3(local + MODEL_SUBPIXEL[3 - sub] + layer.src.xy, -persp); + sub--; + dlen = length(dws); + ray = plane_to_model(world_to_plane(dws / dlen, f), f); + tb = ray_box(ro, ray, lo - vec3(0.02), hi + vec3(0.02)); + cov_s = 0.0; + stage = STAGE_SHADE; + if tb.x < tb.y && tb.y > 0.0 { + stage = STAGE_MARCH; + k = 0; + t = max(tb.x, 0.0); + best = 1e9; + t_best = t; + need_tc = true; + } + } + continue; + } var pos: vec3; if stage == STAGE_MARCH { - pos = ro + rd * t; + pos = ro + ray * t; } else if stage == STAGE_NORMAL { pos = q + 0.002 * model_tetra(k); + } else if stage == STAGE_EDGE { + let side = normalize(cross(n, ray)); + let e = select(side, cross(n, side), k >= 2) * select(1.0, -1.0, k % 2 == 1); + pos = q + e * 0.5 * t_best / dlen; } else if stage == STAGE_AO { pos = q + (0.01 + 0.0175 * f32(k)) * SCULPT_SCALE * n; } else if stage == STAGE_SELF { @@ -1264,22 +1639,57 @@ fn cursor_model(local: vec2) -> vec4 { let d = m.x; if stage == STAGE_MARCH { let fp = t / dlen; - let hit = d < 0.1 * fp; + var hit = d < 0.1 * fp; if hit || d / fp < best { best = select(d / fp, 0.0, hit); t_best = t; + d_best = d; mat = m.y; } t = t + d * stride; k++; + if !hit && t >= tc { + // Le pas depasse le cristal : c'est lui que le rayon touche, avant le serti. + hit = true; + best = 0.0; + t_best = tc; + d_best = 0.0; + mat = PRISM_MAT; + } if hit || t > tb.y || k == 96 { - cov = saturate(1.0 - best); - if cov > 0.0 { - q = ro + rd * t_best; - stage = STAGE_NORMAL; - k = 0; + cov_s = saturate(1.0 - best); + if shaded > 0 { + stage = STAGE_SHADE; + if cov_s > 0.0 { + q = ro + ray * t_best; + n = vec3(0.0); + stage = STAGE_NORMAL; + k = 0; + } } else { - plane_next = true; + cov = cov_s; + if cov > 0.0 { + q = ro + ray * t_best; + stage = STAGE_NORMAL; + k = 0; + } else { + plane_next = true; + } + } + if stage == STAGE_NORMAL && mat > PRISM_MAT - 0.5 { + // Le cristal : la normale de sa facette, ni occlusion ni ombre propre ; un bord + // reel a moins d'un pixel appelle les trois rayons de plus. + n = prism_normal(hc, ray); + k = 0; + if shaded > 0 { + stage = STAGE_SHADE; + } else { + if prism_edge_px(q, hc, t_best / dlen) < 0.75 { + sub = 3; + } + stage = STAGE_DONE; + plane_next = true; + } } } } else if stage == STAGE_NORMAL { @@ -1287,6 +1697,25 @@ fn cursor_model(local: vec2) -> vec4 { k++; if k == 4 { n = normalize(n); + k = 0; + stage = select(STAGE_EDGE, STAGE_SHADE, shaded > 0); + } + } else if stage == STAGE_EDGE { + if m.y != mat || abs(d - d_best) > 0.02 * t_best / dlen { + sub = 3; + } + if k == 0 && mat > 7.5 { + // Le serti de Prism Glow au bord du cristal : son contour a moins d'un pixel. + let ps = prism_shape(); + if ps >= 0 { + let dout = prism_poly_dist(q.xy, PRISM_OUT_START[ps], PRISM_OUT_COUNT[ps]); + if abs(dout) < 0.75 * t_best / dlen { + sub = 3; + } + } + } + k++; + if k == 4 { stage = STAGE_AO; k = 0; ao = 0.0; @@ -1323,7 +1752,7 @@ fn cursor_model(local: vec2) -> vec4 { res = 1.0; k = 1; if !(tbp.x < tbp.y && tbp.y > 0.0) { - stage = STAGE_DONE; + stage = select(STAGE_DONE, STAGE_SHADE, cov > 0.0); } } else { res = min(res, MODEL_SOFTNESS * d / ts.x); @@ -1332,20 +1761,15 @@ fn cursor_model(local: vec2) -> vec4 { if res < 0.002 || ts.x > ts.y || k == 33 { res = saturate(res); dropped = 1.0 - res * res * (3.0 - 2.0 * res); - stage = STAGE_DONE; + stage = select(STAGE_DONE, STAGE_SHADE, cov > 0.0); } } } - var rgb = vec3(0.0); - if cov > 0.0 { - let tl = lamp - q; - let fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); - rgb = model_shade(q, n, rd, normalize(tl), fall, sh, ao, mat, l, fill); - } + let w = 1.0 / f32(max(shaded, 1)); let shadow = inside * max(dropped * MODEL_SHADOW_ALPHA, contact * MODEL_CONTACT_ALPHA); - let a = cov * layer.color.a; - return vec4(rgb * a, a + (1.0 - a) * shadow * layer.color.a); // premultiplie, ombre noire + let a = acc.a * w * layer.color.a; + return vec4(acc.rgb * w * layer.color.a, a + (1.0 - a) * shadow * layer.color.a); // premultiplie, ombre noire } // ---- Impact du clic (mode 16) ---- @@ -1855,12 +2279,14 @@ fn device_frame(local: vec2) -> vec4 { return vec4(rgb * a, a); // premultiplie } -@fragment -fn fs_main(i: VsOut) -> @location(0) vec4 { +// The body of `fs_main`, which only adds the layer's transparency on top of it. +fn fs_layer(i: VsOut) -> vec4 { var rgb: vec3; var alpha: f32; // 1 sauf en detourage, ou il porte le masque du sujet. Cf. la branche fx.z plus bas. var alpha_mask = 1.0; + // 0 where a camera homography finds no camera point (`persp_camera`), 1 everywhere else. + var persp_keep = 1.0; if layer.mode < 0.5 { // Mode 0 — vidéo NV12 + flou de mouvement par vélocité (§8), port 1:1 du @@ -1870,18 +2296,31 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { // du calque sans avoir à transporter un champ de vitesse. let taps = i32(layer.mb.x); let mb_scale = clamp(layer.mb.y, 0.0, 1.0); + var uv_now = i.uv; + if layer.layer_fx.y > 0.5 { + // Camera homography: the texture uv comes from the fragment's place in the quad + // (`src` is not used), scaled to the valid part of the decoder texture. + let cam = persp_camera((i.pout - layer.dst.xy) / layer.dst.zw); + persp_keep = cam.z; + uv_now = cam.xy * layer.fx.xy; + } // `taps` d'abord : un draw qui a oublié `dst_prev` le laisse à zéro, et // la division par `dst_prev.zw` produirait des UV infinis. Dégrader vers // le chemin net est le seul échec acceptable pour un effet cosmétique. if taps <= 1 || mb_scale <= 0.001 || layer.dst_prev.z <= 0.0 || layer.dst_prev.w <= 0.0 { - rgb = sample_yuv(i.uv); + rgb = sample_yuv(uv_now); } else { let localp = (i.pout - layer.dst_prev.xy) / layer.dst_prev.zw; - let uv_prev = layer.src_prev.xy + localp * (layer.src_prev.zw - layer.src_prev.xy); - let duv = i.uv - uv_prev; + var uv_prev = layer.src_prev.xy + localp * (layer.src_prev.zw - layer.src_prev.xy); + if layer.layer_fx.y > 0.5 { + // The previous frame's uv is the same map at the quad's previous place. + let cam_prev = persp_camera(localp); + uv_prev = select(uv_now, cam_prev.xy * layer.fx.xy, cam_prev.z > 0.5); + } + let duv = uv_now - uv_prev; let duv_blur = duv * mb_scale; if dot(duv_blur, duv_blur) < 1e-9 { - rgb = sample_yuv(i.uv); + rgb = sample_yuv(uv_now); } else { // Borne 16 en dur, identique au HLSL et au MSL : `taps` vient d'un // uniform et une boucle sans borne statique ne se déroule pas. @@ -1891,7 +2330,7 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { let step = 1.0 / f32(taps - 1); for (var k: i32 = 0; k < 16; k = k + 1) { if k >= taps { break; } - acc = acc + sample_yuv(i.uv - duv_blur * (1.0 - f32(k) * step)); + acc = acc + sample_yuv(uv_now - duv_blur * (1.0 - f32(k) * step)); } rgb = acc / f32(taps); } @@ -1902,16 +2341,23 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { // l'etendue valide de la texture webcam pour ramener uv dans l'espace du masque. let effect = layer.fx.z; if effect > 0.5 { - let mask_uv = i.uv / max(layer.fx.xy, vec2(1e-6)); + let mask_uv = uv_now / max(layer.fx.xy, vec2(1e-6)); let person = clamp(textureSample(texMask, samp, mask_uv).r, 0.0, 1.0); if effect > 2.5 { rgb = mix(layer.color.rgb, rgb, person); } else if effect > 1.5 { - rgb = mix(blur_webcam_bg(i.uv, layer.fx.w, layer.quad_px, i.local), rgb, person); + rgb = mix(blur_webcam_bg(uv_now, layer.fx.w, layer.quad_px, i.local, layer.fx.xy), rgb, person); } else { alpha_mask = person; } } + + // Desk-view cover: the camera is being tilted, so the whole picture is blurred and + // dimmed (cover.x = strength, cover.y = radius in quad px, cover.z = dim at full cover). + if layer.cover.x > 0.001 { + let hidden = blur_webcam_radius(uv_now, layer.cover.y, layer.quad_px, i.local, layer.fx.xy); + rgb = mix(rgb, hidden, layer.cover.x) * (1.0 - layer.cover.z * layer.cover.x); + } } else if layer.mode < 1.5 { // Mode 1 — couleur pleine. rgb = layer.color.rgb; @@ -2106,9 +2552,13 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { let plane_px = layer.dst_prev.xy; let p = vec2(r.x, r.y) * plane_px - plane_px * 0.5; let rad = max(layer.radius_px, 0.0); - let d = select(sd_round_rect(p, plane_px * 0.5, rad), - sd_screen_under_bar(p, plane_px * 0.5, rad, layer.color.z), - layer.dst_prev.z > 0.5); + // Une branche et non un `select`, qui evaluerait les DEUX distances (trois SDF par pixel). + var d: f32; + if layer.dst_prev.z > 0.5 { + d = sd_screen_under_bar(p, plane_px * 0.5, rad, layer.color.z); + } else { + d = sd_round_rect(p, plane_px * 0.5, rad); + } // Slot d'un layout en bloc (`color.w` = rayon de ses coins, px ; 0 ailleurs, sans effet) : // `dst` EST le slot, dont le rect arrondi rogne le plan (`ScreenMask`, cf. HLSL). let tilt_a = (1.0 - smoothstep(0.0, 1.5, d)) @@ -2238,18 +2688,26 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { frame_rgb = mix(frame_rgb, vec3(0.157, 0.784, 0.251), disc_cov(q, vec2(x0 + 2.0 * dx, bar * 0.5), dr)); let fa = cov * layer.color.a; return vec4(frame_rgb * fa, fa); // premultiplie - } else if layer.mode > 14.5 && layer.mode < 15.5 { - // Mode 15 -- curseur modelise (`cursor_model`). Clip « Clip to canvas » dans - // `dst_prev`, comme au mode 13. - if i.pout.x < layer.dst_prev.x || i.pout.x > layer.dst_prev.x + layer.dst_prev.z - || i.pout.y < layer.dst_prev.y || i.pout.y > layer.dst_prev.y + layer.dst_prev.w { + } else if layer.mode > 14.5 && layer.mode < 17.5 { + // Modes 15 a 17 -- les modeles 3D, que seul le pipeline `LAYER_MODELS` compile (cf. le + // haut du fichier). Le pipeline plat ne les recoit jamais ; s'il en recevait un, il ne + // dessinerait rien plutot qu'une ombre faite de leurs emplacements. + if !LAYER_MODELS { return vec4(0.0, 0.0, 0.0, 0.0); } - return cursor_model(i.local); - } else if layer.mode > 15.5 && layer.mode < 16.5 { - // Mode 16 -- impact du clic sous le curseur modelise (`cursor_impact`). - return cursor_impact(i.local); - } else if layer.mode > 16.5 && layer.mode < 17.5 { + if layer.mode < 15.5 { + // Mode 15 -- curseur modelise (`cursor_model`). Clip « Clip to canvas » dans + // `dst_prev`, comme au mode 13. + if i.pout.x < layer.dst_prev.x || i.pout.x > layer.dst_prev.x + layer.dst_prev.z + || i.pout.y < layer.dst_prev.y || i.pout.y > layer.dst_prev.y + layer.dst_prev.w { + return vec4(0.0, 0.0, 0.0, 0.0); + } + return cursor_model(i.local); + } + if layer.mode < 16.5 { + // Mode 16 -- impact du clic sous le curseur modelise (`cursor_impact`). + return cursor_impact(i.local); + } // Mode 17 -- cadre d'appareil modelise (`device_frame`). return device_frame(i.local); } else if layer.mode > 17.5 { @@ -2272,7 +2730,7 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { if layer.mode > 4.5 && layer.mode < 5.5 { base_alpha = 1.0; } - alpha = base_alpha * alpha_mask; + alpha = base_alpha * alpha_mask * persp_keep; if layer.radius_px > 0.0 { // Feather ~1.5 px sur le bord du quad — parité exacte avec le HLSL @@ -2282,11 +2740,22 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { // cadre (`sd_screen_under_bar`, `mb.z` = la remontee du contour interieur, px). let halfsz = layer.quad_px * 0.5; let p = i.local - layer.quad_px * 0.5; - let d = select(sd_round_rect(p, halfsz, layer.radius_px), - sd_screen_under_bar(p, halfsz, layer.radius_px, layer.mb.z), - layer.mb.w > 0.5); + // Une branche et non un `select` : celui-ci evaluait les DEUX distances, soit trois SDF + // par pixel de l'ecran au lieu d'une -- 0,6 ms par frame 1080p sur une Radeon 610M. + var d: f32; + if layer.mb.w > 0.5 { + d = sd_screen_under_bar(p, halfsz, layer.radius_px, layer.mb.z); + } else { + d = sd_round_rect(p, halfsz, layer.radius_px); + } alpha *= 1.0 - smoothstep(0.0, 1.5, d); } return vec4(rgb * alpha, alpha); // alpha prémultiplié } + +@fragment +fn fs_main(i: VsOut) -> @location(0) vec4 { + // Premultiplied, so the transparency scales all four channels; 0 leaves them as they are. + return fs_layer(i) * (1.0 - layer.layer_fx.x); +} diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs index c8bc45936..1c74e011e 100644 --- a/crates/compositor/tests/compose_linux.rs +++ b/crates/compositor/tests/compose_linux.rs @@ -799,6 +799,7 @@ fn export_linux_mp4() { source_end_sec: 1.0, webcam_offset_sec: 0.0, has_audio: true, + additional_cameras: Vec::new(), }]; let params = ExportParams { width: 640, diff --git a/crates/compositor/tests/cursor_model_render.rs b/crates/compositor/tests/cursor_model_render.rs index 86d28bf70..70151158f 100644 --- a/crates/compositor/tests/cursor_model_render.rs +++ b/crates/compositor/tests/cursor_model_render.rs @@ -161,13 +161,13 @@ fn cursors_dir() -> String { .replace('\\', "/") } -/// Les cinq thèmes d'origine et les hotspots de leur flèche et de leur main plates -/// (`CURSOR_THEMES`, `src/lib/cursor/cursorThemes.ts`, sur 32). +/// Les thèmes d'origine et les hotspots de leur flèche et de leur main plates (`CURSOR_THEMES`, +/// `src/lib/cursor/cursorThemes.ts`, sur 32). const SCULPTED: [(&str, [f32; 2], [f32; 2]); 5] = [ ("studio-ink", [6.2304, 2.0992], [12.848, 2.0704]), ("prism-glow", [6.3456, 2.0672], [11.968, 2.0352]), ("pop-coral", [10.4768, 2.1792], [12.3456, 2.0]), - ("pixel-candy", [7.3664, 2.0], [13.376, 1.9264]), + ("pixel-candy", [6.5, 2.0], [10.75, 2.0]), ("star-sprout", [4.7232, 2.1152], [13.1712, 2.0384]), ]; @@ -447,6 +447,20 @@ fn opaque_mask(on_blue: &[u8], on_orange: &[u8], bare_blue: &[u8]) -> Vec .collect() } +/// Les pixels que couvre un curseur de VERRE (le cristal de Prism Glow, qui laisse voir le contenu +/// et le réfracte) : opaques, ou changés sur les deux teintes sans y être plus sombres (une ombre +/// ne fait qu'assombrir ; le verre réfracte, reflète et luit). +fn covered_mask(on_blue: &[u8], on_orange: &[u8], bare_blue: &[u8], bare_orange: &[u8]) -> Vec { + let opaque = opaque_mask(on_blue, on_orange, bare_blue); + let px = |img: &[u8], i: usize| [img[i * 4], img[i * 4 + 1], img[i * 4 + 2], 255]; + (0..1280 * 720) + .map(|i| { + let lit = |on: &[u8], bare: &[u8]| px(on, i) != px(bare, i) && luma(px(on, i)) >= luma(px(bare, i)) - 2.0; + opaque[i] || (lit(on_blue, bare_blue) && lit(on_orange, bare_orange)) + }) + .collect() +} + fn iou(a: &[bool], b: &[bool]) -> f32 { let inter = a.iter().zip(b).filter(|(x, y)| **x && **y).count(); let union = a.iter().zip(b).filter(|(x, y)| **x || **y).count(); @@ -765,16 +779,18 @@ fn the_motion_blur_trail_draws_modelled_copies() { assert!(rgba != flat, "la traînée 3D est celle du sprite plat"); } -/// Les dix curseurs sculptés (cinq thèmes, flèche et main), dessinés par le shader et non -/// extrudés d'un PNG : chacun est là, sa pointe sur le hotspot, son corps en bas à droite de -/// celle-ci (y vers le bas : un modèle retourné finirait au-dessus), et il porte son ombre en -/// l'air. +/// Les curseurs sculptés (flèche et main de chaque thème), dessinés par le shader et non extrudés +/// d'un PNG : chacun est là, sa pointe sur le hotspot, son corps en bas à droite de celle-ci (y +/// vers le bas : un modèle retourné finirait au-dessus), et il porte son ombre en l'air. Le +/// cristal de Prism Glow est de verre : son corps compte ce qu'il couvre, et il doit laisser voir +/// le contenu (une bonne part de lui change avec la teinte de l'écran). #[test] fn the_sculpted_cursors_stand_at_the_hotspot() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 1280, 720).expect("compositor"); let (blue, orange) = (FakeFrame::new(&gpu, Tint::Blue), FakeFrame::new(&gpu, Tint::Orange)); let bare = render_any(&comp, &blue, &hidden_json("null"), &resting("sculpt-bare", false)).0; + let bare_orange = render_any(&comp, &orange, &hidden_json("null"), &resting("sculpt-bare", false)).0; let mut failures = Vec::new(); for (theme, ..) in SCULPTED { for state in ["arrow", "pointer"] { @@ -783,7 +799,8 @@ fn the_sculpted_cursors_stand_at_the_hotspot() { let (hover, p) = render(&comp, &blue, &json, &still); let hover_b = render(&comp, &orange, &json, &still).0; save(&format!("sculpt-{theme}-{state}"), &hover); - let mask = opaque_mask(&hover, &hover_b, &bare); + let glass = theme == "prism-glow"; + let mask = if glass { covered_mask(&hover, &hover_b, &bare, &bare_orange) } else { opaque_mask(&hover, &hover_b, &bare) }; let (mut body, mut c, mut near) = (0usize, [0.0f32; 2], f32::MAX); for y in 0..720 { for x in 0..1280 { @@ -819,6 +836,13 @@ fn the_sculpted_cursors_stand_at_the_hotspot() { if shadow * 10 < body * 3 { failures.push(format!("{theme}/{state} : pas d'ombre en l'air ({shadow} px)")); } + if glass { + let through = (0..1280 * 720).filter(|&i| mask[i] && hover[i * 4..i * 4 + 3] != hover_b[i * 4..i * 4 + 3]).count(); + println!("{theme}/{state} : {through} px du corps laissent voir l'écran"); + if through * 10 < body * 3 { + failures.push(format!("{theme}/{state} : le cristal ne laisse voir l'écran que sur {through} px")); + } + } } } assert!(failures.is_empty(), "{failures:#?}"); @@ -865,7 +889,7 @@ fn contact_sheets() { } tilted.save(format!("{dir}/states-touch-flat-and-iso.png")).expect("planche"); - // Les dix curseurs sculptés : un thème par ligne, flèche et main en l'air sur l'écran à plat, + // Les curseurs sculptés : un thème par ligne, flèche et main en l'air sur l'écran à plat, // puis posées sur l'écran incliné. let cases = [("arrow", "null", false), ("pointer", "null", false), ("arrow", r#""iso""#, true), ("pointer", r#""iso""#, true)]; let mut sculpted = image::RgbaImage::new(4 * CELL, SCULPTED.len() as u32 * CELL); diff --git a/crates/compositor/tests/export_timing.rs b/crates/compositor/tests/export_timing.rs index 34d5e8de4..adb90bb43 100644 --- a/crates/compositor/tests/export_timing.rs +++ b/crates/compositor/tests/export_timing.rs @@ -117,6 +117,7 @@ fn whole_clip(dir: &PathBuf) -> ClipSource { source_end_sec: SOURCE_SEC, webcam_offset_sec: 0.0, has_audio: false, + additional_cameras: Vec::new(), } } diff --git a/crates/compositor/tests/no_camera_stand_in.rs b/crates/compositor/tests/no_camera_stand_in.rs new file mode 100644 index 000000000..d741f12e9 --- /dev/null +++ b/crates/compositor/tests/no_camera_stand_in.rs @@ -0,0 +1,109 @@ +//! Sans caméra, la preview compose partout avec la frame écran à la place de la webcam : le +//! décodeur remplaçant n'est plus ni avancé ni recherché (`live::open_webcam_or_stand_in`). +//! +//! Couvre les chemins qui composent : le seek en pause (`present_frame`), y compris au-delà de +//! la dernière image, la lecture libre (`step`), et la recomposition à l'arrêt (`recompose`). +//! Une frame webcam nulle les ferait tous rendre `false`, preview figée. Vérifie aussi que le +//! remplaçant ne bouge pas, `seek_active` compris : composer suffit à ne rien prouver, puisqu'il +//! ouvre le même fichier valide que l'écran. +//! +//! La CI ne le joue pas : il faut un GPU et une source vidéo, sinon il se saute. N'importe quel +//! MP4 d'au moins 4 s convient : +//! +//! ```sh +//! OPENSCREEN_LIVE_SOURCE=/chemin/source.mp4 cargo test -p openscreen-compositor --test no_camera_stand_in -- --nocapture +//! ``` + +#![cfg(any(windows, target_os = "macos"))] + +use openscreen_compositor::compositor::Compositor; +use openscreen_compositor::config; +use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::frame_geometry::live_params_from_scene; +use openscreen_compositor::live::Player; +use openscreen_compositor::scene::Scene; + +fn scene_json(source: &str) -> String { + let s = source.replace('\\', "/"); + format!( + r##"{{ + "clips": [{{"screenPath":"{s}","webcamPath":"","sourceStartSec":0,"sourceEndSec":100000,"webcamOffsetSec":0,"hasAudio":false}}], + "layout": {{"preset":"no-webcam","webcamSize":1.0,"webcamShape":"rounded","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, + "effects": {{"padding":0.1,"blur":false,"shadow":0.0,"roundnessFrac":0,"motionBlur":0.0}}, + "background": {{"kind":"color","color":"#303030"}}, + "zoomRegions": [], + "speedRegions": [], + "cursor": {{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}}, + "cropByClip": [null], + "output": {{"width":480,"height":270,"fps":30}} + }}"## + ) +} + +#[test] +fn every_compose_path_works_without_a_camera() { + let Ok(source) = std::env::var("OPENSCREEN_LIVE_SOURCE") else { + println!("SKIP: definir OPENSCREEN_LIVE_SOURCE (voir l'en-tete du fichier)."); + return; + }; + let gpu = Gpu::create(false).expect("device"); + let mut cfg = config::all().pop().expect("au moins une config"); + cfg.zoom = false; + cfg.layout_anim = false; + let comp = Compositor::new_sized(&gpu, 480, 270).expect("compositor"); + let scene = Scene::from_json(&scene_json(&source)).expect("scene valide"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_scene(Some(scene.clone())); + comp.clear_cursor(); + + unsafe { + let mut player = Player::open(&source, "", &gpu).expect("ouvrir la source"); + assert!(!player.webcam_decoder_is_real(), "pas de caméra déclarée : décodeur remplaçant"); + player.set_programme_clock(Some(&scene), 0); + + let stand_in_time = player.webcam_time_sec(); + + assert!(player.present_frame(&comp, &cfg, 1.0).expect("seek"), "seek en pause"); + let start = player.screen_time_sec(); + + // Lecture libre à 2× sur une seconde d'horloge, au pas de 1/60 s. À l'EOF, `step` + // reboucle à 0 : la cible ne serait alors plus jamais atteinte, d'où l'arrêt sur un + // temps qui recule. + let mut target = start; + let mut composed = 0; + for _ in 0..60 { + target += 2.0 / 60.0; + loop { + let before = player.screen_time_sec(); + if !player.step(&comp, &cfg, target).expect("step") { + break; + } + assert!( + player.screen_time_sec() >= before, + "la source a rebouclé avant {target:.2} s : il faut une source d'au moins 4 s" + ); + composed += 1; + } + } + assert!(composed > 0, "la lecture libre n'a composé aucune frame"); + assert!( + player.screen_time_sec() > start + 1.5, + "la lecture libre a avancé de {:.3} s au lieu de ~2 s", + player.screen_time_sec() - start + ); + + assert!(player.recompose(&comp, &cfg).expect("recompose"), "recomposition à l'arrêt"); + + // Au-delà de la dernière image : la dernière image, composée. + assert!(player.present_frame(&comp, &cfg, 100_000.0).expect("seek fin"), "seek au-delà de la fin"); + + assert!(player.seek_active(0.5).expect("seek_active"), "seek_active"); + assert!(player.step(&comp, &cfg, 0.5).expect("step après seek_active"), "step après seek_active"); + + assert_eq!( + player.webcam_time_sec().to_bits(), + stand_in_time.to_bits(), + "le décodeur remplaçant a bougé : il ne doit être ni avancé ni recherché" + ); + } +} diff --git a/crates/compositor/tests/shared_frame_handoff.rs b/crates/compositor/tests/shared_frame_handoff.rs new file mode 100644 index 000000000..2465f2b98 --- /dev/null +++ b/crates/compositor/tests/shared_frame_handoff.rs @@ -0,0 +1,135 @@ +//! La texture qu'une vue live livre par handle NT, rouverte par un AUTRE device D3D11 — la +//! place qu'occupe le processus GPU de Chromium quand Electron l'importe. Ce que le +//! consommateur lit doit être exactement ce que le compositeur a composé : pas une image +//! à moitié copiée, pas une case périmée, et une case redimensionnée doit livrer un handle +//! neuf à la nouvelle taille. +//! +//! Demande un device matériel : saute sans lui (runner sans adaptateur). + +#![cfg(windows)] + +use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::shared_frames::SharedRing; +use windows::core::Interface; +use windows::Win32::Foundation::{HANDLE, HMODULE}; +use windows::Win32::Graphics::Direct3D::{D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL_11_1}; +use windows::Win32::Graphics::Direct3D11::{ + D3D11CreateDevice, ID3D11Device, ID3D11Device1, ID3D11DeviceContext, ID3D11Texture2D, + D3D11_BIND_SHADER_RESOURCE, D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_FLAG, + D3D11_MAPPED_SUBRESOURCE, D3D11_MAP_READ, D3D11_SDK_VERSION, D3D11_SUBRESOURCE_DATA, + D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, +}; +use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_SAMPLE_DESC}; + +fn pattern(width: u32, height: u32, seed: u8) -> Vec { + (0..width * height) + .flat_map(|i| { + let (x, y) = (i % width, i / width); + [x as u8 ^ seed, y as u8, seed, 255] + }) + .collect() +} + +fn texture_with(gpu: &Gpu, width: u32, height: u32, pixels: &[u8]) -> ID3D11Texture2D { + let desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_R8G8B8A8_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let init = D3D11_SUBRESOURCE_DATA { + pSysMem: pixels.as_ptr().cast(), + SysMemPitch: width * 4, + SysMemSlicePitch: 0, + }; + let mut tex = None; + unsafe { gpu.device.CreateTexture2D(&desc, Some(&init), Some(&mut tex)) }.expect("texture"); + tex.expect("texture") +} + +/// Le second device, sans rien de commun avec celui du compositeur que l'adaptateur. +fn consumer() -> (ID3D11Device, ID3D11DeviceContext) { + let mut device = None; + let mut context = None; + unsafe { + D3D11CreateDevice( + None, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + D3D11_CREATE_DEVICE_FLAG(0), + Some(&[D3D_FEATURE_LEVEL_11_1]), + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + } + .expect("second device"); + (device.unwrap(), context.unwrap()) +} + +/// Ce que le consommateur lit à travers le handle, en RGBA serré. +fn read_through(handle: u64, width: u32, height: u32) -> Vec { + let (device, context) = consumer(); + let device1: ID3D11Device1 = device.cast().expect("ID3D11Device1"); + let shared: ID3D11Texture2D = + unsafe { device1.OpenSharedResource1(HANDLE(handle as *mut core::ffi::c_void)) } + .expect("le handle doit s'ouvrir depuis un autre device"); + let mut desc = D3D11_TEXTURE2D_DESC::default(); + unsafe { shared.GetDesc(&mut desc) }; + assert_eq!((desc.Width, desc.Height), (width, height)); + desc.Usage = D3D11_USAGE_STAGING; + desc.BindFlags = 0; + desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ.0 as u32; + desc.MiscFlags = 0; + let mut staging = None; + unsafe { device.CreateTexture2D(&desc, None, Some(&mut staging)) }.expect("staging"); + let staging = staging.unwrap(); + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + let mut out = vec![0u8; (width * height * 4) as usize]; + unsafe { + context.CopyResource(&staging, &shared); + context.Map(&staging, 0, D3D11_MAP_READ, 0, Some(&mut mapped)).expect("map"); + for y in 0..height as usize { + std::ptr::copy_nonoverlapping( + (mapped.pData as *const u8).add(y * mapped.RowPitch as usize), + out.as_mut_ptr().add(y * width as usize * 4), + width as usize * 4, + ); + } + context.Unmap(&staging, 0); + } + out +} + +#[test] +fn another_device_reads_exactly_the_composed_frame() { + let Ok(gpu) = Gpu::create(false) else { + eprintln!("pas de device D3D11 matériel — saute"); + return; + }; + let mut ring = SharedRing::new(&gpu).expect("anneau"); + + let (w, h) = (64, 36); + let first = pattern(w, h, 0x11); + let handle = unsafe { ring.write(0, &texture_with(&gpu, w, h, &first), w, h) }.expect("écriture"); + assert_eq!(read_through(handle, w, h), first); + + // La même case réécrite : même texture, même handle, nouveau contenu. + let second = pattern(w, h, 0x22); + let again = unsafe { ring.write(0, &texture_with(&gpu, w, h, &second), w, h) }.expect("réécriture"); + assert_eq!(again, handle); + assert_eq!(read_through(again, w, h), second); + + // Le rendu change de taille : la case est recréée, avec un handle à la nouvelle taille. + let (w2, h2) = (32, 18); + let resized = pattern(w2, h2, 0x33); + let fresh = unsafe { ring.write(0, &texture_with(&gpu, w2, h2, &resized), w2, h2) }.expect("taille"); + assert_eq!(read_through(fresh, w2, h2), resized); +} diff --git a/crates/poc-d3d/src/bench.rs b/crates/poc-d3d/src/bench.rs index 3da91883b..2b657b91a 100644 --- a/crates/poc-d3d/src/bench.rs +++ b/crates/poc-d3d/src/bench.rs @@ -154,6 +154,7 @@ fn run_bench(args: &[String]) -> Result<()> { source_end_sec: 6.0, // la fixture entière (§ fixture.json : 6 s, 360 frames) webcam_offset_sec: 0.0, has_audio: false, + additional_cameras: Vec::new(), }; let path = format!("{out}/{}_{:?}.mp4", cfg.name, backend).to_lowercase(); let s = pipeline::run_composited_multi( @@ -296,6 +297,7 @@ fn run_gif_bench( source_end_sec: f64::MAX, webcam_offset_sec: 0.0, has_audio: false, + additional_cameras: Vec::new(), }]; for r in 0..repeat { // Each run writes to the same path — the last frame wins. The diff --git a/design/cursors/3d-concept.png b/design/cursors/3d-concept.png index fc828101b..4b7a01c98 100644 Binary files a/design/cursors/3d-concept.png and b/design/cursors/3d-concept.png differ diff --git a/design/cursors/3d-direction.md b/design/cursors/3d-direction.md index 6333c23d0..1205d3174 100644 --- a/design/cursors/3d-direction.md +++ b/design/cursors/3d-direction.md @@ -2,21 +2,47 @@ `3d-concept.png` is a visual reference, not a render from OpenScreen. -With the 3D option on, mode 15 ray-marches each original theme's arrow and hand -as a sculpted model: signed distance functions written in the compositor's three -shaders (HLSL, Metal, WGSL), not a PNG face or a height map. -`crates/compositor/src/sculpt.rs` names the ten models and holds their bounding -boxes and hotspots. The scene names the model (`"/"`); text, -resize, move and the other OS states keep their built-in art and the -compositor's beveled extrusion. +With the 3D option on, mode 15 ray-marches the arrow and hand of Studio Ink, Pop +Coral, Pixel Candy and Star Sprout as sculpted models: signed distance functions +written in the compositor's three shaders (HLSL, Metal, WGSL), not a PNG face or +a height map. Prism Glow's arrow and hand are glass crystals: a triangle mesh +traced on its drawing, ray-traced in a navy rim. `crates/compositor/src/sculpt.rs` +names the ten models and holds their bounding boxes and hotspots. The scene names +the model (`"/"`); text, resize, move and the other OS states keep +their built-in art and the compositor's beveled extrusion. | Theme | Arrow model | Hand model | | --- | --- | --- | -| Studio Ink | Matte black body with an ivory piping set in from the edge. | Ivory glove with a dark cuff. | -| Prism Glow | Cut crystal: flat facets, a polygonal silhouette, cyan to violet dispersion. | The same glove cut into faceted fingers and palm. | -| Pop Coral | Soft coral rubber with rounded edges and a gentle dome. | Plump yellow rubber glove with a navy cuff. | -| Pixel Candy | Beveled voxels: a pink face with mint and pale pink edges, purple blocks one step behind. | Voxel glove with the same colours and purple back layer. | -| Star Sprout | Glossy mint ceramic carrying a star with a face and two leaves. | Ivory glove with a mint cuff and the same star. | +| Studio Ink | Black rim, a raised ivory band, a recessed satin black field. | Ivory glove cushion in a black rim, black ridges between the fingers. | +| Prism Glow | A glass crystal cut along the facets of its drawing, set in a thin navy rim. The picture under it shows through, refracted with a slight dispersion; each facet glows faintly in its drawn colour, and a light line marks its folds. | The same crystal: a facet for each plane of the drawn hand, the fingers and thumb included. | +| Pop Coral | Cut paper: flat-topped sheets, no rounded bead or gloss. A coral sheet on a navy sheet, a yellow sheet behind them offset down-left, two yellow click dashes. | A yellow sheet on a navy sheet, the navy showing between the fingers, a coral sheet behind offset down-left, two coral click dashes. | +| Pixel Candy | Simplified pixel art, one cube per pixel: a plum outline, a pink body with a pale highlight down the lit edge and a darker shade along the outline. The 3D model and the 2D sprite come from the same grid. | The same, a pixel hand: the index up, three knuckles apart, the thumb. | +| Star Sprout | Glossy mint cushion framed by the navy outline of its drawing, carrying a star with a face and two leaves. | Ivory glove cushion in the same navy frame, navy ridges between the fingers, a mint cuff and the same star. | + +Studio Ink, Pop Coral and Star Sprout keep their drawn outline, as in +`3d-concept.png` (and `star-sprout/3d-reference.png` for the Star Sprout hand). +Each piece is a tray in the outline colour under its silhouette, a rounded bead +along the stroke and a puffy colour cushion inside it; the grooves between the +fingers become ridges of the outline colour. The three arrows share one polygon +and the three gloves one construction, each glove measured on its own drawing +(`RIM_POLY`, `RIM_GLOVE`). Pop Coral turns the same shapes into flat paper +sheets instead (`s_paper`). What stands in front is theme-specific: Studio +Ink's ivory band, Pop Coral's yellow and coral layers and click dashes, Star +Sprout's cuff, star, leaves and eyes. + +Prism Glow is built from its 2D drawing alone (`prism-glow/source.png`), not from +`3d-concept.png`. `prism-glow/model/` holds the mesh: `prism_geo.py` lists its +vertices, heights and facets and writes `prism.json`; `build_blend.py` builds +`prism-glow.blend` and `prism-glow.glb` from it in Blender; `export_compositor.py` +writes the mesh into `crates/compositor/src/prism_mesh.rs` and the three shaders. +The compositor ray-traces the crystal, its triangles grouped in bounding boxes that +a ray skips when it misses them: refraction per colour +channel, total internal reflection, and an exit through its flat base onto the +picture under the cursor. That picture is a copy of the composed frame taken just +before the cursor is drawn: the recording, then its privacy blurs, then the cursor, +sharp on top, so only blurred pixels show through the glass. The rim is its +silhouette extruded, a distance field +like the other models: it gives the coverage and the shadows. All models share one close lamp: a gradient and a highlight even on flat faces, soft self shadows, ambient occlusion and a studio environment in the @@ -27,4 +53,8 @@ click and size changes. To change a model, edit it in all three shaders (`sculpt_proto`, `sculpt_material`) and keep `sculpt.rs` in step: its boxes must contain the shapes, and a test checks that the shaders mirror its constants. Pixel Candy's -arrow voxels are sampled from the arrow polygon; `sculpt.rs` says how. +voxel grids are tables written by `scripts/generate-pixel-candy-voxels.mjs`: +change a grid there and run the script; it rewrites all three shaders, and a +test checks the tables agree. To change Prism Glow, edit `prism_geo.py`, run it, +run `build_blend.py` in Blender, then `export_compositor.py`; a test checks that +the three shaders carry the mesh of `prism_mesh.rs`. diff --git a/design/cursors/README.md b/design/cursors/README.md index b3a21cd3c..c6cd471c4 100644 --- a/design/cursors/README.md +++ b/design/cursors/README.md @@ -18,14 +18,17 @@ fringe pixels, and resize them into transparent 128 × 128 PNGs under `public/cursors//`. It prints the normalized tip hotspots; copy them into `src/lib/cursor/cursorThemes.ts` after changing a master. -With the 3D option on, the compositor draws each theme's arrow and hand as a -model sculpted in its shaders (`crates/compositor/src/sculpt.rs`), not from these -PNGs. The theme picker and the flat cursor keep using the PNGs. Text, resize, -move and other states remain state-accurate through the built-in art and -receive the shared 3D extrusion. +With the 3D option on, the compositor draws the arrow and hand of Studio Ink, +Pop Coral, Pixel Candy and Star Sprout as models sculpted in its shaders +(`crates/compositor/src/sculpt.rs`), not from these PNGs. Prism Glow's arrow and +hand are glass crystals meshed on its drawing (`prism-glow/model/`). The theme +picker and the flat cursor keep using the PNGs. Text, resize, move and other +states remain state-accurate through the built-in art and receive the shared 3D +extrusion. `contact-sheet.png` shows the sprites enlarged on a light background; `dark-32px.png` shows them at their 32-pixel reference size on a dark background. -The 3D modeling direction and concept image are in `3d-direction.md` and -`3d-concept.png`. +The 3D modeling direction and the reference sheet the models follow are in +`3d-direction.md` and `3d-concept.png`; `star-sprout/3d-reference.png` is a closer +view of the Star Sprout hand. The consolidated acceptance list is in `requirements.md`. diff --git a/design/cursors/pixel-candy/source.png b/design/cursors/pixel-candy/source.png index 79b5c859d..daeed19be 100644 Binary files a/design/cursors/pixel-candy/source.png and b/design/cursors/pixel-candy/source.png differ diff --git a/design/cursors/prism-glow/model/build_blend.py b/design/cursors/prism-glow/model/build_blend.py new file mode 100644 index 000000000..4a941e04c --- /dev/null +++ b/design/cursors/prism-glow/model/build_blend.py @@ -0,0 +1,265 @@ +"""Build the Prism Glow cursor in Blender from prism.json, then save prism-glow.blend and export +prism-glow.glb next to this script. + + blender --background --python build_blend.py + +Per cursor, two objects: the faceted crystal (a closed mesh, one flat face per facet of the 2D art, +with its colour in the face attribute `concept`) and the navy rim (a filled 2D curve with a hole, +extruded and rounded). Units: 1 = 1000 px of source.png, origin on the hotspot, y up, z toward the +viewer, rim top at z = 0, everything's underside at z = -0.04. +""" + +import json +import math +import os + +import bmesh +import bpy +from mathutils import Matrix, Vector + +HERE = globals().get("HERE") or os.path.dirname(os.path.abspath(__file__)) +S = 0.001 +T_RIM = 40.0 # rim and crystal-body thickness, px +BEVEL = 5.0 # rim edge rounding, px +HAND_AT = ((1220 - 208) * S, -(50 - 42) * S) # the hand stands where it is in source.png + +D = json.load(open(os.path.join(HERE, "prism.json"))) + + +def srgb_to_linear(c): + return c / 12.92 if c <= 0.04045 else ((c + 0.055) / 1.055) ** 2.4 + + +def hex_rgba(h): + return tuple(srgb_to_linear(int(h[i : i + 2], 16) / 255) for i in (1, 3, 5)) + (1.0,) + + +def signed_area(pts): + return 0.5 * sum(a[0] * b[1] - b[0] * a[1] for a, b in zip(pts, pts[1:] + pts[:1])) + + +def model_xy(sh, x, y): + hx, hy = sh["hotspot"] + return ((x - hx) * S, -(y - hy) * S) + + +def fresh_collection(name): + col = bpy.data.collections.get(name) or bpy.data.collections.new(name) + if col.name not in bpy.context.scene.collection.children: + bpy.context.scene.collection.children.link(col) + for ob in list(col.objects): + bpy.data.objects.remove(ob) + return col + + +def build_crystal(name, sh): + V = sh["verts"] + bm = bmesh.new() + concept = bm.faces.layers.float_color.new("concept") + top = {k: bm.verts.new((*model_xy(sh, x, y), z * S)) for k, (x, y, z) in V.items()} + outline = list(sh["outline"]) + if signed_area([model_xy(sh, *V[k][:2]) for k in outline]) < 0: + outline.reverse() + bot = {k: bm.verts.new((*model_xy(sh, *V[k][:2]), -T_RIM * S)) for k in outline} + for f, hexcol in zip(sh["facets"], sh["colors"]): + vs = [top[k] for k in f] + if signed_area([v.co.xy[:] for v in vs]) < 0: + vs.reverse() + bm.faces.new(vs)[concept] = hex_rgba(hexcol) + side = hex_rgba("#1a3cc8") + for a, b in zip(outline, outline[1:] + outline[:1]): + bm.faces.new((top[a], bot[a], bot[b], top[b]))[concept] = side + bm.faces.new([bot[k] for k in reversed(outline)])[concept] = side + bmesh.ops.triangulate(bm, faces=[f for f in bm.faces if len(f.verts) > 3], + quad_method="BEAUTY", ngon_method="BEAUTY") + bmesh.ops.recalc_face_normals(bm, faces=bm.faces[:]) + me = bpy.data.meshes.get(name) or bpy.data.meshes.new(name) + bm.to_mesh(me) + bm.free() + # glTF carries colours per corner: mirror the face colour there + corner = me.attributes.get("Col") or me.attributes.new("Col", "FLOAT_COLOR", "CORNER") + for p in me.polygons: + p.use_smooth = False + for li in p.loop_indices: + corner.data[li].color = me.attributes["concept"].data[p.index].color + me.color_attributes.active_color = corner + return bpy.data.objects.new(name, me) + + +def inset(loop, d): + """Offset a closed counter-clockwise loop inward by d (mitred).""" + out = [] + for i in range(len(loop)): + p0, p1, p2 = Vector(loop[i - 1]), Vector(loop[i]), Vector(loop[(i + 1) % len(loop)]) + e0, e1 = (p1 - p0).normalized(), (p2 - p1).normalized() + n0, n1 = Vector((-e0.y, e0.x)), Vector((-e1.y, e1.x)) + m = (n0 + n1).normalized() if (n0 + n1).length > 1e-6 else n0 + out.append(tuple(p1 + m * (d / max(m.dot(n0), 0.25)))) + return out + + +def build_rim(name, sh): + outer = [model_xy(sh, x, y) for x, y in sh["silhouette"]] + if signed_area(outer) < 0: + outer.reverse() + inner = [model_xy(sh, *sh["verts"][k][:2]) for k in sh["outline"]] + # the bevel rounds outward from the spline: pull the outer loop in so the silhouette stays exact + outer = inset(outer, BEVEL * S) + cu = bpy.data.curves.get(name) or bpy.data.curves.new(name, "CURVE") + cu.splines.clear() + cu.dimensions = "2D" + cu.fill_mode = "BOTH" + cu.extrude = (T_RIM / 2 - BEVEL) * S + cu.bevel_depth = BEVEL * S + cu.bevel_resolution = 3 + for loop in (outer, inner): + sp = cu.splines.new("POLY") + sp.points.add(len(loop) - 1) + for p, (x, y) in zip(sp.points, loop): + p.co = (x, y, 0.0, 1.0) + sp.use_cyclic_u = True + ob = bpy.data.objects.new(name, cu) + ob.location.z = -T_RIM / 2 * S + return ob + + +def nodes_of(name): + m = bpy.data.materials.get(name) or bpy.data.materials.new(name) + m.use_nodes = True + m.node_tree.nodes.clear() + return m, m.node_tree, m.node_tree.nodes.new("ShaderNodeOutputMaterial") + + +def material_crystal(): + """Clear glass whose facets glow with their colour in the art (the look chosen on 30/09/2026).""" + m, nt, out = nodes_of("Prism Crystal") + bsdf = nt.nodes.new("ShaderNodeBsdfPrincipled") + attr = nt.nodes.new("ShaderNodeAttribute") + attr.attribute_name = "concept" + nt.links.new(attr.outputs["Color"], bsdf.inputs["Emission Color"]) + bsdf.inputs["Base Color"].default_value = (1, 1, 1, 1) + bsdf.inputs["Emission Strength"].default_value = 0.45 + bsdf.inputs["Transmission Weight"].default_value = 1.0 + bsdf.inputs["IOR"].default_value = 1.61 + bsdf.inputs["Roughness"].default_value = 0.0 + nt.links.new(bsdf.outputs["BSDF"], out.inputs["Surface"]) + return m + + +def material_rim(): + m, nt, out = nodes_of("Prism Rim") + bsdf = nt.nodes.new("ShaderNodeBsdfPrincipled") + bsdf.inputs["Base Color"].default_value = hex_rgba("#040c5a") + bsdf.inputs["Roughness"].default_value = 0.22 + bsdf.inputs["Coat Weight"].default_value = 1.0 + bsdf.inputs["Coat Roughness"].default_value = 0.05 + nt.links.new(bsdf.outputs["BSDF"], out.inputs["Surface"]) + return m + + +def build(): + col = fresh_collection("Prism Glow") + crystal, rim = material_crystal(), material_rim() + obs = [] + for key, at in (("arrow", (0.0, 0.0)), ("hand", HAND_AT)): + c = build_crystal(f"prism_{key}_crystal", D[key]) + r = build_rim(f"prism_{key}_rim", D[key]) + c.data.materials.clear() + c.data.materials.append(crystal) + r.data.materials.clear() + r.data.materials.append(rim) + for ob in (c, r): + ob.location.x, ob.location.y = at + col.objects.link(ob) + obs += [c, r] + return obs + + +def studio(): + """Studio HDRI for light and reflections, the editor screenshot as the screen, a 3/4 camera.""" + sc = bpy.context.scene + w = sc.world or bpy.data.worlds.new("World") + sc.world = w + w.use_nodes = True + nt = w.node_tree + nt.nodes.clear() + env = nt.nodes.new("ShaderNodeTexEnvironment") + env.image = bpy.data.images.load(os.path.join(bpy.utils.system_resource("DATAFILES"), "studiolights", "world", "studio.exr"), check_existing=True) + env.image.pack() # the install path differs per machine; 100 KB inside the .blend does not + bg, flat = nt.nodes.new("ShaderNodeBackground"), nt.nodes.new("ShaderNodeBackground") + flat.inputs["Color"].default_value = (0.06, 0.06, 0.07, 1) + lp, mix, out = nt.nodes.new("ShaderNodeLightPath"), nt.nodes.new("ShaderNodeMixShader"), nt.nodes.new("ShaderNodeOutputWorld") + nt.links.new(env.outputs["Color"], bg.inputs["Color"]) + nt.links.new(lp.outputs["Is Camera Ray"], mix.inputs["Fac"]) + nt.links.new(bg.outputs["Background"], mix.inputs[1]) + nt.links.new(flat.outputs["Background"], mix.inputs[2]) + nt.links.new(mix.outputs["Shader"], out.inputs["Surface"]) + + shot = os.path.normpath(os.path.join(HERE, "..", "..", "..", "..", "public", "preview4.png")) + screen = bpy.data.objects.get("screen") + if screen is None and os.path.exists(shot): + me = bpy.data.meshes.new("screen") + me.from_pydata([(-3, -1.66, 0), (3, -1.66, 0), (3, 1.66, 0), (-3, 1.66, 0)], [], [(0, 1, 2, 3)]) + uv = me.uv_layers.new() + for li, st in enumerate(((0, 0), (1, 0), (1, 1), (0, 1))): + uv.data[li].uv = st + m, mnt, mout = nodes_of("screen") + em, tex = mnt.nodes.new("ShaderNodeEmission"), mnt.nodes.new("ShaderNodeTexImage") + tex.image = bpy.data.images.load(shot, check_existing=True) + mnt.links.new(tex.outputs["Color"], em.inputs["Color"]) + mnt.links.new(em.outputs["Emission"], mout.inputs["Surface"]) + me.materials.append(m) + screen = bpy.data.objects.new("screen", me) + screen.location = (1.9, -1.0, -T_RIM * S - 0.0005) + sc.collection.objects.link(screen) + + target = Vector((0.679, -0.43, 0.0)) + cam = bpy.data.objects.get("cam_34") or bpy.data.objects.new("cam_34", bpy.data.cameras.new("cam_34")) + if cam.name not in sc.collection.objects: + sc.collection.objects.link(cam) + eye = target + Matrix.Rotation(math.radians(-28), 3, "Y") @ Matrix.Rotation(math.radians(-38), 3, "X") @ Vector((0, 0, 4.4)) + f = (target - eye).normalized() + r = f.cross(Vector((0, 1, 0))).normalized() + cam.location = eye + cam.rotation_euler = Matrix((r, r.cross(f), -f)).transposed().to_euler() + cam.data.lens = 85 + sc.camera = cam + sc.render.engine = "CYCLES" + sc.cycles.device = "GPU" + sc.cycles.samples = 128 + sc.cycles.transmission_bounces = sc.cycles.max_bounces = 24 + sc.render.resolution_x, sc.render.resolution_y = 1600, 1000 + sc.view_settings.view_transform = "AgX" + sc.view_settings.look = "AgX - Medium High Contrast" + + +def export_glb(obs, path): + """The rims are curves; glTF wants meshes: bake them into temporary twins for the export.""" + dg = bpy.context.evaluated_depsgraph_get() + twins = [] + for ob in obs: + if ob.type == "CURVE": + name = ob.name + me = bpy.data.meshes.new_from_object(ob.evaluated_get(dg)) + ob.name = name + "_curve" # the twin takes the name the glTF node should carry + tw = bpy.data.objects.new(name, me) + tw.matrix_world = ob.matrix_world + me.materials.append(ob.data.materials[0]) + bpy.context.scene.collection.objects.link(tw) + twins.append((tw, ob, name)) + bpy.ops.object.select_all(action="DESELECT") + for ob in [o for o in obs if o.type == "MESH"] + [tw for tw, _, _ in twins]: + ob.select_set(True) + bpy.ops.export_scene.gltf(filepath=path, export_format="GLB", use_selection=True, + export_vertex_color="ACTIVE", export_apply=True) + for tw, ob, name in twins: + me = tw.data + bpy.data.objects.remove(tw) + bpy.data.meshes.remove(me) + ob.name = name + + +obs = build() +studio() +export_glb(obs, os.path.join(HERE, "prism-glow.glb")) +bpy.ops.wm.save_as_mainfile(filepath=os.path.join(HERE, "prism-glow.blend"), relative_remap=True) diff --git a/design/cursors/prism-glow/model/export_compositor.py b/design/cursors/prism-glow/model/export_compositor.py new file mode 100644 index 000000000..15cd261af --- /dev/null +++ b/design/cursors/prism-glow/model/export_compositor.py @@ -0,0 +1,363 @@ +"""Write the Prism Glow meshes into the compositor: `crates/compositor/src/prism_mesh.rs` and the +matching tables in the three shaders, between their `prism mesh` markers. + + python export_compositor.py # after build_blend.py has written prism-glow.glb + +Everything is expressed in the mode 15 model frame: the unit is the side of the theme's 128 px +sprite (the flat cursor's size), origin on the hotspot, x right, y DOWN, z toward the camera, +rim top at z = 0. The source-to-sprite transform is the one scripts/generate-original-cursor-themes.mjs +applies, so the crystal lands exactly on the flat cursor. +""" + +import json +import math +import struct +from pathlib import Path + +import cv2 +import numpy as np + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[3] +SRC = REPO / "crates" / "compositor" / "src" +SPRITE, INNER, ALPHA_MIN = 128, 112, 28 # generate-original-cursor-themes.mjs +BEVEL_PX = 12.0 # rounding of the rim's edges, source px +EDGE_FOLD_DEG = 25.0 +LEAF = 8 # triangles per bounding box at most +BOX_PAD = 1e-3 # model units: past prism_tri's edge slack + + +def glb_meshes(path): + """{node name: (positions (n,3), triangle indices (m,3), colours (n,3))}, glTF frame (y up).""" + b = path.read_bytes() + n = struct.unpack_from("= ALPHA_MIN + ys, xs = np.nonzero(a) + w, h = xs.max() - xs.min() + 1, ys.max() - ys.min() + 1 + scale = INNER / max(w, h) + tx, ty = (SPRITE - w * scale) / 2, (SPRITE - h * scale) / 2 + hx = (tx + (hotspot[0] - x0 - xs.min()) * scale) / SPRITE + hy = (ty + (hotspot[1] - ys.min()) * scale) / SPRITE + return scale / SPRITE, (hx, hy), ty / SPRITE + + +def crystal_triangles(pos, tri, col, sheet, s): + """Triangles of the crystal (facets and walls; the flat bottom is left to the shader), each + with its outward normal, its colour in the art and which of its edges are real fold lines.""" + # glTF (x, y up, z) -> Blender (x, y, z up), hotspot at the origin, then the model frame + blender = np.c_[pos[:, 0], -pos[:, 2], pos[:, 1]] - np.array([sheet[0], sheet[1], 0.0]) + model = np.c_[blender[:, 0], -blender[:, 1], blender[:, 2]] * (1000.0 * s) + zmin = model[:, 2].min() + tris = [] + for t in tri: + a, b, c = model[t] + nb = np.cross(blender[t[1]] - blender[t[0]], blender[t[2]] - blender[t[0]]) + nb /= np.linalg.norm(nb) + n = np.array([nb[0], -nb[1], nb[2]]) # the y mirror keeps the normal outward + if n[2] < -0.9 and abs(a[2] - zmin) < 1e-6: + continue + tris.append({"v": (a, b, c), "n": n, "col": col[t[0]], "key": tuple(map(tuple, np.round(model[t], 6)))}) + # real edges: the mesh's boundary, a change of facet colour, or a sharp fold + owners = {} + for i, t in enumerate(tris): + for e in range(3): + k = tuple(sorted((t["key"][(e + 1) % 3], t["key"][(e + 2) % 3]))) + owners.setdefault(k, []).append(i) + for i, t in enumerate(tris): + bits = 0 + for e in range(3): + k = tuple(sorted((t["key"][(e + 1) % 3], t["key"][(e + 2) % 3]))) + other = [j for j in owners[k] if j != i] + real = not other or np.abs(tris[other[0]]["col"] - t["col"]).sum() > 1e-3 or ( + tris[other[0]]["n"] @ t["n"] < math.cos(math.radians(EDGE_FOLD_DEG))) + bits |= int(real) << e + t["flags"] = (1 if abs(t["n"][2]) < 0.2 else 0) + 2 * bits + return tris, float(zmin), float(model[:, 2].max()), model + + +def cluster(tris): + """The triangles regrouped by neighbourhood, and one bounding box per group: a ray that + misses a box skips all its triangles. Groups are halves of the centroids' longest extent, + split until they hold LEAF triangles at most; the triangles come back in group order.""" + groups = [] + + def split(idx): + if len(idx) <= LEAF: + groups.append(idx) + return + cent = np.array([np.mean(tris[i]["v"], axis=0) for i in idx]) + axis = int(np.argmax(cent.max(0) - cent.min(0))) + order = [idx[j] for j in np.argsort(cent[:, axis], kind="stable")] + split(order[: len(order) // 2]) + split(order[len(order) // 2 :]) + + split(list(range(len(tris)))) + ordered, leaves = [], [] + for g in groups: + pts = np.array([v for i in g for v in tris[i]["v"]]) + leaves.append((pts.min(0) - BOX_PAD, pts.max(0) + BOX_PAD, len(ordered), len(g))) + ordered += [tris[i] for i in g] + return ordered, leaves + + +def main(): + img = cv2.imread(str(HERE.parent / "source.png"), cv2.IMREAD_UNCHANGED) + geo = json.loads((HERE / "prism.json").read_text()) + meshes = glb_meshes(HERE / "prism-glow.glb") + shapes = [] + for kind, node, (x0, x1), sheet in ( + ("arrow", "prism_arrow_crystal", (0, 887), (0.0, 0.0)), + ("hand", "prism_hand_crystal", (887, 1774), ((1220 - 208) / 1000, -(50 - 42) / 1000)), + ): + hot = geo[kind]["hotspot"] + s, (hx, hy), top = sprite_transform(img, x0, x1, hot) + tris, zmin, zmax, model = crystal_triangles(*meshes[node], sheet, s) + tris, leaves = cluster(tris) + sil = [((x - hot[0]) * s, (y - hot[1]) * s) for x, y in geo[kind]["silhouette"]] + out = [((geo[kind]["verts"][k][0] - hot[0]) * s, (geo[kind]["verts"][k][1] - hot[1]) * s) + for k in geo[kind]["outline"]] + sx = [p[0] for p in sil] + # the draw box: the sprite's full height (1 unit), the silhouette's width + bx0, bx1 = min(sx) - 0.01, max(sx) + 0.01 + shapes.append({ + "kind": kind, "tris": tris, "leaves": leaves, "sil": sil, "out": out, "thick": -zmin, + "height": zmax, + "size": (bx1 - bx0, 1.0), "hotspot": (-bx0 / (bx1 - bx0), hy), "top": top, + "per_px": s, + }) + print(f"{kind}: {len(tris)} triangles in {len(leaves)} boxes, {len(sil)} silhouette points, " + f"box {bx1 - bx0:.4f} x 1, hotspot ({-bx0 / (bx1 - bx0):.4f}, {hy:.4f}), " + f"height {zmax:.4f}, thick {-zmin:.4f}") + write_rust(shapes) + write_shaders(shapes) + + +def f(x): + return f"{x:.6f}".rstrip("0").rstrip(".") if abs(x) > 5e-7 else "0.0" + + +def fl(x): # a float literal every language reads as a float + s = f(x) + return s if "." in s else s + ".0" + + +def bevel(shapes): + return BEVEL_PX * shapes[0]["per_px"] + + +def polygons(shapes): + """Silhouettes, then outlines, in one table; their starts.""" + poly, sil_start, out_start = [], [], [] + for sh in shapes: + sil_start.append(len(poly)) + poly += sh["sil"] + for sh in shapes: + out_start.append(len(poly)) + poly += sh["out"] + return poly, sil_start, out_start + + +def records(shapes): + """Per triangle, four float4: (v0, flags), (e1, r), (e2, g), (n, b).""" + rows = [] + for sh in shapes: + for t in sh["tris"]: + a, b, c = t["v"] + e1, e2 = b - a, c - a + r, g, bl = t["col"] + rows += [(*a, t["flags"]), (*e1, r), (*e2, g), (*t["n"], bl)] + return rows + + +def boxes(shapes): + """Per box, two float4: (lo, first triangle), (hi, triangle count); each model's first box.""" + rows, box_start, k = [], [], 0 + for sh in shapes: + box_start.append(len(rows) // 2) + for lo, hi, first, count in sh["leaves"]: + rows += [(*lo, k + first), (*hi, count)] + k += len(sh["tris"]) + return rows, box_start + + +def write_rust(shapes): + tri_start, rows = [], records(shapes) + k = 0 + for sh in shapes: + tri_start.append(k) + k += len(sh["tris"]) + poly, sil_start, out_start = polygons(shapes) + box_rows, box_start = boxes(shapes) + lines = [ + "//! Les deux curseurs de Prism Glow en MAILLAGE : les facettes tracées sur l'art 2D", + "//! (`design/cursors/prism-glow/model`), taillées en cristal. Fichier GÉNÉRÉ par", + "//! `design/cursors/prism-glow/model/export_compositor.py` : ne pas l'éditer à la main.", + "//!", + "//! Repère du modèle du mode 15 : unité = côté du sprite 128 px du thème, origine au hotspot,", + "//! x à droite, y vers le bas, z vers la caméra, dessus du serti en z = 0. Chaque triangle du", + "//! cristal tient en quatre float4 : (v0, drapeaux), (e1, r), (e2, g), (normale, b) ; drapeaux =", + "//! paroi (1) + 2 × arêtes réelles (bit k : l'arête opposée au sommet k). Le fond plat, en", + "//! z = -épaisseur, est laissé au shader. `POLYS` tient les silhouettes (bord extérieur du", + "//! serti) puis les contours du cristal (son bord intérieur). `BOXES` range les triangles", + "//! par voisinage : deux float4 par boîte, (coin bas, premier triangle), (coin haut, nombre).", + "", + "/// Un des deux modèles : ses triangles et sa silhouette dans les tables, sa boîte de dessin.", + "pub struct PrismModel {", + " pub tri_start: usize,", + " pub tri_count: usize,", + " pub sil_start: usize,", + " pub sil_count: usize,", + " pub out_start: usize,", + " pub out_count: usize,", + " pub box_start: usize,", + " pub box_count: usize,", + " /// Largeur de la boîte, hauteur 1 (le sprite), en unités du modèle.", + " pub size: [f32; 2],", + " /// Hotspot, fraction de la boîte.", + " pub hotspot: [f32; 2],", + " /// Haut de la silhouette, fraction de la hauteur.", + " pub top: f32,", + " /// Sommet du cristal au-dessus de z = 0, et épaisseur sous z = 0.", + " pub height: f32,", + " pub thick: f32,", + "}", + "", + "/// Arrondi des arêtes du serti, en unités du modèle (`PRISM_BEVEL` des shaders).", + f"pub const BEVEL: f32 = {fl(bevel(shapes))};", + "", + "/// Flèche puis main, dans l'ordre des identifiants du shader.", + "pub const MODELS: [PrismModel; 2] = [", + ] + for i, sh in enumerate(shapes): + lines += [ + " PrismModel {", + f" tri_start: {tri_start[i]},", + f" tri_count: {len(sh['tris'])},", + f" sil_start: {sil_start[i]},", + f" sil_count: {len(sh['sil'])},", + f" out_start: {out_start[i]},", + f" out_count: {len(sh['out'])},", + f" box_start: {box_start[i]},", + f" box_count: {len(sh['leaves'])},", + f" size: [{fl(sh['size'][0])}, {fl(sh['size'][1])}],", + f" hotspot: [{fl(sh['hotspot'][0])}, {fl(sh['hotspot'][1])}],", + f" top: {fl(sh['top'])},", + f" height: {fl(sh['height'])},", + f" thick: {fl(sh['thick'])},", + " },", + ] + lines += ["];", "", f"pub const TRIS: [[f32; 4]; {len(rows)}] = ["] + lines += [f" [{', '.join(fl(v) for v in r)}]," for r in rows] + lines += ["];", "", f"pub const POLYS: [[f32; 2]; {len(poly)}] = ["] + lines += [f" [{fl(x)}, {fl(y)}]," for x, y in poly] + lines += ["];", "", f"pub const BOXES: [[f32; 4]; {len(box_rows)}] = ["] + lines += [f" [{', '.join(fl(v) for v in r)}]," for r in box_rows] + lines += ["];", ""] + (SRC / "prism_mesh.rs").write_text("\n".join(lines), encoding="utf-8", newline="\n") + + +def table_lines(shapes, lang): + rows = records(shapes) + poly, sil_start, out_start = polygons(shapes) + box_rows, box_start = boxes(shapes) + bcounts = [len(sh["leaves"]) for sh in shapes] + counts = [len(sh["tris"]) for sh in shapes] + scounts = [len(sh["sil"]) for sh in shapes] + ocounts = [len(sh["out"]) for sh in shapes] + if lang == "hlsl": + v4 = lambda r: f"float4({', '.join(fl(v) for v in r)})" + v2 = lambda p: f"float2({fl(p[0])}, {fl(p[1])})" + head4 = f"static const float4 PRISM_TRIS[{len(rows)}] = {{" + headb = f"static const float4 PRISM_BOXES[{len(box_rows)}] = {{" + head2 = f"static const float2 PRISM_POLY[{len(poly)}] = {{" + tail = "};" + ints = lambda name, vals: f"static const int {name}[2] = {{ {vals[0]}, {vals[1]} }};" + scalar = f"static const float PRISM_BEVEL = {fl(bevel(shapes))};" + elif lang == "metal": + v4 = lambda r: f"float4({', '.join(fl(v) for v in r)})" + v2 = lambda p: f"float2({fl(p[0])}, {fl(p[1])})" + head4 = f"constant float4 PRISM_TRIS[{len(rows)}] = {{" + headb = f"constant float4 PRISM_BOXES[{len(box_rows)}] = {{" + head2 = f"constant float2 PRISM_POLY[{len(poly)}] = {{" + tail = "};" + ints = lambda name, vals: f"constant int {name}[2] = {{ {vals[0]}, {vals[1]} }};" + scalar = f"constant float PRISM_BEVEL = {fl(bevel(shapes))};" + else: + ints = lambda name, vals: f"var {name} = array({vals[0]}, {vals[1]});" + scalar = f"const PRISM_BEVEL: f32 = {fl(bevel(shapes))};" + out = [ints("PRISM_TRI_START", [0, counts[0]]), ints("PRISM_TRI_COUNT", counts), + ints("PRISM_SIL_START", sil_start), ints("PRISM_SIL_COUNT", scounts), + ints("PRISM_OUT_START", out_start), ints("PRISM_OUT_COUNT", ocounts), + ints("PRISM_BOX_START", box_start), ints("PRISM_BOX_COUNT", bcounts)] + if lang == "wgsl": + # The triangles, polygons and boxes themselves go in a uniform buffer that compositor_linux.rs + # fills from prism_mesh.rs (a polygon point per vec4, the stride of a uniform array). As + # a `const` or `var` table, lavapipe copies them into every invocation of every + # layer: a frame without any cursor rendered 3.4 times slower. + return out + ["struct PrismMesh {", f" tris: array, {len(rows)}>,", + f" poly: array, {len(poly)}>,", + f" boxes: array, {len(box_rows)}>,", "}", scalar] + out.append(head4) + out += [f" {v4(r)}," for r in rows] + out[-1] = out[-1].rstrip(",") + out += [tail, headb] + out += [f" {v4(r)}," for r in box_rows] + out[-1] = out[-1].rstrip(",") + out += [tail, head2] + out += [f" {v2(p)}," for p in poly] + out[-1] = out[-1].rstrip(",") + out += [tail, scalar] + return out + + +BEGIN = "prism mesh: generated by design/cursors/prism-glow/model/export_compositor.py" +END = "end of the prism mesh" + + +def write_shaders(shapes): + for name, lang in (("shaders.hlsl", "hlsl"), ("shaders.metal", "metal"), ("vk_shaders/layer.wgsl", "wgsl")): + path = SRC / name + text = path.read_text(encoding="utf-8") + a, b = text.find(BEGIN), text.find(END) + if a < 0 or b < 0: + print(f"{name}: no markers yet, tables not written") + continue + start = text.index("\n", a) + 1 + stop = text.rindex("\n", 0, b) + 1 + text = text[:start] + "\n".join(table_lines(shapes, lang)) + "\n" + text[stop:] + path.write_text(text, encoding="utf-8", newline="\n") + print(f"{name}: tables written") + + +if __name__ == "__main__": + main() diff --git a/design/cursors/prism-glow/model/prism-glow.blend b/design/cursors/prism-glow/model/prism-glow.blend new file mode 100644 index 000000000..8edc4553b Binary files /dev/null and b/design/cursors/prism-glow/model/prism-glow.blend differ diff --git a/design/cursors/prism-glow/model/prism-glow.glb b/design/cursors/prism-glow/model/prism-glow.glb new file mode 100644 index 000000000..2aaa2d145 Binary files /dev/null and b/design/cursors/prism-glow/model/prism-glow.glb differ diff --git a/design/cursors/prism-glow/model/prism.json b/design/cursors/prism-glow/model/prism.json new file mode 100644 index 000000000..5aae7a713 --- /dev/null +++ b/design/cursors/prism-glow/model/prism.json @@ -0,0 +1,985 @@ +{ + "arrow": { + "hotspot": [ + 208, + 42 + ], + "verts": { + "A": [ + 237, + 100, + 0.0 + ], + "B": [ + 443, + 290, + 0.0 + ], + "F": [ + 712, + 549, + 0.0 + ], + "K": [ + 543, + 563, + 0.0 + ], + "J": [ + 612, + 744, + 0.0 + ], + "I": [ + 492, + 807, + 0.0 + ], + "H": [ + 394, + 597, + 0.0 + ], + "G": [ + 241, + 732, + 0.0 + ], + "D": [ + 241, + 491, + 0.0 + ], + "C": [ + 366, + 374, + 108.0 + ], + "E": [ + 458, + 539, + 76.0 + ], + "X": [ + 515, + 449, + 66.0 + ] + }, + "outline": [ + "A", + "B", + "F", + "K", + "J", + "I", + "H", + "G", + "D" + ], + "facets": [ + [ + "A", + "C", + "D" + ], + [ + "A", + "B", + "C" + ], + [ + "B", + "X", + "C" + ], + [ + "B", + "F", + "X" + ], + [ + "C", + "X", + "E" + ], + [ + "X", + "F", + "E" + ], + [ + "E", + "F", + "K" + ], + [ + "C", + "E", + "H" + ], + [ + "C", + "H", + "G" + ], + [ + "C", + "G", + "D" + ], + [ + "E", + "I", + "H" + ], + [ + "E", + "J", + "I" + ], + [ + "E", + "K", + "J" + ] + ], + "colors": [ + "#01f8ee", + "#cadffc", + "#a643fc", + "#c348fc", + "#033df4", + "#002cc0", + "#124efb", + "#00208a", + "#001b80", + "#00b7fb", + "#0022b7", + "#008afc", + "#00fbfb" + ], + "silhouette": [ + [ + 214, + 40 + ], + [ + 207, + 43 + ], + [ + 204, + 49 + ], + [ + 204, + 793 + ], + [ + 207, + 796 + ], + [ + 212, + 796 + ], + [ + 229, + 784 + ], + [ + 381, + 653 + ], + [ + 470, + 846 + ], + [ + 475, + 853 + ], + [ + 481, + 854 + ], + [ + 642, + 770 + ], + [ + 654, + 762 + ], + [ + 654, + 753 + ], + [ + 588, + 594 + ], + [ + 628, + 588 + ], + [ + 768, + 576 + ], + [ + 773, + 572 + ], + [ + 773, + 568 + ], + [ + 768, + 560 + ], + [ + 675, + 464 + ], + [ + 604, + 393 + ], + [ + 478, + 272 + ], + [ + 284, + 96 + ], + [ + 221, + 42 + ] + ] + }, + "hand": { + "hotspot": [ + 1220, + 50 + ], + "verts": { + "I1": [ + 1180, + 122, + 0.0 + ], + "I2": [ + 1221, + 85, + 0.0 + ], + "I3": [ + 1261, + 90, + 0.0 + ], + "I4": [ + 1290, + 132, + 0.0 + ], + "IT1": [ + 1224, + 118, + 46.0 + ], + "IT2": [ + 1274, + 135, + 35.0 + ], + "IT3": [ + 1191, + 262, + 54.0 + ], + "IL": [ + 1179, + 265, + 0.0 + ], + "P1": [ + 1292, + 427, + 0.0 + ], + "W1": [ + 1321, + 456, + 0.0 + ], + "M1": [ + 1323, + 334, + 0.0 + ], + "M2": [ + 1356, + 302, + 0.0 + ], + "M3": [ + 1395, + 309, + 0.0 + ], + "M4": [ + 1415, + 340, + 0.0 + ], + "MT1": [ + 1354, + 334, + 46.0 + ], + "MT2": [ + 1390, + 344, + 35.0 + ], + "P2": [ + 1339, + 479, + 59.0 + ], + "MB1": [ + 1386, + 481, + 54.0 + ], + "MB2": [ + 1415, + 462, + 0.0 + ], + "RL": [ + 1441, + 467, + 0.0 + ], + "R1": [ + 1443, + 368, + 0.0 + ], + "R2": [ + 1476, + 338, + 0.0 + ], + "R3": [ + 1515, + 345, + 0.0 + ], + "R4": [ + 1536, + 377, + 0.0 + ], + "RT1": [ + 1480, + 367, + 43.0 + ], + "RT2": [ + 1513, + 381, + 34.0 + ], + "RB": [ + 1448, + 475, + 57.0 + ], + "RB1": [ + 1476, + 498, + 54.0 + ], + "RB2": [ + 1535, + 489, + 0.0 + ], + "PK": [ + 1567, + 502, + 0.0 + ], + "K1": [ + 1568, + 403, + 0.0 + ], + "K2": [ + 1600, + 375, + 0.0 + ], + "K3": [ + 1637, + 382, + 0.0 + ], + "K4": [ + 1658, + 412, + 0.0 + ], + "KT1": [ + 1601, + 404, + 40.0 + ], + "KT2": [ + 1640, + 420, + 30.0 + ], + "K5": [ + 1658, + 608, + 0.0 + ], + "PR": [ + 1651, + 622, + 0.0 + ], + "BR": [ + 1522, + 818, + 0.0 + ], + "BL": [ + 1208, + 820, + 0.0 + ], + "HC": [ + 1406, + 640, + 76.0 + ], + "Q": [ + 1178, + 572, + 49.0 + ], + "TB": [ + 1128, + 682, + 0.0 + ], + "TT2": [ + 1015, + 490, + 0.0 + ], + "TL": [ + 1013, + 427, + 0.0 + ], + "TT": [ + 1069, + 402, + 0.0 + ], + "TR": [ + 1133, + 449, + 0.0 + ], + "TT1": [ + 1065, + 435, + 43.0 + ], + "TH": [ + 1144, + 541, + 40.0 + ] + }, + "outline": [ + "I1", + "I2", + "I3", + "I4", + "P1", + "W1", + "M1", + "M2", + "M3", + "M4", + "MB2", + "RL", + "R1", + "R2", + "R3", + "R4", + "RB2", + "PK", + "K1", + "K2", + "K3", + "K4", + "K5", + "PR", + "BR", + "BL", + "TB", + "TT2", + "TL", + "TT", + "TR", + "TH", + "Q", + "IL" + ], + "facets": [ + [ + "IT1", + "IT2", + "IT3" + ], + [ + "IT1", + "I2", + "I3", + "IT2" + ], + [ + "I1", + "I2", + "IT1" + ], + [ + "I1", + "IT1", + "IT3", + "IL" + ], + [ + "I3", + "I4", + "IT2" + ], + [ + "IT2", + "I4", + "P1", + "IT3" + ], + [ + "IL", + "IT3", + "P1", + "Q" + ], + [ + "Q", + "P1", + "W1" + ], + [ + "Q", + "W1", + "P2" + ], + [ + "Q", + "P2", + "HC" + ], + [ + "MT1", + "MT2", + "P2" + ], + [ + "MT1", + "M2", + "M3", + "MT2" + ], + [ + "M1", + "M2", + "MT1" + ], + [ + "M1", + "MT1", + "P2", + "W1" + ], + [ + "M3", + "M4", + "MT2" + ], + [ + "MT2", + "M4", + "MB2", + "MB1", + "P2" + ], + [ + "RT1", + "RT2", + "RB" + ], + [ + "RT1", + "R2", + "R3", + "RT2" + ], + [ + "R1", + "R2", + "RT1" + ], + [ + "R1", + "RT1", + "RB", + "RL" + ], + [ + "R3", + "R4", + "RT2" + ], + [ + "RT2", + "R4", + "RB2", + "RB1", + "RB" + ], + [ + "KT1", + "KT2", + "PK" + ], + [ + "KT1", + "K2", + "K3", + "KT2" + ], + [ + "K1", + "K2", + "KT1" + ], + [ + "K1", + "KT1", + "PK" + ], + [ + "K3", + "K4", + "KT2" + ], + [ + "KT2", + "K4", + "K5", + "PR", + "PK" + ], + [ + "P2", + "MB1", + "MB2", + "RL", + "RB", + "RB1", + "RB2", + "PK", + "HC" + ], + [ + "HC", + "PK", + "BR" + ], + [ + "PK", + "PR", + "BR" + ], + [ + "HC", + "BR", + "BL" + ], + [ + "Q", + "HC", + "BL" + ], + [ + "TT1", + "TH", + "TT2" + ], + [ + "TL", + "TT", + "TT1", + "TT2" + ], + [ + "TT", + "TR", + "TH", + "TT1" + ], + [ + "TT2", + "TH", + "TB" + ], + [ + "TH", + "Q", + "TB" + ], + [ + "Q", + "BL", + "TB" + ] + ], + "colors": [ + "#b2fcfc", + "#01fbfc", + "#00faf3", + "#01faf4", + "#00b7f4", + "#00a5eb", + "#bf54fc", + "#662ffa", + "#7237fa", + "#7651fc", + "#88fbfc", + "#00e8ee", + "#00f5f4", + "#76fafb", + "#00beed", + "#00b1f0", + "#8cfcfc", + "#00f3f3", + "#00e1ed", + "#01edf3", + "#00cef6", + "#00b7f7", + "#82fbfc", + "#01f1f5", + "#00e0ee", + "#01f1f3", + "#00c0f2", + "#6f31fb", + "#0129a6", + "#1542fb", + "#0021af", + "#002eba", + "#01f7fc", + "#bffcfc", + "#01fafc", + "#00e3f0", + "#00ddee", + "#00beef", + "#008cf5" + ], + "silhouette": [ + [ + 1209, + 50 + ], + [ + 1169, + 85 + ], + [ + 1145, + 109 + ], + [ + 1143, + 113 + ], + [ + 1142, + 404 + ], + [ + 1137, + 404 + ], + [ + 1073, + 360 + ], + [ + 1066, + 361 + ], + [ + 988, + 401 + ], + [ + 978, + 409 + ], + [ + 976, + 477 + ], + [ + 977, + 500 + ], + [ + 1060, + 640 + ], + [ + 1179, + 846 + ], + [ + 1186, + 852 + ], + [ + 1197, + 856 + ], + [ + 1530, + 856 + ], + [ + 1543, + 852 + ], + [ + 1556, + 839 + ], + [ + 1695, + 623 + ], + [ + 1695, + 401 + ], + [ + 1651, + 348 + ], + [ + 1647, + 346 + ], + [ + 1589, + 338 + ], + [ + 1584, + 340 + ], + [ + 1566, + 357 + ], + [ + 1538, + 317 + ], + [ + 1531, + 311 + ], + [ + 1469, + 301 + ], + [ + 1464, + 303 + ], + [ + 1445, + 320 + ], + [ + 1416, + 276 + ], + [ + 1412, + 273 + ], + [ + 1358, + 264 + ], + [ + 1348, + 264 + ], + [ + 1330, + 280 + ], + [ + 1328, + 256 + ], + [ + 1329, + 200 + ], + [ + 1327, + 121 + ], + [ + 1283, + 61 + ], + [ + 1278, + 57 + ], + [ + 1225, + 50 + ] + ] + } +} diff --git a/design/cursors/prism-glow/model/prism_geo.py b/design/cursors/prism-glow/model/prism_geo.py new file mode 100644 index 000000000..28a115496 --- /dev/null +++ b/design/cursors/prism-glow/model/prism_geo.py @@ -0,0 +1,157 @@ +"""Prism Glow cursor as a faceted mesh: the facets traced from ../source.png, and their heights. + +Coordinates are source-PNG pixels (x right, y down); `z` is the height above the navy rim's top in +the same pixels (unlisted vertices sit on the rim). `outline` is the crystal's edge against the rim, +`facets` are the flat faces of the 2D art, each an ordered polygon. + + python prism_geo.py # writes prism.json: vertices, facets, their colours in the art, silhouettes + +Then build the Blender scene and the glTF from it with build_blend.py. +""" + +import json +from pathlib import Path + +import cv2 +import numpy as np + +HERE = Path(__file__).resolve().parent + +ARROW = { + "hotspot": (208, 42), + "verts": { + "A": (237, 100), "B": (443, 290), "F": (712, 549), "K": (543, 563), "J": (612, 744), + "I": (492, 807), "H": (394, 597), "G": (241, 732), "D": (241, 491), + "C": (366, 374), "E": (458, 539), "X": (515, 449), + }, + "outline": ["A", "B", "F", "K", "J", "I", "H", "G", "D"], + "z": {"C": 108, "E": 76, "X": 66}, + "facets": [ + ["A", "C", "D"], ["A", "B", "C"], ["B", "X", "C"], ["B", "F", "X"], + ["C", "X", "E"], ["X", "F", "E"], ["E", "F", "K"], + # the notch H sits on the E-G line: one concept facet, folded along C-H + ["C", "E", "H"], ["C", "H", "G"], ["C", "G", "D"], + ["E", "I", "H"], ["E", "J", "I"], ["E", "K", "J"], + ], +} + +HAND = { + "hotspot": (1220, 50), + "verts": { + # index finger + "I1": (1180, 122), "I2": (1221, 85), "I3": (1261, 90), "I4": (1290, 132), + "IT1": (1224, 118), "IT2": (1274, 135), "IT3": (1191, 262), "IL": (1179, 265), + "P1": (1292, 427), + # middle finger + "W1": (1321, 456), "M1": (1323, 334), "M2": (1356, 302), "M3": (1395, 309), + "M4": (1415, 340), "MT1": (1354, 334), "MT2": (1390, 344), "P2": (1339, 479), + "MB1": (1386, 481), "MB2": (1415, 462), + # ring finger + "RL": (1441, 467), "R1": (1443, 368), "R2": (1476, 338), "R3": (1515, 345), + "R4": (1536, 377), "RT1": (1480, 367), "RT2": (1513, 381), "RB": (1448, 475), + "RB1": (1476, 498), "RB2": (1535, 489), + # pinky + "PK": (1567, 502), "K1": (1568, 403), "K2": (1600, 375), "K3": (1637, 382), + "K4": (1658, 412), "KT1": (1601, 404), "KT2": (1640, 420), "K5": (1658, 608), + "PR": (1651, 622), + # palm + "BR": (1522, 818), "BL": (1208, 820), "HC": (1406, 640), "Q": (1178, 572), + # thumb + "TB": (1128, 682), "TT2": (1015, 490), "TL": (1013, 427), "TT": (1069, 402), + "TR": (1133, 449), "TT1": (1065, 435), "TH": (1144, 541), + }, + # Each fingertip is cut like a gem: a table (the white facet), crown facets down to the rim. + # The thumb web (TH, Q) rises above the rim so the thumb's lower facets are not flat. + "z": { + "IT1": 46, "IT2": 35, "IT3": 54, + "MT1": 46, "MT2": 35, "P2": 59, "MB1": 54, + "RT1": 43, "RT2": 34, "RB": 57, "RB1": 54, + "KT1": 40, "KT2": 30, + "TT1": 43, "TH": 40, "Q": 49, + "HC": 76, + }, + "outline": [ + "I1", "I2", "I3", "I4", "P1", "W1", "M1", "M2", "M3", "M4", "MB2", "RL", "R1", "R2", + "R3", "R4", "RB2", "PK", "K1", "K2", "K3", "K4", "K5", "PR", "BR", "BL", "TB", "TT2", + "TL", "TT", "TR", "TH", "Q", "IL", + ], + "facets": [ + # index: table, crown, sides + ["IT1", "IT2", "IT3"], ["IT1", "I2", "I3", "IT2"], ["I1", "I2", "IT1"], + ["I1", "IT1", "IT3", "IL"], ["I3", "I4", "IT2"], ["IT2", "I4", "P1", "IT3"], + ["IL", "IT3", "P1", "Q"], + # palm under the index (W1 sits on the P1-P2 line) + ["Q", "P1", "W1"], ["Q", "W1", "P2"], ["Q", "P2", "HC"], + # middle finger + ["MT1", "MT2", "P2"], ["MT1", "M2", "M3", "MT2"], ["M1", "M2", "MT1"], + ["M1", "MT1", "P2", "W1"], ["M3", "M4", "MT2"], ["MT2", "M4", "MB2", "MB1", "P2"], + # ring finger + ["RT1", "RT2", "RB"], ["RT1", "R2", "R3", "RT2"], ["R1", "R2", "RT1"], + ["R1", "RT1", "RB", "RL"], ["R3", "R4", "RT2"], ["RT2", "R4", "RB2", "RB1", "RB"], + # pinky + ["KT1", "KT2", "PK"], ["KT1", "K2", "K3", "KT2"], ["K1", "K2", "KT1"], + ["K1", "KT1", "PK"], ["K3", "K4", "KT2"], ["KT2", "K4", "K5", "PR", "PK"], + # palm + ["P2", "MB1", "MB2", "RL", "RB", "RB1", "RB2", "PK", "HC"], + ["HC", "PK", "BR"], ["PK", "PR", "BR"], ["HC", "BR", "BL"], ["Q", "HC", "BL"], + # thumb + ["TT1", "TH", "TT2"], ["TL", "TT", "TT1", "TT2"], ["TT", "TR", "TH", "TT1"], + ["TT2", "TH", "TB"], ["TH", "Q", "TB"], ["Q", "BL", "TB"], + ], +} + + +def area(poly): + p = np.asarray(poly, float) + return 0.5 * np.sum(p[:, 0] * np.roll(p[:, 1], -1) - np.roll(p[:, 0], -1) * p[:, 1]) + + +def check(shape, name): + """The facets tile the outline exactly: no gap, no overlap, no stray vertex.""" + v = shape["verts"] + outline = abs(area([v[k] for k in shape["outline"]])) + facets = sum(abs(area([v[k] for k in f])) for f in shape["facets"]) + unused = set(v) - {k for f in shape["facets"] for k in f} + assert abs(facets / outline - 1) < 1e-9 and not unused, (name, facets / outline, unused) + + +def facet_colors(shape, img): + """Median colour of each facet in the art, away from its edges.""" + v = shape["verts"] + out = [] + for f in shape["facets"]: + mask = np.zeros(img.shape[:2], np.uint8) + cv2.fillPoly(mask, [np.array([v[k] for k in f], np.int32)], 255) + inner = cv2.erode(mask, np.ones((7, 7), np.uint8)) + px = img[(inner if inner.any() else mask) > 0][:, :3] + b, g, r = np.median(px, axis=0) + out.append("#%02x%02x%02x" % (int(r), int(g), int(b))) + return out + + +def silhouette(img, x0, x1): + """Outer edge of the navy rim, from the art's alpha.""" + a = (img[:, :, 3] > 127).astype(np.uint8) + a[:, :x0] = 0 + a[:, x1:] = 0 + contours, _ = cv2.findContours(a, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) + c = max(contours, key=cv2.contourArea) + return cv2.approxPolyDP(c, 1.2, True)[:, 0, :].tolist() + + +if __name__ == "__main__": + img = cv2.imread(str(HERE.parent / "source.png"), cv2.IMREAD_UNCHANGED) + out = {} + for shape, name, (x0, x1) in ((ARROW, "arrow", (0, 887)), (HAND, "hand", (887, 1774))): + check(shape, name) + z = shape["z"] + out[name] = { + "hotspot": shape["hotspot"], + "verts": {k: [*p, float(z.get(k, 0))] for k, p in shape["verts"].items()}, + "outline": shape["outline"], + "facets": shape["facets"], + "colors": facet_colors(shape, img), + "silhouette": silhouette(img, x0, x1), + } + (HERE / "prism.json").write_text(json.dumps(out, indent=1) + "\n", newline="\n") + print({k: f"{len(v['facets'])} facets" for k, v in out.items()}) diff --git a/design/cursors/requirements.md b/design/cursors/requirements.md index 3274014c7..1cf03d987 100644 --- a/design/cursors/requirements.md +++ b/design/cursors/requirements.md @@ -21,16 +21,17 @@ their 3D models. - Every replacement theme and supported cursor state must work with the existing 3D cursor option. -- Every replacement arrow and hand is a real volume, sculpted in the +- Every replacement arrow and hand is a real volume, modelled in the compositor's shaders (`crates/compositor/src/sculpt.rs`): lighting, normals, self shadows and contact shadows respond to cursor tilt and rotation. A PNG face or a height map is not a model. - A rounded extrusion of the PNG silhouette remains the fallback for cursor - states that do not have a dedicated theme model. The five replacement packs - add style-specific volume: a glove with separate fingers, cut crystal facets, - puffy rubber, beveled voxels, and ceramic accents. -- Model the arrow and hand separately for every theme. Do not apply one - contour, material, or roundness rule to all designs. + states that do not have a dedicated theme model. Each pack adds + style-specific volume: gloves with separate fingers, beveled voxels, flat + cut-paper sheets, cushions set in the rim of their drawn outline, and a + faceted glass crystal that refracts the picture under it. +- Model the arrow and hand separately for every sculpted theme. Do not apply + one contour, material, or roundness rule to all designs. - Preserve the cursor hotspot through hover, tilt, yaw, click, and size changes. - Reuse the compositor's 3D lighting and contact shadows. The surface must change with the compositor camera; a static image of a 3D render is not a @@ -40,11 +41,11 @@ their 3D models. | Theme | Arrow | Hand | | --- | --- | --- | -| Studio Ink | Matte black sculpted body; ivory mark becomes a piping set in from the edge. | Ivory glove with a convex palm and separate fingers; remove the dark outline. | -| Prism Glow | Translucent faceted crystal; no navy ink contour. | Crystal palm and distinct faceted fingers; no navy ink contour. | -| Pop Coral | Rounded coral rubber; omit the yellow offset and navy rim. | Plump yellow rubber glove; no navy outline, accents become physical details. | -| Pixel Candy | Keep the stepped voxel form; purple is side/back blocks, not a flat outline. | Stepped voxel fingers and palm; preserve purple as physical side/back blocks. | -| Star Sprout | Soft mint ceramic; no heavy blue outline, raised star and leaves. | Rounded ivory glove with no blue outline; separate mint cuff, star, and leaves. | +| Studio Ink | Black rim, raised ivory band, recessed satin black field. | Ivory glove cushion; the black outline stays as a physical rim, with black ridges between the fingers. | +| Prism Glow | A glass crystal cut along the drawing's facets, in a thin navy rim; the picture under it shows through, refracted with a slight dispersion, privacy blurs included; the facets glow faintly in their drawn colours. Modelled from the 2D drawing alone, keeping its polygonal simplicity. | The same, a crystal hand. | +| Pop Coral | Cut paper: a coral sheet on a navy sheet, flat tops, matte; the yellow offset becomes a flat sheet behind; the click dashes become flat pieces. | A yellow sheet on a navy sheet, flat tops, matte; the coral offset becomes a flat sheet behind; the coral click dashes become flat pieces. | +| Pixel Candy | Simplified pixel art that reads at 20 px as at full size; the 3D model is one cube per pixel of the same grid as the sprite. | A pixel hand from the same kind of grid: index, three knuckles, thumb. | +| Star Sprout | Puffy mint cushion; the navy outline stays as a physical rim (a navy tray and a rounded bead) around it, the star and its leaves in front, each in its own navy rim. | Puffy ivory glove in the same navy rim, with navy ridges between the fingers; separate mint cuff framed in navy; the star and leaves in front, as in `star-sprout/3d-reference.png`. | The 2D PNG may keep a drawn contour where it helps readability. Its contour must not be copied blindly onto the 3D material: decide per model whether it @@ -53,7 +54,11 @@ becomes an inlay, side material, separate piece, or disappears. ## Review references - `contact-sheet.png`: 2D PNG themes. -- `3d-concept.png`: visual direction only, not rendered by the app. +- `3d-concept.png`: the 3D reference sheet the eight sculpted models follow + (Prism Glow's crystal follows its 2D drawing instead), not rendered by the + app. +- `star-sprout/3d-reference.png`: a closer view of the Star Sprout hand; also + not rendered by the app. - `3d-direction.md`: per-theme modeling notes. - `README.md`: source PNG and preparation workflow. diff --git a/design/cursors/star-sprout/3d-reference.png b/design/cursors/star-sprout/3d-reference.png new file mode 100644 index 000000000..760ed6b62 Binary files /dev/null and b/design/cursors/star-sprout/3d-reference.png differ diff --git a/docs/3d-effects-v2.md b/docs/3d-effects-v2.md index 978cd1a80..119c5b3bb 100644 --- a/docs/3d-effects-v2.md +++ b/docs/3d-effects-v2.md @@ -233,12 +233,11 @@ inconnues sont ignorées). ### B.2 Le réglage **Un seul interrupteur**, `cursor.model3d` (« 3D cursor », **éteint par défaut**) : il passe -**chaque état** du curseur en 3D. La flèche et la main des cinq thèmes d'origine deviennent leur -modèle sculpté (B.3 bis) ; les autres états, et tous ceux du thème par défaut (les seize de -`DEFAULT_CURSOR_SPRITES` : flèche, I, main, croix, mains ouverte et fermée, redimensionnements, -déplacement, interdit, attente…), leur sprite extrudé. Curseur masqué, l'interrupteur est grisé et -son info-bulle dit pourquoi. Éteint, la frame est celle d'avant **à l'octet** (vérifié à plat et -incliné contre le commit de base). +**chaque état** du curseur en 3D. La flèche et la main des cinq thèmes d'origine deviennent +leur modèle (B.3 bis) ; les autres états, et tous ceux du thème par défaut (les seize de `DEFAULT_CURSOR_SPRITES` : flèche, I, main, croix, mains +ouverte et fermée, redimensionnements, déplacement, interdit, attente…), leur sprite extrudé. +Curseur masqué, l'interrupteur est grisé et son info-bulle dit pourquoi. Éteint, la frame est +celle d'avant **à l'octet** (vérifié à plat et incliné contre le commit de base). Tuyauterie : `CursorVisualSettings.model3d`, clé legacy `cursorModel3d`, préréglages (absent → éteint), `SceneCursor.model3d` (`serde(default)`), `LiveParams.cursor_model3d`, paramètre live @@ -287,24 +286,44 @@ du bord (0,48 au pire), sous 1 texel au-delà (le flou arrondit les crêtes). ### B.3 bis Les curseurs sculptés -La flèche et la main des cinq thèmes d'origine sont **modélisées à la main**, en fonctions de -distance signée écrites dans les trois shaders (`sculpt_proto`, `sculpt_material`) : capsules et -unions lissées pour les gants, extrusion arrondie et bombée pour les flèches, voxels biseautés -pour Pixel Candy, polyèdres taillés pour le cristal de Prism Glow. Le PNG du thème reste l'art en -2D. En 3D, la scène nomme le modèle ; `sculpt.rs` en tient la boîte, qui pose le hotspot (pointe -de la flèche, bout de l'index), règle la garde au sol et borne la boîte de dessin. Emplacement du -cbuffer : `trail_a` = [modèle, épaisseur sous z = 0, hauteur au-dessus, 0] (`cursor_model_cb`). +La flèche et la main des thèmes d'origine sont **modélisées à la main**, en fonctions de +distance signée écrites dans les trois shaders (`sculpt_proto`, `sculpt_material`) : voxels +pour Pixel Candy ; pour Studio Ink et Star Sprout des pièces qui gardent le trait de +leur dessin : un plateau et un jonc de la couleur du trait, un coussin de couleur dedans +(`s_rimmed`) ; pour Pop Coral les mêmes formes en papier découpé, des feuilles au dessus plat +(`s_paper`). Prism Glow est à part : un **cristal en maillage**, ses facettes tracées sur son +seul dessin 2D (`design/cursors/prism-glow/model/`), serti dans sa silhouette extrudée, marine. +Le shader le lance de rayons, triangles rangés par boîtes englobantes (un rayon saute les +boîtes qu'il rate) : réfraction par canal (une légère dispersion), réflexions totales internes, +et sortie par son fond plat sur l'image sous le curseur ; chaque facette luit un peu de sa +couleur du dessin, ses plis d'un liseré clair. Cette image est une copie de la frame composée +prise juste avant le curseur : le métrage, puis ses flous de confidentialité, puis le curseur, +net par-dessus. Seuls des pixels déjà floutés passent donc à travers le verre. +`export_compositor.py` écrit le maillage dans `prism_mesh.rs` et dans les trois shaders. Le PNG +du thème reste l'art en 2D. En 3D, la scène nomme le modèle ; `sculpt.rs` en tient la boîte, qui +pose le hotspot (pointe de la flèche, bout de l'index), règle la garde au sol et borne la boîte +de dessin. Emplacement du cbuffer : `trail_a` = [modèle, épaisseur sous z = 0, hauteur +au-dessus, 0] (`cursor_model_cb`). +Pixel Candy est un pixel art dessiné une fois, en grille, dans +`scripts/generate-pixel-candy-voxels.mjs` : le script en tire ses PNG 2D et les tables des trois +shaders, un cube par pixel plein, tous de même hauteur, colorés comme leur pixel (contour prune, +rose, reflet rose pâle, ombre rose foncé). - **Éclairage**, sprites extrudés compris : une lampe proche en haut à gauche, qui met un dégradé et un reflet même sur une face plane ; une lumière d'appoint faible ; le côté ombré teinté par la matière ; occlusion ambiante, ombre propre douce vers la lampe, studio dans les reflets, liseré de Fresnel ; tone map Khronos PBR Neutral. -- **Ombre sur l'écran** : celle de B.3. Le cristal et les voxels donnent de mauvaises distances - loin de leur surface (bornes de plans, champ de grille) : l'ombre prend la forme lisse de Pop - Coral pour le cristal et le contour extrudé pour les voxels. +- **Ombre sur l'écran** : celle de B.3. Les voxels donnent de mauvaises distances loin de leur + surface (champ de grille) : l'ombre de Pixel Candy vient de sa forme extrudée. +- **Antialiasing** : la silhouette l'est par la marche (la distance minimale frôlée, en pixels, + donne la couverture). Les bords intérieurs (deux matières, une arête, un arrondi serré, un + joint de voxels) sont suréchantillonnés : quatre sondes à un demi-pixel du point touché, dans + son plan tangent, et si la matière change ou que la surface s'en écarte, trois rayons de plus + dans le pixel, ombrés avec l'occlusion et les ombres du premier (`STAGE_EDGE`, `STAGE_SHADE`). + Sur un grand curseur, ~+50 % du coût du curseur (rendu logiciel), rien ailleurs. - **Coût de compilation** : FXC recopie chaque appel. La marche, les normales, l'occlusion et les - ombres forment donc une seule boucle à étapes avec un seul appel au modèle, et l'étoile de Star - Sprout un seul appel pour les deux formes. + ombres forment donc une seule boucle à étapes avec un seul appel au modèle, et les thèmes + cerclés un seul appel pour leurs deux formes. ### B.4 La caméra et l'ancrage @@ -490,7 +509,7 @@ C'est le cadre qui s'adapte au métrage — jamais l'inverse : un téléphone au est un téléphone COUCHÉ, avec son œil de caméra sur le bord qui est devenu son haut. **Roundness sous un cadre.** Le slider parcourt 0 → le plafond du cadre choisi (C.5) et se lit -en **%** de cette course, avec l'infobulle « La course s'adapte au cadre » (`roundnessFrameHelp`, +en **%** de cette course, avec l'infobulle « La course dépend du cadre » (`roundnessFrameHelp`, 15 langues). La valeur stockée reste en px ; le natif en relit la position (`roundnessFrac` × petit côté de la sortie ÷ `ROUNDNESS_SLIDER_MAX_PX` = 64, miroir de `paramUnits.ts`). Sans cadre, le slider reste en px et le rendu est celui d'avant, à l'octet. diff --git a/docs/assets/demo.gif b/docs/assets/demo.gif index 9c2a0270e..23ab2fa65 100644 Binary files a/docs/assets/demo.gif and b/docs/assets/demo.gif differ diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index fc8ba4085..f612c92eb 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -1211,6 +1211,96 @@ describe("a full-camera region needs a camera", () => { }); }); +describe("full-camera regions and layout sections share one lane", () => { + /** A side-by-side section of cameras 1 and 2 over 10–15 s of clip_1. */ + function withLayoutSection(document: AxcutDocument): AxcutDocument { + return documentSchema.parse({ + ...document, + legacyEditor: { + ...((document.legacyEditor as Record) ?? {}), + cameraLayoutRegions: [ + { + id: "camlayout_1", + startMs: 10_000, + endMs: 15_000, + clipId: "clip_1", + assetId: "asset_1", + sourceStartSec: 10, + sourceEndSec: 15, + template: "side-by-side", + slots: [{ camera: 0 }, { camera: 1 }], + }, + ], + }, + }); + } + + it("exposes layout sections read-only in the snapshot", () => { + const document = withLayoutSection(withCameraTrack(fixtureDocument())); + const snapshot = JSON.parse(executeAgentTool(document, "getCurrentDocument", "").resultJson); + expect(snapshot.cameraLayoutRegions).toEqual([ + { id: "camlayout_1", startSec: 10, endSec: 15, template: "side-by-side", cameras: [1, 2] }, + ]); + expect(snapshot.cameraLayoutNote).toMatch(/read-only/); + expect(snapshot.timeBaseNote).toMatch(/cameraLayoutRegions/); + }); + + it("an empty project reports no layout sections", () => { + const snapshot = JSON.parse( + executeAgentTool(fixtureDocument(), "getCurrentDocument", "").resultJson, + ); + expect(snapshot.cameraLayoutRegions).toEqual([]); + }); + + it("refuses to add a full-camera region over a layout section", () => { + const document = withLayoutSection(withCameraTrack(fixtureDocument())); + const result = executeAgentTool( + document, + "addCameraFullscreen", + JSON.stringify({ startSec: 12, endSec: 18 }), + ); + expect(result.ok).toBe(false); + expect(result.document).toBeUndefined(); + const error = JSON.parse(result.resultJson).error as string; + expect(error).toMatch(/camlayout_1/); + expect(error).toMatch(/side-by-side, 10–15 s/); + expect(error).toMatch(/may not overlap/); + }); + + it("allows a full-camera region that only touches a layout section", () => { + const document = withLayoutSection(withCameraTrack(fixtureDocument())); + const result = executeAgentTool( + document, + "addCameraFullscreen", + JSON.stringify({ startSec: 5, endSec: 10 }), + ); + expect(result.ok).toBe(true); + }); + + it("refuses to move a full-camera region onto a layout section", () => { + const document = withLayoutSection(withCameraTrack(fixtureDocument())); + const added = executeAgentTool( + document, + "addCameraFullscreen", + JSON.stringify({ startSec: 0, endSec: 5 }), + ); + expect(added.ok).toBe(true); + const regionId = ( + (added.document?.legacyEditor as Record).cameraFullscreenRegions as Array<{ + id: string; + }> + )[0].id; + const moved = executeAgentTool( + added.document as AxcutDocument, + "setCameraFullscreen", + JSON.stringify({ cameraFullscreenId: regionId, startSec: 8, endSec: 11 }), + ); + expect(moved.ok).toBe(false); + expect(moved.document).toBeUndefined(); + expect(JSON.parse(moved.resultJson).error).toMatch(/camlayout_1/); + }); +}); + // ── D-DESTRUCT ────────────────────────────────────────────────────────────── // // "Swap the two clips: put the demo first." There was no tool for it — while diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 3ffc9b215..efc2444d8 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -57,6 +57,10 @@ import { effectiveZoomScale, ZOOM_DEPTH_LEGEND, } from "../../src/lib/ai-edition/timeline/zoom-scale"; +import { + cameraSectionsOverlapping, + normalizeCameraLayoutRegions, +} from "../../src/lib/cameraLayouts"; import { SETTING_BOUNDS } from "../../src/lib/projectDefaults"; export interface AgentToolExecution { @@ -97,6 +101,10 @@ function modifierIds(document: AxcutDocument, kind: ModifierKind): string[] { return ((legacy.cameraFullscreenRegions as Array<{ id: string }> | undefined) ?? []).map( (region) => region.id, ); + case "cameraLayout": + return ((legacy.cameraLayoutRegions as Array<{ id: string }> | undefined) ?? []).map( + (region) => region.id, + ); case "audio": // Fragments of one user-visible track share `trackId` and render as a single pill, // so the id that disappears is the group key once, not one per fragment. @@ -265,6 +273,39 @@ function noCameraUnderSpan( ); } +/** The editor's file the agent reads layout sections from, coalesced to the pills the ruler draws. */ +function cameraLayoutPillsForAgent(document: AxcutDocument) { + const legacy = document.legacyEditor as Record | null; + return coalesceForAgent(normalizeCameraLayoutRegions(legacy?.cameraLayoutRegions)); +} + +/** + * Refuse a full-camera region that would land on a camera layout section. Both share one + * lane on the timeline and may never overlap (the editor refuses the same add as + * "occupied"); two overlapping sections would leave the scene to drop one of them. + * Layout sections are not editable from here, so the message names the one in the way. + */ +function cameraLayoutInTheWay( + document: AxcutDocument, + startSec: number, + endSec: number, +): AgentToolExecution | null { + const blocking = cameraSectionsOverlapping( + cameraLayoutPillsForAgent(document), + toMs(startSec), + toMs(endSec), + )[0]; + if (!blocking) return null; + return failure( + `The span ${startSec.toFixed(1)}–${endSec.toFixed(1)} s overlaps the camera layout ` + + `section ${blocking.id} (${blocking.template}, ${roundSec(blocking.startMs)}–` + + `${roundSec(blocking.endMs)} s), so no full-camera region was written. Full-camera ` + + "regions and layout sections share one camera lane and may not overlap. Pick a span " + + "outside every entry of cameraLayoutRegions in getCurrentDocument, or ask the user " + + "to change the layout section in the editor.", + ); +} + /** The span the edited timeline actually occupies, for an actionable refusal. */ function editedExtentSec(document: AxcutDocument): { startSec: number; endSec: number } { const clips = document.timeline.clips; @@ -335,18 +376,21 @@ function landingSuffix( return parts.length ? ` (${parts.join(", ")})` : ""; } -/** Ids of every modifier in the document, all four families at once — the basis +/** Ids of every modifier in the document, every family at once — the basis * for naming what a destructive edit took with it. */ function modifierIdsOf(document: AxcutDocument): string[] { const legacy = (document.legacyEditor as Record) ?? {}; const speedRegions = (legacy.speedRegions as Array<{ id: string }> | undefined) ?? []; const cameraFullscreenRegions = (legacy.cameraFullscreenRegions as Array<{ id: string }> | undefined) ?? []; + const cameraLayoutRegions = + (legacy.cameraLayoutRegions as Array<{ id: string }> | undefined) ?? []; return [ ...document.zoomRanges.map((r) => r.id), ...document.annotations.map((r) => r.id), ...speedRegions.map((r) => r.id), ...cameraFullscreenRegions.map((r) => r.id), + ...cameraLayoutRegions.map((r) => r.id), ]; } @@ -758,7 +802,11 @@ export function documentSnapshotForModel( const autoFocusAll = legacy?.autoFocusAll === true; return { timeBaseNote: - "clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions and audioTracks are in virtual (edited-timeline) seconds.", + "clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions, cameraLayoutRegions and audioTracks are in virtual (edited-timeline) seconds.", + cameraLayoutNote: + "cameraLayoutRegions are multi-camera layout sections the user placed in the editor; they are read-only here. " + + "They share one camera lane with cameraFullscreenRegions and the two may never overlap, so addCameraFullscreen / setCameraFullscreen refuse a span that lands on one. " + + "cameras lists the cameras shown, in place order, numbered as the user sees them (1 = the recording's main webcam).", audioNote: "audioTracks are imported voiceover / music files laid over the recording. They are clip-anchored like every other region, so they travel with their clip through reorder and trim, and they play at 1x whatever a speed region does to the picture under them. addAudio places an EXISTING asset of kind 'audio'; nothing here can import a file from disk or record one, so if the project has no audio asset, say so rather than inventing an id.", zoomNote: @@ -854,6 +902,13 @@ export function documentSnapshotForModel( startSec: roundSec(c.startMs), endSec: roundSec(c.endMs), })), + cameraLayoutRegions: cameraLayoutPillsForAgent(document).map((l) => ({ + id: l.id, + startSec: roundSec(l.startMs), + endSec: roundSec(l.endMs), + template: l.template, + cameras: l.slots.map((slot) => slot.camera + 1), + })), // Imported audio, collapsed to the pills the ruler draws — a track ventilated // across a clip boundary is several fragments the user sees as one thing, and // the model has to name what the user sees. @@ -2034,6 +2089,8 @@ export function executeAgentTool( } const blind = noCameraUnderSpan(document, landing.startSec, landing.endSec); if (blind) return blind; + const occupied = cameraLayoutInTheWay(document, landing.startSec, landing.endSec); + if (occupied) return occupied; const next: AxcutDocument = { ...document, legacyEditor: { ...legacy, cameraFullscreenRegions: [...prev, ...placed] }, @@ -2078,6 +2135,8 @@ export function executeAgentTool( } const blindMove = noCameraUnderSpan(document, landing.startSec, landing.endSec); if (blindMove) return blindMove; + const occupiedMove = cameraLayoutInTheWay(document, landing.startSec, landing.endSec); + if (occupiedMove) return occupiedMove; const next: AxcutDocument = { ...document, legacyEditor: { ...legacy, cameraFullscreenRegions: rebuiltCamera }, diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 64bcef03b..90a1b815f 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -26,6 +26,7 @@ import type { AxcutDocument } from "../../../src/lib/ai-edition/schema"; // wrong at both ends of a table that actually runs 1.25× to 5.0×. import { ZOOM_DEPTH_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-scale"; import { + type AgentToolExecution, addAnnotationArgs, addAudioArgs, addCameraFullscreenArgs, @@ -176,9 +177,9 @@ export const TOOL_DESCRIPTIONS: Record = { setAnnotation: "Move, resize, or edit the text of an existing annotation by id (virtual-timeline seconds). Only the fields you pass are changed.", addCameraFullscreen: - "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.", + "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying. It is also refused over a multi-camera layout section (cameraLayoutRegions): the two share one lane and may not overlap.", setCameraFullscreen: - "Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.", + "Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam or on a layout section (cameraLayoutRegions).", addAudio: "Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them; nothing here can import a file from disk or record one, so if there is none, say so instead of guessing an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.", setAudio: @@ -218,7 +219,7 @@ export interface CursorTelemetryReader { probe?(input: { assetId: string; originalPath: string | null }): Promise; } -interface ToolRuntime { +export interface ToolRuntime { cursor?: CursorTelemetryReader; availableByAssetId?: Record; } @@ -268,18 +269,7 @@ function documentTool( return tool( async (args: z.infer) => { sink.toolStart(name, args); - // ponytail: the ONE async step the pure executor cannot take. Reading a - // sidecar is IO; `executeAgentTool` is synchronous by design (it is the - // gate every mutation passes through, and it has to stay testable - // without a filesystem). So the load happens here and its verdict — - // including "I could not look" — goes in as data. - const load = TOOLS_READING_CURSOR.has(name) - ? await loadCursorTelemetry(holder.current, args, runtime) - : undefined; - const execution = executeAgentTool(holder.current, name, JSON.stringify(args), { - editsAllowed, - cursorTelemetry: { availableByAssetId: runtime.availableByAssetId, load }, - }); + const execution = await runDocumentTool(holder.current, name, args, editsAllowed, runtime); if (execution.document) holder.current = execution.document; sink.toolEnd(name, execution.ok, execution.summary); return execution.resultJson; @@ -288,6 +278,33 @@ function documentTool( ); } +/** + * Run one tool against a document: the cursor read it may need, then the shared + * executor. The in-app agent (`documentTool`) and the MCP server both call this, + * so a tool behaves the same whichever agent is driving it. + * + * ponytail: the ONE async step the pure executor cannot take. Reading a sidecar + * is IO; `executeAgentTool` is synchronous by design (it is the gate every + * mutation passes through, and it has to stay testable without a filesystem). + * So the load happens here and its verdict — including "I could not look" — goes + * in as data. + */ +export async function runDocumentTool( + document: AxcutDocument, + name: string, + args: unknown, + editsAllowed: boolean, + runtime: ToolRuntime, +): Promise { + const load = TOOLS_READING_CURSOR.has(name) + ? await loadCursorTelemetry(document, args, runtime) + : undefined; + return executeAgentTool(document, name, JSON.stringify(args ?? {}), { + editsAllowed, + cursorTelemetry: { availableByAssetId: runtime.availableByAssetId, load }, + }); +} + /** Reads the sidecar for whichever asset the call names, defaulting to the * primary one — the same resolution the executor will use to report it. */ async function loadCursorTelemetry( @@ -331,37 +348,44 @@ export function buildTools( editsAllowed = true, runtime: ToolRuntime = {}, ) { - const build = (name: string, schema: S) => - documentTool(holder, sink, name, schema, editsAllowed, runtime); - return [ - build("getCurrentDocument", z.object({})), - build("getTranscript", getTranscriptArgs), - build("getTranscriptWords", getTranscriptWordsArgs), - build("getCursorTrack", getCursorTrackArgs), - build("setWordText", setWordTextArgs), - build("addTrim", addTrimArgs), - build("addTrims", addTrimsArgs), - build("setTrim", setTrimArgs), - build("setClipRange", setClipRangeArgs), - build("moveClip", moveClipArgs), - build("replaceTimeline", replaceTimelineArgs), - build("addZoom", addZoomArgs), - build("addZooms", addZoomsArgs), - build("setZoom", setZoomArgs), - build("addSpeed", addSpeedArgs), - build("setSpeed", setSpeedArgs), - build("addAnnotation", addAnnotationArgs), - build("setAnnotation", setAnnotationArgs), - build("addCameraFullscreen", addCameraFullscreenArgs), - build("setCameraFullscreen", setCameraFullscreenArgs), - build("addAudio", addAudioArgs), - build("setAudio", setAudioArgs), - build("removeTrim", removeTrimArgs), - build("removeModifier", removeModifierArgs), - build("removeClip", removeClipArgs), - ]; + return TOOL_ARG_SCHEMAS.map(([name, schema]) => + documentTool(holder, sink, name, schema, editsAllowed, runtime), + ); } +/** + * Every tool's name and argument schema, in the order the model is handed them. + * `buildTools` wraps these for the in-app agent and `electron/mcp/` exposes the + * same list over MCP, so the two surfaces cannot drift apart. + */ +export const TOOL_ARG_SCHEMAS: ReadonlyArray = [ + ["getCurrentDocument", z.object({})], + ["getTranscript", getTranscriptArgs], + ["getTranscriptWords", getTranscriptWordsArgs], + ["getCursorTrack", getCursorTrackArgs], + ["setWordText", setWordTextArgs], + ["addTrim", addTrimArgs], + ["addTrims", addTrimsArgs], + ["setTrim", setTrimArgs], + ["setClipRange", setClipRangeArgs], + ["moveClip", moveClipArgs], + ["replaceTimeline", replaceTimelineArgs], + ["addZoom", addZoomArgs], + ["addZooms", addZoomsArgs], + ["setZoom", setZoomArgs], + ["addSpeed", addSpeedArgs], + ["setSpeed", setSpeedArgs], + ["addAnnotation", addAnnotationArgs], + ["setAnnotation", setAnnotationArgs], + ["addCameraFullscreen", addCameraFullscreenArgs], + ["setCameraFullscreen", setCameraFullscreenArgs], + ["addAudio", addAudioArgs], + ["setAudio", setAudioArgs], + ["removeTrim", removeTrimArgs], + ["removeModifier", removeModifierArgs], + ["removeClip", removeClipArgs], +]; + /** * Prompt caching for the Anthropic-wire providers, which `createDeepAgent` * used to add for us (`isAnthropicModel` → `anthropicPromptCachingMiddleware`). @@ -401,7 +425,7 @@ export interface InvokeArgs { /** One cheap probe per asset, run before the tools are built so the very first * `getCurrentDocument` can already say whether telemetry exists. */ -async function probeCursorTelemetry( +export async function probeCursorTelemetry( document: AxcutDocument, cursor: CursorTelemetryReader | undefined, ): Promise | undefined> { diff --git a/electron/app-settings.test.ts b/electron/app-settings.test.ts index c6c9e74f3..f7590fdb6 100644 --- a/electron/app-settings.test.ts +++ b/electron/app-settings.test.ts @@ -129,4 +129,95 @@ describe("app settings store", () => { store.dismissStarPrompt(); expect(store.getSnapshot().recording).toMatchObject({ micEnabled: true, micDeviceId: "mic" }); }); + + it("stores a camera quality and rejects one it cannot capture at", () => { + const dir = temp(); + const store = new AppSettingsStore(dir); + + expect(store.setRecordingPreferences({ camQuality: "1080p" }).recording.camQuality).toBe( + "1080p", + ); + expect(() => store.setRecordingPreferences({ camQuality: "4320p" as never })).toThrow( + TypeError, + ); + expect(store.getSnapshot().recording.camQuality).toBe("1080p"); + }); + + it("leaves the camera quality alone when a patch does not mention it", () => { + // Every other window writes narrow patches; a camera toggle from the HUD + // must not quietly reset the resolution the user picked. + const dir = temp(); + const store = new AppSettingsStore(dir); + store.setRecordingPreferences({ camQuality: "1440p" }); + + store.setRecordingPreferences({ camEnabled: true }); + + expect(store.getSnapshot().recording.camQuality).toBe("1440p"); + }); + + it("reads a settings file written before the camera had a quality", () => { + const dir = temp(); + writeFileSync( + path.join(dir, "recording-settings.json"), + JSON.stringify({ micEnabled: true }), + "utf8", + ); + + expect(new AppSettingsStore(dir).getSnapshot().recording.camQuality).toBe("2160p"); + }); + + it("reads a missing additional camera list as empty", () => { + const dir = temp(); + writeFileSync( + path.join(dir, "recording-settings.json"), + JSON.stringify({ camEnabled: true }), + "utf8", + ); + + expect(new AppSettingsStore(dir).getSnapshot().recording.camAdditionalDevices).toEqual([]); + }); + + it("drops additional camera entries without a name and keeps at most three", () => { + const store = new AppSettingsStore(temp()); + store.setRecordingPreferences({ + camAdditionalDevices: [ + { id: "a", name: "A" }, + { id: null, name: "B" }, + { id: "x", name: "" }, + { id: "c", name: "C" }, + { id: "d", name: "D" }, + { id: "e", name: "E" }, + ], + }); + + expect(store.getSnapshot().recording.camAdditionalDevices).toEqual([ + { id: "a", name: "A" }, + { id: null, name: "B" }, + { id: "c", name: "C" }, + ]); + }); + + it("ignores junk in a stored additional camera list", () => { + const dir = temp(); + writeFileSync( + path.join(dir, "recording-settings.json"), + JSON.stringify({ + camAdditionalDevices: [null, 3, "x", { name: 5 }, { id: 7, name: "Kept" }, { name: "Ok" }], + }), + "utf8", + ); + + expect(new AppSettingsStore(dir).getSnapshot().recording.camAdditionalDevices).toEqual([ + { id: null, name: "Kept" }, + { id: null, name: "Ok" }, + ]); + }); + + it("rejects an additional camera list that is not a list", () => { + const store = new AppSettingsStore(temp()); + + expect(() => store.setRecordingPreferences({ camAdditionalDevices: "x" as never })).toThrow( + TypeError, + ); + }); }); diff --git a/electron/app-settings.ts b/electron/app-settings.ts index 2cc1cee25..d1efb9aae 100644 --- a/electron/app-settings.ts +++ b/electron/app-settings.ts @@ -1,5 +1,11 @@ import { readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import path from "node:path"; +import { + DEFAULT_WEBCAM_QUALITY, + WEBCAM_QUALITY_IDS, + type WebcamQualityId, + webcamQualityFrom, +} from "../src/hooks/webcamCaptureTarget"; import type { CursorCaptureMode } from "../src/lib/recordingSession"; export interface RecordingPreferences { @@ -9,6 +15,10 @@ export interface RecordingPreferences { camEnabled: boolean; camDeviceId: string | null; camDeviceName: string | null; + /** Cameras 2-4 of a native Windows recording, in the order they were picked. At most three. */ + camAdditionalDevices: Array<{ id: string | null; name: string }>; + /** Capture resolution for the camera. See WEBCAM_QUALITY_PRESETS. */ + camQuality: WebcamQualityId; systemAudioEnabled: boolean; cursorCaptureMode: CursorCaptureMode; /** Display captures on macOS and Windows. Opt-in: on Windows the icons also leave the real desktop while recording. */ @@ -30,6 +40,8 @@ export const DEFAULT_RECORDING_PREFERENCES: RecordingPreferences = { camEnabled: false, camDeviceId: null, camDeviceName: null, + camAdditionalDevices: [], + camQuality: DEFAULT_WEBCAM_QUALITY, systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, @@ -92,6 +104,22 @@ const bool = (value: unknown, fallback: boolean) => (typeof value === "boolean" const nullableString = (value: unknown, fallback: string | null) => value === null || typeof value === "string" ? value : fallback; +const MAX_ADDITIONAL_CAMERAS = 3; + +/** Keeps the entries that name a camera, in order, up to the cap; everything else is junk. */ +function additionalCameras(value: unknown): RecordingPreferences["camAdditionalDevices"] { + if (!Array.isArray(value)) return []; + const kept: RecordingPreferences["camAdditionalDevices"] = []; + for (const entry of value) { + if (kept.length >= MAX_ADDITIONAL_CAMERAS) break; + if (!entry || typeof entry !== "object") continue; + const { id, name } = entry as Record; + if (typeof name !== "string" || name.length === 0) continue; + kept.push({ id: typeof id === "string" ? id : null, name }); + } + return kept; +} + function parseRecording(raw: RawSettings): RecordingPreferences { return { micEnabled: bool(raw.micEnabled, DEFAULT_RECORDING_PREFERENCES.micEnabled), @@ -100,6 +128,10 @@ function parseRecording(raw: RawSettings): RecordingPreferences { camEnabled: bool(raw.camEnabled, DEFAULT_RECORDING_PREFERENCES.camEnabled), camDeviceId: nullableString(raw.camDeviceId, DEFAULT_RECORDING_PREFERENCES.camDeviceId), camDeviceName: nullableString(raw.camDeviceName, DEFAULT_RECORDING_PREFERENCES.camDeviceName), + camAdditionalDevices: additionalCameras(raw.camAdditionalDevices), + // Unset in every settings file written before the camera had a quality + // setting, and `webcamQualityFrom` answers those with the default. + camQuality: webcamQualityFrom(raw.camQuality), systemAudioEnabled: bool( raw.systemAudioEnabled, DEFAULT_RECORDING_PREFERENCES.systemAudioEnabled, @@ -157,6 +189,10 @@ function validateRecordingPatch(patch: Partial): void { if ((key.endsWith("Enabled") || key === "hideDesktopIcons") && typeof value !== "boolean") { throw new TypeError(`${key} must be a boolean`); } + if (key === "camAdditionalDevices") { + if (!Array.isArray(value)) throw new TypeError("camAdditionalDevices must be a list"); + continue; + } if ( (key.endsWith("DeviceId") || key.endsWith("DeviceName")) && value !== null && @@ -167,6 +203,12 @@ function validateRecordingPatch(patch: Partial): void { if (key === "cursorCaptureMode" && value !== "system" && value !== "editable-overlay") { throw new TypeError("cursorCaptureMode is invalid"); } + // Rejected here rather than coerced on read, so a bad write is a visible + // error at its source instead of a resolution that silently is not the + // one the caller asked for. + if (key === "camQuality" && !WEBCAM_QUALITY_IDS.includes(value as WebcamQualityId)) { + throw new TypeError("camQuality is invalid"); + } } } @@ -189,6 +231,9 @@ export class AppSettingsStore { const next = Object.fromEntries( Object.entries(patch).filter(([, value]) => value !== undefined), ) as Partial; + if (next.camAdditionalDevices) { + next.camAdditionalDevices = additionalCameras(next.camAdditionalDevices); + } atomicWrite(this.userData, { ...raw, ...current, ...next }); return this.getSnapshot(); } diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 29d5f828b..1c28c8b4f 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -37,6 +37,15 @@ interface Window { * `compositor.export`/`compositor.exportMulti` runs. Distinct from `exportOnFrameAck`, * the OLD web/CPU pipeline's per-frame ack, not a progress signal. */ onNativeExportProgress?: (callback: (frames: number, exportId?: string) => void) => () => void; + /** Preview frames the main process hands over as shared GPU textures (Windows). The + * listener draws `frame` and closes it; one listener at a time. Returns the + * unsubscribe. Optional: shim/web contexts have no bridge. */ + onCompositorFrame?: ( + listener: ( + frame: VideoFrame, + meta: import("../src/native/contracts").CompositorSharedFrameMeta, + ) => void, + ) => () => void; getSources: (opts: Electron.SourcesOptions) => Promise; switchToEditor: () => Promise; switchToHud: () => Promise; @@ -47,6 +56,8 @@ interface Window { }>; /** Sources are picked in Apple's system picker (macOS 15.2+), not in an app list. */ usesSystemSourcePicker?: () => Promise; + /** Name of the last source picked in Apple's picker, for display only: it is not selected. */ + getLastPickedSource?: () => Promise; openNotes: () => Promise<{ opened: boolean; reason?: string; @@ -71,6 +82,12 @@ interface Window { onAiEditionChatEvent: ( callback: (event: import("../src/native/contracts").AiEditionChatEvent) => void, ) => () => void; + /** Optional: absent in the browser shim and in tests that stub electronAPI. */ + onAiEditionMcpRequest?: ( + callback: ( + request: import("../src/native/contracts").AiEditionMcpHostRequest, + ) => Promise, + ) => () => void; requestCameraAccess: () => Promise<{ success: boolean; granted: boolean; @@ -160,6 +177,17 @@ interface Window { * saved without it. Still a success — the screen video is intact. */ webcamDropped?: boolean; + /** + * Labels (device name, else `Camera `) of additional cameras (2-4) that + * produced nothing usable and were left out of the session. Camera 1 is + * `webcamDropped`. + */ + droppedWebcams?: string[]; + /** + * Labels of cameras (camera 1 included) that the helper disabled mid-take. + * Their partial files are kept in the session; the user is told which. + */ + webcamsStoppedEarly?: string[]; }>; pauseNativeWindowsRecording: () => Promise<{ success: boolean; @@ -172,6 +200,7 @@ interface Window { startNativeMacRecording: ( request: import("../src/lib/nativeMacRecording").NativeMacRecordingRequest, ) => Promise; + onNativeMacSystemAudioUnavailable: (callback: () => void) => () => void; pauseNativeMacRecording: () => Promise<{ success: boolean; error?: string; @@ -335,12 +364,9 @@ interface Window { /** Why this recording ended before it was stopped, when it did. */ warning?: string; }>; - findRecordingCamera: (videoPath: string) => Promise<{ - success: boolean; - webcamVideoPath?: string; - offsetMs?: number; - error?: string; - }>; + findRecordingCamera: ( + videoPath: string, + ) => Promise; readBinaryFile: (filePath: string) => Promise<{ success: boolean; data?: ArrayBuffer; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index bd19f1059..8270c69da 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -13,10 +13,12 @@ import { desktopCapturer, dialog, ipcMain, + safeStorage, screen, shell, systemPreferences, } from "electron"; +import { DEFAULT_WEBCAM_QUALITY, type WebcamQualityId } from "../../src/hooks/webcamCaptureTarget"; import { type AxcutDocument, isAxcutDocumentFile, @@ -32,7 +34,9 @@ import { } from "../../src/lib/nativeMacRecording"; import type { NativeWindowsRecordingRequest } from "../../src/lib/nativeWindowsRecording"; import { + type AdditionalWebcam, type CursorCaptureMode, + type FindRecordingCameraResult, normalizeCursorCaptureMode, normalizeProjectMedia, normalizeRecordingSession, @@ -68,6 +72,9 @@ import { isDiagnosticModeEnabled, mainLogBuffer } from "../diagnostics/main-log- import { mainT } from "../i18n"; import { getInstallChannel } from "../install-channel"; import { RECORDINGS_DIR } from "../main"; +import { EditorDocumentHost } from "../mcp/editor-document-host"; +import { McpController } from "../mcp/mcp-controller"; +import { McpSettingsStore } from "../mcp/mcp-settings-store"; import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks"; import { readCursorRecordingFile as readCursorRecordingFileFrom, @@ -116,17 +123,33 @@ import { NATIVE_WINDOWS_SALVAGEABLE_OUTPUT_BYTES, readMicrophoneDefaulted, readMicrophoneUnavailable, - readWebcamFormat, - readWebcamUnavailable, + readReportedWebcamPaths, + readSecondaryWindowsApplied, + readStoppedWebcamPaths, + readUnavailableWebcamIndices, + readWebcamFormatAt, terminateNativeWindowsCapture, waitForNativeWindowsCaptureStop, } from "../recording/nativeWindowsCaptureStop"; +import { + additionalWebcamLabels, + buildHelperWebcamConfig, + collectStoppedWebcams, + dedupeAdditionalWebcams, + isWebcamSidecarFile, + labelsOfUnavailableAdditionalWebcams, + labelsOfWebcamsStoppedEarly, + stripWebcamSuffix, + webcamOutputPath, +} from "../recording/nativeWindowsWebcams"; import { patchWebmDurationOnDisk } from "../recording/webm-duration"; import { reindexRecordingOnDisk } from "../recording/webm-seek-index"; import { + describeMacPickerSource, describeRecordingSource, enumerationIncludesSourceKind, mergeEnumeratedSources, + rememberedPickerSourceName, resolveRecordingSource, restoreRecordingSourceAfterEnumeration, shouldEnumerateRecordingSources, @@ -590,9 +613,25 @@ async function getApprovedProjectSession( throw new Error("Project references an invalid or unsupported webcam video path"); } - return webcamVideoPath - ? { screenVideoPath, webcamVideoPath, createdAt: Date.now() } - : { screenVideoPath, createdAt: Date.now() }; + // Additional cameras go through the same approval as camera 1; one that no + // longer resolves is dropped rather than failing the whole project. + const additionalWebcams: AdditionalWebcam[] = []; + for (const extra of media.additionalWebcams ?? []) { + const approved = await approveReadableVideoPath( + await resolveWithSiblingFallback(extra.path), + trustedDirs, + ); + if (approved) { + additionalWebcams.push({ path: approved, label: extra.label }); + } + } + + return { + screenVideoPath, + ...(webcamVideoPath ? { webcamVideoPath } : {}), + ...(additionalWebcams.length > 0 ? { additionalWebcams } : {}), + createdAt: Date.now(), + }; } type SelectedSource = { @@ -649,6 +688,10 @@ export interface RecordingPrefs { camDeviceId: string | null; /** Camera label paired with the preferred id for restart-safe resolution. */ camDeviceName: string | null; + /** Cameras 2-4 of a native Windows recording, in pick order. At most three. */ + camAdditionalDevices: Array<{ id: string | null; name: string }>; + /** Capture resolution for the camera. See WEBCAM_QUALITY_PRESETS. */ + camQuality: WebcamQualityId; systemAudioEnabled: boolean; cursorCaptureMode: CursorCaptureMode; hideDesktopIcons: boolean; @@ -662,6 +705,8 @@ const defaultRecordingPrefs: RecordingPrefs = { camEnabled: false, camDeviceId: null, camDeviceName: null, + camAdditionalDevices: [], + camQuality: DEFAULT_WEBCAM_QUALITY, systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, @@ -715,6 +760,18 @@ let nativeWindowsCaptureProcess: ChildProcessWithoutNullStreams | null = null; let nativeWindowsCaptureOutput = ""; let nativeWindowsCaptureTargetPath: string | null = null; let nativeWindowsCaptureWebcamTargetPath: string | null = null; +/** + * Cameras 2-4 of the running take, in helper order after camera 1, each with + * the label it is reported under. Only paths generated here ever land in it. + */ +let nativeWindowsCaptureAdditionalWebcamTargets: Array<{ path: string; label: string }> = []; +/** Camera 1's label for notices: its device name, else "Camera 1". */ +let nativeWindowsCaptureWebcamLabel = "Camera 1"; +/** + * Files of cameras the helper dropped at start. The helper deletes them, but a + * delete that failed leaves a 0-byte stub; stop and discard remove it if empty. + */ +let nativeWindowsCaptureDroppedWebcamPaths: string[] = []; let nativeWindowsCaptureRecordingId: number | null = null; let nativeWindowsCursorOffsetMs = 0; let nativeWindowsCursorCaptureMode: CursorCaptureMode = "editable-overlay"; @@ -741,6 +798,9 @@ function resetNativeWindowsCaptureState() { nativeWindowsCaptureProcess = null; nativeWindowsCaptureTargetPath = null; nativeWindowsCaptureWebcamTargetPath = null; + nativeWindowsCaptureAdditionalWebcamTargets = []; + nativeWindowsCaptureWebcamLabel = "Camera 1"; + nativeWindowsCaptureDroppedWebcamPaths = []; nativeWindowsCaptureRecordingId = null; nativeWindowsCursorOffsetMs = 0; nativeWindowsCursorCaptureMode = "editable-overlay"; @@ -768,12 +828,12 @@ async function salvageNativeWindowsFragmentedCapture(screenVideoPath: string | n */ async function removeNativeWindowsCaptureOutputs( screenVideoPath: string | null, - webcamVideoPath: string | null, + webcamVideoPaths: Array, options: { onlyIfUnusable?: boolean } = {}, ) { const targets = [ screenVideoPath, - webcamVideoPath, + ...webcamVideoPaths, screenVideoPath ? `${screenVideoPath}.cursor.json` : null, ]; @@ -799,6 +859,22 @@ async function removeNativeWindowsCaptureOutputs( } } } + +/** Removes the 0-byte stubs of cameras dropped at start; anything with data stays. */ +async function removeEmptyNativeWindowsWebcamFiles(paths: string[]) { + for (const target of paths) { + if (!isPathWithinDir(target, RECORDINGS_DIR)) { + continue; + } + const stats = await fs.stat(target).catch(() => null); + if (stats?.size !== 0) { + continue; + } + await fs.rm(target, { force: true }).catch((error) => { + console.warn("[native-wgc] could not remove an empty camera file:", target, error); + }); + } +} let nativeMacCaptureProcess: ChildProcessWithoutNullStreams | null = null; /** * Apple's system picker session (macOS 15.2+), started on first use and kept for the app's @@ -1054,7 +1130,9 @@ async function getMacPickerSession(): Promise { return session; } -function selectedSourceFromPick(pick: MacPickerSelection): SelectedSource { +function selectedSourceFromPick( + pick: MacPickerSelection, +): SelectedSource & { id: string; display_id: string } { const display = pick.displayId !== null ? screen.getAllDisplays().find((candidate) => candidate.id === pick.displayId) @@ -1328,6 +1406,7 @@ async function registerRecordingMediaLinks( options: { webcamVideoPath?: string; webcamOffsetMs?: number; + additionalWebcams?: AdditionalWebcam[]; cursorCaptureMode?: CursorCaptureMode; }, ) { @@ -1342,6 +1421,9 @@ async function registerRecordingMediaLinks( ...(options.webcamVideoPath && Number.isFinite(options.webcamOffsetMs) ? { webcamOffsetMs: options.webcamOffsetMs } : {}), + ...(options.additionalWebcams?.length + ? { additionalWebcams: options.additionalWebcams } + : {}), ...(hasCursorTelemetry ? { cursorTelemetryPath } : {}), ...(options.cursorCaptureMode ? { cursorCaptureMode: options.cursorCaptureMode } : {}), }); @@ -1605,6 +1687,7 @@ function inspectNativeMacCaptureOutput() { function attachNativeMacCaptureOutputDrain( proc: ChildProcessWithoutNullStreams, onTakeEnded: () => void, + onSystemAudioUnavailable: () => void, ) { let lineBuffer = ""; // Hooked here rather than on `nativeMacCaptureEvents`, which the start wait @@ -1612,6 +1695,7 @@ function attachNativeMacCaptureOutputDrain( const watchLiveTake = createNativeMacMidCaptureErrorWatch( () => nativeMacCaptureProcess === proc && !nativeMacStopInFlight, onTakeEnded, + onSystemAudioUnavailable, ); const drain = (chunk: Buffer) => { const text = chunk.toString(); @@ -1721,9 +1805,7 @@ function setCurrentRecordingSessionState(session: RecordingSession | null) { function getSessionManifestPathForVideo(videoPath: string) { const parsedPath = path.parse(videoPath); - const baseName = parsedPath.name.endsWith("-webcam") - ? parsedPath.name.slice(0, -"-webcam".length) - : parsedPath.name; + const baseName = stripWebcamSuffix(parsedPath.name); return path.join(parsedPath.dir, `${baseName}${RECORDING_SESSION_SUFFIX}`); } @@ -1777,10 +1859,34 @@ async function loadRecordedSessionForVideoPath( } } + if (session.additionalWebcams) { + const approvedExtras: AdditionalWebcam[] = []; + for (const extra of session.additionalWebcams) { + let extraPath: string | null = extra.path; + if (!isPathAllowed(extraPath)) { + extraPath = await approveReadableVideoPath(extraPath, [ + path.dirname(manifestPath), + RECORDINGS_DIR, + ]); + } + if (extraPath) { + approvedExtras.push({ path: extraPath, label: extra.label }); + } + } + if (approvedExtras.length > 0) { + session.additionalWebcams = approvedExtras; + } else { + delete session.additionalWebcams; + } + } + approveFilePath(session.screenVideoPath); if (session.webcamVideoPath) { approveFilePath(session.webcamVideoPath); } + for (const extra of session.additionalWebcams ?? []) { + approveFilePath(extra.path); + } return session; } catch (error) { const nodeError = error as NodeJS.ErrnoException; @@ -1800,6 +1906,7 @@ async function loadRecordedSessionForVideoPath( async function resolveMediaLinksForVideo(videoPath: string): Promise<{ webcamVideoPath?: string; webcamOffsetMs?: number; + additionalWebcams?: AdditionalWebcam[]; cursorTelemetryPath?: string; resolvedVia: "sidecar" | "fingerprint" | "none"; }> { @@ -1810,6 +1917,7 @@ async function resolveMediaLinksForVideo(videoPath: string): Promise<{ .then(() => true) .catch(() => false); + const sessionAdditionalWebcams = session?.additionalWebcams ?? []; if (session?.webcamVideoPath || hasCursorTelemetry) { // Opportunistic backfill so the link survives a later move even if this // recording predates the registry, or if its sidecar doesn't travel with it. @@ -1818,6 +1926,9 @@ async function resolveMediaLinksForVideo(videoPath: string): Promise<{ ...(session?.webcamVideoPath && Number.isFinite(session.webcamOffsetMs) ? { webcamOffsetMs: session.webcamOffsetMs } : {}), + ...(sessionAdditionalWebcams.length > 0 + ? { additionalWebcams: sessionAdditionalWebcams } + : {}), ...(hasCursorTelemetry ? { cursorTelemetryPath } : {}), }).catch((error) => console.warn("[media-links] backfill failed:", error)); @@ -1828,6 +1939,9 @@ async function resolveMediaLinksForVideo(videoPath: string): Promise<{ webcamOffsetMs: session.webcamOffsetMs ?? 0, } : {}), + ...(sessionAdditionalWebcams.length > 0 + ? { additionalWebcams: sessionAdditionalWebcams } + : {}), ...(hasCursorTelemetry ? { cursorTelemetryPath } : {}), resolvedVia: "sidecar", }; @@ -1841,8 +1955,19 @@ async function resolveMediaLinksForVideo(videoPath: string): Promise<{ webcamVideoPath = (await approveReadableVideoPath(webcamVideoPath, [RECORDINGS_DIR])) ?? undefined; } + const additionalWebcams: AdditionalWebcam[] = []; + for (const extra of links.additionalWebcams ?? []) { + let extraPath: string | null = extra.path; + if (!isPathAllowed(extraPath)) { + extraPath = await approveReadableVideoPath(extraPath, [RECORDINGS_DIR]); + } + if (extraPath) { + additionalWebcams.push({ path: extraPath, label: extra.label }); + } + } return { ...(webcamVideoPath ? { webcamVideoPath, webcamOffsetMs: links.webcamOffsetMs ?? 0 } : {}), + ...(additionalWebcams.length > 0 ? { additionalWebcams } : {}), ...(links.cursorTelemetryPath ? { cursorTelemetryPath: links.cursorTelemetryPath } : {}), resolvedVia: "fingerprint", }; @@ -2081,13 +2206,19 @@ export function registerIpcHandlers( }, ); - async function presentMacSystemPicker(session: MacPickerSession) { - // The HUD and the notes window belong to this process, not to the helper, so a display - // pick would record them unless the picker is told to leave them out. + /** + * The HUD and the notes window belong to this process, not to the helper, so a display + * pick would record them unless the picker is told to leave them out. + */ + function macPickerExcludedWindowIds() { const appWindowSourceIds = [getMainWindow(), getNotesWindow()] .filter((window): window is BrowserWindow => !!window && !window.isDestroyed()) .map((window) => window.getMediaSourceId()); - const excludedWindowIds = collectMacCaptureExcludedWindowIds(appWindowSourceIds); + return collectMacCaptureExcludedWindowIds(appWindowSourceIds); + } + + async function presentMacSystemPicker(session: MacPickerSession) { + const excludedWindowIds = macPickerExcludedWindowIds(); // Out of the way while the picker is up. The HUD window is far larger than the bar // it draws (a transparent reserve above it), and Apple's picker targets windows by // their frame, not by where clicks land -- so that invisible rectangle hid every @@ -2119,8 +2250,15 @@ export function registerIpcHandlers( } return; } - selectedSource = selectedSourceFromPick(pick); + const picked = selectedSourceFromPick(pick); + selectedSource = picked; selectedDesktopSource = null; + // Written only so the next launch can name it: the pick itself dies with the helper. + try { + appSettings.setLastSource(describeMacPickerSource(picked, pick.kind)); + } catch (error) { + console.warn("Failed to persist the picked recording source:", error); + } broadcastSelectedSource(selectedSource); } @@ -2132,6 +2270,12 @@ export function registerIpcHandlers( // picker they must hand the choice to `open-source-selector` rather than enumerate. ipcMain.handle("uses-system-source-picker", () => macPickerOwnsSources()); + // Named in the HUD and Record mode after a relaunch, while `get-selected-source` still + // answers null: the pick cannot be restored, so it is never reported as selected. + ipcMain.handle("get-last-picked-source", () => + rememberedPickerSourceName(appSettings.getSnapshot().lastSource, macPickerOwnsSources()), + ); + ipcMain.handle("get-selected-source", async () => { const previousSelectedSource = selectedSource; if (process.platform === "linux" && findPipeWireCursorHelperPath()) { @@ -2151,6 +2295,9 @@ export function registerIpcHandlers( if (!isMacPickerSourceId(selectedSource?.id)) { return selectedDesktopSource ? selectedSource : null; } + // A screen pick that would record the current HUD or notes window is no pick at all: + // answering null makes the next Record present the picker again (#965). + macPickerSession?.forgetSelectionUnlessExcluding(macPickerExcludedWindowIds()); if (!macPickerSession?.getSelection()) { selectedSource = null; selectedDesktopSource = null; @@ -2287,12 +2434,11 @@ export function registerIpcHandlers( return access; } - // The helper that answered has just raised macOS' own Accessibility prompt on - // its way up, so the window must offer System Settings, not a second prompt. It - // explains what the grant is for and tracks it live, where a message box could - // only say "go to System Settings" in English. + // Accessibility improves cursor shape hints but is not required to record. + // Remember that the helper raised the system prompt so a later visit to the + // permissions window can direct the user to Settings, but don't reopen that + // window from every Record press. getMacPermissions().noteRequested("accessibility"); - showPermissionsWindow(); } return access; @@ -2755,10 +2901,7 @@ export function registerIpcHandlers( ? request.recordingId : Date.now(); const outputPath = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`); - const webcamOutputPath = path.join( - RECORDINGS_DIR, - `${RECORDING_FILE_PREFIX}${recordingId}-webcam.mp4`, - ); + const webcamPath = webcamOutputPath(RECORDINGS_DIR, RECORDING_FILE_PREFIX, recordingId, 1); const sourceDisplay = request.source.type === "display" && typeof request.source.displayId === "number" ? (screen.getAllDisplays().find((display) => display.id === request.source.displayId) ?? @@ -2777,6 +2920,24 @@ export function registerIpcHandlers( const webcamDirectShowClsid = request.webcam.enabled ? await resolveDirectShowWebcamClsid(request.webcam.deviceName) : null; + // Cameras 2-4 only while camera 1 is on: its toggle governs every + // camera. Files are numbered from 2 in the order they are sent. + const keptExtras = request.webcam.enabled + ? dedupeAdditionalWebcams( + request.webcam, + Array.isArray(request.additionalWebcams) ? request.additionalWebcams : [], + ) + : []; + const extraLabels = additionalWebcamLabels(request.webcam.deviceName, keptExtras); + const additionalWebcams = await Promise.all( + keptExtras.map(async (extra, i) => ({ + deviceId: extra.deviceId, + deviceName: extra.deviceName, + label: extraLabels[i], + clsid: await resolveDirectShowWebcamClsid(extra.deviceName), + path: webcamOutputPath(RECORDINGS_DIR, RECORDING_FILE_PREFIX, recordingId, i + 2), + })), + ); const cursorCaptureMode = normalizeCursorCaptureMode(request.cursor?.mode) ?? "editable-overlay"; const envPreferSoftwareEncoder = (process.env.OPENSCREEN_WGC_PREFER_SOFTWARE_ENCODER ?? "") @@ -2808,13 +2969,12 @@ export function registerIpcHandlers( microphoneDeviceId: request.audio.microphone.deviceId ?? null, microphoneDeviceName: request.audio.microphone.deviceName ?? null, microphoneGain: request.audio.microphone.gain, - webcamEnabled: request.webcam.enabled, - webcamDeviceId: request.webcam.deviceId ?? null, - webcamDeviceName: request.webcam.deviceName ?? null, - webcamDirectShowClsid, - webcamWidth: request.webcam.width, - webcamHeight: request.webcam.height, - webcamFps: request.webcam.fps, + ...buildHelperWebcamConfig({ + camera1: request.webcam, + camera1Clsid: webcamDirectShowClsid, + camera1Path: webcamPath, + extras: additionalWebcams, + }), captureCursor: cursorCaptureMode === "system", cursorCaptureMode, hideDesktopIcons: @@ -2822,7 +2982,7 @@ export function registerIpcHandlers( appSettings.getSnapshot().recording.hideDesktopIcons, outputs: { screenPath: outputPath, - webcamPath: webcamOutputPath, + webcamPath, }, source: { type: request.source.type, @@ -2844,6 +3004,10 @@ export function registerIpcHandlers( source: request.source, audio: request.audio, webcam: request.webcam, + additionalWebcams: additionalWebcams.map(({ label, path: cameraPath }) => ({ + label, + path: cameraPath, + })), encoder: { preferSoftwareEncoder }, cursor: { mode: cursorCaptureMode }, // Both spaces, deliberately: the helper's own errors quote the physical @@ -2858,7 +3022,12 @@ export function registerIpcHandlers( await fs.mkdir(RECORDINGS_DIR, { recursive: true }); nativeWindowsCaptureOutput = ""; nativeWindowsCaptureTargetPath = outputPath; - nativeWindowsCaptureWebcamTargetPath = request.webcam.enabled ? webcamOutputPath : null; + nativeWindowsCaptureWebcamTargetPath = request.webcam.enabled ? webcamPath : null; + nativeWindowsCaptureAdditionalWebcamTargets = additionalWebcams.map( + ({ label, path: cameraPath }) => ({ label, path: cameraPath }), + ); + nativeWindowsCaptureWebcamLabel = request.webcam.deviceName?.trim() || "Camera 1"; + nativeWindowsCaptureDroppedWebcamPaths = []; nativeWindowsCaptureRecordingId = recordingId; nativeWindowsCursorOffsetMs = 0; nativeWindowsCursorCaptureMode = cursorCaptureMode; @@ -2894,7 +3063,13 @@ export function registerIpcHandlers( cursorCaptureMode === "editable-overlay" ? Math.max(0, captureStartedAtMs - cursorStartTimeMs) : 0; - const webcamFormat = readWebcamFormat(nativeWindowsCaptureOutput); + // Index-aware readers: with several cameras the helper prints one + // format line and possibly one unavailable warning per camera, and + // only index 0 (or no index, from an old helper) is camera 1. + const webcamFormat = readWebcamFormatAt(nativeWindowsCaptureOutput, 0); + const unavailableWebcamIndices = new Set( + readUnavailableWebcamIndices(nativeWindowsCaptureOutput), + ); const encoderSelection = readNativeWindowsEncoderSelection(nativeWindowsCaptureOutput); // Captured now because stop may have no helper left to ask. A helper // killed mid-recording is exactly the case where this matters most. @@ -2903,7 +3078,13 @@ export function registerIpcHandlers( captureStartedAtMs, cursorOffsetMs: nativeWindowsCursorOffsetMs, webcamFormat, + additionalWebcamFormats: nativeWindowsCaptureAdditionalWebcamTargets.map((_, i) => + readWebcamFormatAt(nativeWindowsCaptureOutput, i + 1), + ), encoderSelection, + // Logged only: menus missing from a window take on Windows before 11 + // 24H2 are a platform limit, not something to put in front of the user. + secondaryWindowsApplied: readSecondaryWindowsApplied(nativeWindowsCaptureOutput), }); const source = selectedSource || { name: "Screen" }; @@ -2919,8 +3100,7 @@ export function registerIpcHandlers( // missing `webcamFormat`: absence of the format line also means "the // line could not be parsed", which would put a red toast on a recording // whose camera is working perfectly. - const webcamUnavailable = - request.webcam.enabled && readWebcamUnavailable(nativeWindowsCaptureOutput); + const webcamUnavailable = request.webcam.enabled && unavailableWebcamIndices.has(0); // Same shape as the camera notice: the helper records the Windows // default input rather than failing, so this take is usable but is // almost certainly the wrong microphone. @@ -2938,6 +3118,35 @@ export function registerIpcHandlers( deviceName: request.webcam.deviceName, }); } + // Helper indices follow the `webcams` list: camera 1 at 0, extras after. + const startedWebcams = [ + { path: webcamPath, label: request.webcam.deviceName ?? "" }, + ...nativeWindowsCaptureAdditionalWebcamTargets, + ]; + const unavailableWebcams = request.webcam.enabled + ? labelsOfUnavailableAdditionalWebcams(startedWebcams, [...unavailableWebcamIndices]) + : []; + // The helper deletes a dropped camera's file but may fail to; a stub + // left behind is removed at stop or discard if it is still empty. + nativeWindowsCaptureDroppedWebcamPaths = request.webcam.enabled + ? startedWebcams + .filter((_, i) => unavailableWebcamIndices.has(i)) + .map((camera) => camera.path) + : []; + if (unavailableWebcams.length > 0) { + console.warn( + "[native-wgc] recording without additional cameras the helper could not open", + { + unavailableWebcams, + }, + ); + // Already reported now; the helper deleted their files, so leaving + // them in the targets would report them a second time at stop. + nativeWindowsCaptureAdditionalWebcamTargets = + nativeWindowsCaptureAdditionalWebcamTargets.filter( + (_, i) => !unavailableWebcamIndices.has(i + 1), + ); + } return { success: true, @@ -2947,6 +3156,7 @@ export function registerIpcHandlers( videoEncoderSelection: encoderSelection?.video ?? null, videoEncoderRuntime: encoderSelection?.videoEncoderRuntime ?? null, webcamUnavailable, + ...(unavailableWebcams.length > 0 ? { unavailableWebcams } : {}), microphoneDefaulted, }; } catch (error) { @@ -2988,6 +3198,8 @@ export function registerIpcHandlers( // A source from Apple's picker records through the session that holds the pick, // and needs no Screen Recording grant -- so nothing here may go near one. const pickerSession = isMacPickerSourceId(request.source.sourceId) ? macPickerSession : null; + // Normally caught by `get-selected-source` first; this is the last word (#965). + pickerSession?.forgetSelectionUnlessExcluding(macPickerExcludedWindowIds()); const pick = pickerSession?.getSelection() ?? null; if (isMacPickerSourceId(request.source.sourceId) && !pick) { selectedSource = null; @@ -3108,12 +3320,21 @@ export function registerIpcHandlers( // When the take ends without the user — the helper reported an error or // exited — this drives the renderer's own stop, the same one the tray's Stop // Recording sends: it clears the HUD and surfaces the result. - attachNativeMacCaptureOutputDrain(proc, () => { - const hudWindow = getMainWindow(); - if (hudWindow && !hudWindow.isDestroyed()) { - hudWindow.webContents.send("stop-recording-from-tray"); - } - }); + attachNativeMacCaptureOutputDrain( + proc, + () => { + const hudWindow = getMainWindow(); + if (hudWindow && !hudWindow.isDestroyed()) { + hudWindow.webContents.send("stop-recording-from-tray"); + } + }, + () => { + const hudWindow = getMainWindow(); + if (hudWindow && !hudWindow.isDestroyed()) { + hudWindow.webContents.send("native-mac-system-audio-unavailable"); + } + }, + ); await waitForNativeMacCaptureStart(proc); const captureStartedAtMs = Date.now(); @@ -3271,6 +3492,16 @@ export function registerIpcHandlers( const proc = nativeWindowsCaptureProcess; const preferredPath = nativeWindowsCaptureTargetPath; const preferredWebcamPath = nativeWindowsCaptureWebcamTargetPath; + const additionalWebcamTargets = nativeWindowsCaptureAdditionalWebcamTargets; + const camera1Label = nativeWindowsCaptureWebcamLabel; + const droppedWebcamPaths = nativeWindowsCaptureDroppedWebcamPaths; + // Start-dropped cameras ride along so a discard or a failed stop also + // removes a stub the helper could not delete (both are empty). + const allWebcamPaths = [ + preferredWebcamPath, + ...additionalWebcamTargets.map((target) => target.path), + ...droppedWebcamPaths, + ]; const recordingId = nativeWindowsCaptureRecordingId ?? Date.now(); const cursorCaptureMode = nativeWindowsCursorCaptureMode; @@ -3293,7 +3524,7 @@ export function registerIpcHandlers( if (!exited) { detachNativeWindowsCaptureOutputDrain(); } - await removeNativeWindowsCaptureOutputs(preferredPath, preferredWebcamPath); + await removeNativeWindowsCaptureOutputs(preferredPath, allWebcamPaths); return { success: true, discarded: true }; } finally { // Unconditional. Killing a wedged helper can itself throw, and @@ -3367,7 +3598,7 @@ export function registerIpcHandlers( // explain. Size-gate it anyway: throwing away a recording to tidy // up after a failed stop is the worse mistake of the two, and the // gate is the same one the salvage check above uses. - await removeNativeWindowsCaptureOutputs(preferredPath, preferredWebcamPath, { + await removeNativeWindowsCaptureOutputs(preferredPath, allWebcamPaths, { onlyIfUnusable: true, }); // The helper log goes to console/diagnostics above, not into this @@ -3403,31 +3634,70 @@ export function registerIpcHandlers( shiftPendingCursorTelemetry(nativeWindowsCursorOffsetMs); await writePendingCursorTelemetry(screenVideoPath); } - let webcamVideoPath: string | undefined; - if (preferredWebcamPath) { - try { - // Size, not just existence. A camera that opened but delivered no - // frame still gets a file created for it, and its `Finalize()` then - // fails, leaving nought bytes on disk. Admitting that file put a - // camera track in the document pointing at something no demuxer can - // read, and the preview compositor answers an unreadable camera by - // drawing the SCREEN recording inside the little camera rectangle — - // which is how a webcam that never recorded showed up as the desktop - // duplicated into its own corner (getopenscreen/openscreen#387). - const webcamStat = await fs.stat(preferredWebcamPath); - webcamVideoPath = webcamStat.size > 0 ? preferredWebcamPath : undefined; - if (!webcamVideoPath) { - console.warn("[native-wgc] the webcam file is empty; saving without a camera", { - path: preferredWebcamPath, - }); - } - } catch { - webcamVideoPath = undefined; + // Size, not just existence. A camera that opened but delivered no frame + // still gets a file created for it, and its `Finalize()` then fails, + // leaving nought bytes on disk. Admitting that file put a camera track in + // the document pointing at something no demuxer can read, and the preview + // compositor answers an unreadable camera by drawing the SCREEN recording + // inside the little camera rectangle — which is how a webcam that never + // recorded showed up as the desktop duplicated into its own corner + // (getopenscreen/openscreen#387). Every camera is judged by its own file, + // not by the helper's list at stop (see `collectStoppedWebcams`). + const requestedWebcams = [ + ...(preferredWebcamPath ? [{ path: preferredWebcamPath, label: camera1Label }] : []), + ...additionalWebcamTargets, + ]; + const webcamSizes = new Map(); + for (const camera of requestedWebcams) { + const stat = await fs.stat(camera.path).catch(() => null); + if (stat) { + webcamSizes.set(camera.path, stat.size); } } - const session: RecordingSession = webcamVideoPath - ? { screenVideoPath, webcamVideoPath, createdAt: recordingId, cursorCaptureMode } - : { screenVideoPath, createdAt: recordingId, cursorCaptureMode }; + const stoppedWebcams = collectStoppedWebcams({ + camera1Enabled: Boolean(preferredWebcamPath), + requested: requestedWebcams, + sizes: webcamSizes, + }); + const webcamVideoPath = stoppedWebcams.camera1; + const additionalWebcams = stoppedWebcams.additional; + if (preferredWebcamPath && !webcamVideoPath && webcamSizes.has(preferredWebcamPath)) { + console.warn("[native-wgc] the webcam file is empty; saving without a camera", { + path: preferredWebcamPath, + }); + } + if (stoppedWebcams.dropped.length > 0) { + console.warn("[native-wgc] additional cameras produced nothing usable", { + dropped: stoppedWebcams.dropped, + helperWebcamPaths: readStoppedWebcamPaths(nativeWindowsCaptureOutput), + }); + } + // A camera the helper disabled mid-take keeps its partial file (it is + // in the take) but is named, so the user knows why it ends early. Only + // a helper that sends `webcamPaths` can tell; otherwise nothing is said. + const webcamsStoppedEarly = labelsOfWebcamsStoppedEarly({ + requested: requestedWebcams, + sizes: webcamSizes, + helperWebcamPaths: readReportedWebcamPaths(nativeWindowsCaptureOutput), + }); + if (webcamsStoppedEarly.length > 0) { + console.warn("[native-wgc] cameras stopped before the end of the take", { + webcamsStoppedEarly, + }); + } + await removeEmptyNativeWindowsWebcamFiles(droppedWebcamPaths); + // Generated by the start handler, never taken from the renderer or the + // helper; approved like the session's other media. + for (const extra of additionalWebcams) { + approveFilePath(extra.path); + } + const session: RecordingSession = { + screenVideoPath, + ...(webcamVideoPath ? { webcamVideoPath } : {}), + ...(additionalWebcams.length > 0 ? { additionalWebcams } : {}), + createdAt: recordingId, + cursorCaptureMode, + }; setCurrentRecordingSessionState(session); currentProjectPath = null; @@ -3436,7 +3706,11 @@ export function registerIpcHandlers( `${path.parse(screenVideoPath).name}${RECORDING_SESSION_SUFFIX}`, ); await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); - await registerRecordingMediaLinks(screenVideoPath, { webcamVideoPath, cursorCaptureMode }); + await registerRecordingMediaLinks(screenVideoPath, { + webcamVideoPath, + ...(additionalWebcams.length > 0 ? { additionalWebcams } : {}), + cursorCaptureMode, + }); return { success: true, @@ -3451,6 +3725,8 @@ export function registerIpcHandlers( // unreported, the user would find out in the editor — which is exactly // the silence this change exists to end. webcamDropped: Boolean(preferredWebcamPath) && !webcamVideoPath, + ...(stoppedWebcams.dropped.length > 0 ? { droppedWebcams: stoppedWebcams.dropped } : {}), + ...(webcamsStoppedEarly.length > 0 ? { webcamsStoppedEarly } : {}), message: recovered ? "Native Windows recording recovered from a failed stop" : "Native Windows recording session stored successfully", @@ -3919,7 +4195,7 @@ export function registerIpcHandlers( const files = await fs.readdir(RECORDINGS_DIR); const videoFiles = files.filter( - (file) => file.endsWith(".webm") && !file.endsWith("-webcam.webm"), + (file) => file.endsWith(".webm") && !isWebcamSidecarFile(file), ); if (videoFiles.length === 0) { @@ -4677,21 +4953,16 @@ export function registerIpcHandlers( // `addAsset` in the new editor's project store. ipcMain.handle( "find-recording-camera", - async ( - _event, - videoPath: string, - ): Promise<{ - success: boolean; - webcamVideoPath?: string; - offsetMs?: number; - error?: string; - }> => { + async (_event, videoPath: string): Promise => { try { const normalized = normalizeVideoSourcePath(videoPath); if (!normalized || !isPathAllowed(normalized)) { return { success: false, error: "Video path has not been approved" }; } const resolution = await resolveMediaLinksForVideo(normalized); + // Additional cameras are only returned alongside camera 1. That relies + // on R6 (extras are recorded only while camera 1 is on), so extras + // without camera 1 means camera 1's file came out empty. if (!resolution.webcamVideoPath) { return { success: false, error: "No camera attached to this recording" }; } @@ -4699,6 +4970,9 @@ export function registerIpcHandlers( success: true, webcamVideoPath: resolution.webcamVideoPath, offsetMs: resolution.webcamOffsetMs ?? 0, + ...(resolution.additionalWebcams?.length + ? { additionalWebcams: resolution.additionalWebcams } + : {}), }; } catch (err) { return { @@ -4822,6 +5096,21 @@ export function registerIpcHandlers( return aiEditionLlmConfigInstance; }; + // The local MCP server offers the agent's tools to MCP clients the user runs + // (Claude Code, Codex…). Built here because this is where the agent's own + // dependencies live, but NOT started here: the headless CLI shares this + // function and must never bind the port a running app is listening on. + // `main.ts` starts it. Its writes have their own switch, off by default and + // read on every call — separate from the in-app agent's "Project edits", so + // turning the server on grants a client read access and nothing more. + const mcpSettings = new McpSettingsStore(app.getPath("userData"), safeStorage); + const mcpController = new McpController(mcpSettings, { + host: new EditorDocumentHost(ipcMain), + editsAllowed: () => mcpSettings.getSettings().allowEdits, + cursor: agentCursorTelemetryReader, + version: app.getVersion(), + }); + registerNativeBridgeHandlers({ getPlatform: () => process.platform, getCurrentProjectPath: () => currentProjectPath, @@ -4877,5 +5166,8 @@ export function registerIpcHandlers( renameAiEditionChatSession: (projectId, sessionId, title) => renameSession(projectId, sessionId, title), deleteAiEditionChatSession: (projectId, sessionId) => deleteSession(projectId, sessionId), + getMcpController: () => mcpController, }); + + return { mcpController }; } diff --git a/electron/ipc/nativeBridge.mcp.test.ts b/electron/ipc/nativeBridge.mcp.test.ts new file mode 100644 index 000000000..59b477c93 --- /dev/null +++ b/electron/ipc/nativeBridge.mcp.test.ts @@ -0,0 +1,82 @@ +// The MCP settings actions take renderer input, so the bridge checks its types +// before anything reaches the settings store: a bad value is an INVALID_REQUEST, +// not an INTERNAL_ERROR thrown further in. + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { NativeBridgeResponse } from "../../src/native/contracts"; +import { type NativeBridgeContext, registerNativeBridgeHandlers } from "./nativeBridge"; + +const electron = vi.hoisted(() => ({ handle: vi.fn() })); +vi.mock("electron", () => ({ + app: { getAppPath: () => "", isPackaged: false }, + ipcMain: { handle: electron.handle, removeHandler: vi.fn() }, + shell: {}, +})); +vi.mock("../native-bridge/services/compositorViewService", () => ({ + CompositorViewService: class {}, +})); + +const service = vi.hoisted(() => ({ + mcpSetEnabled: vi.fn(async () => ({ ok: "enabled" })), + mcpSetPort: vi.fn(async () => ({ ok: "port" })), + mcpSetAllowEdits: vi.fn(async () => ({ ok: "allowEdits" })), +})); +vi.mock("../native-bridge/services/aiEditionService", () => ({ + AiEditionService: class { + mcpSetEnabled = service.mcpSetEnabled; + mcpSetPort = service.mcpSetPort; + mcpSetAllowEdits = service.mcpSetAllowEdits; + }, +})); + +let invoke: (action: string, payload: unknown) => Promise; + +beforeEach(() => { + electron.handle.mockClear(); + for (const fn of Object.values(service)) fn.mockClear(); + registerNativeBridgeHandlers({ + getPlatform: () => "linux", + getAiEditionDocuments: () => ({}), + getAiEditionLlmConfig: () => ({}), + } as unknown as NativeBridgeContext); + const handler = electron.handle.mock.calls[0]?.[1] as ( + event: unknown, + request: unknown, + ) => Promise; + invoke = (action, payload) => + handler({ sender: {} }, { domain: "aiEdition", action, payload, requestId: "r" }); +}); + +function errorCode(response: NativeBridgeResponse): string | undefined { + return response.ok ? undefined : response.error?.code; +} + +describe("native bridge MCP settings", () => { + it.each([80, 70000, 4.5, "47821", null])("rejects port %s as a bad request", async (port) => { + expect(errorCode(await invoke("mcp.setPort", { port }))).toBe("INVALID_REQUEST"); + expect(service.mcpSetPort).not.toHaveBeenCalled(); + }); + + it("passes a valid port on", async () => { + const response = await invoke("mcp.setPort", { port: 47821 }); + expect(response.ok).toBe(true); + expect(service.mcpSetPort).toHaveBeenCalledWith(47821); + }); + + it.each(["true", 1, undefined])("rejects enabled=%s as a bad request", async (enabled) => { + expect(errorCode(await invoke("mcp.setEnabled", { enabled }))).toBe("INVALID_REQUEST"); + expect(service.mcpSetEnabled).not.toHaveBeenCalled(); + }); + + it.each(["yes", 0, undefined])("rejects allowEdits=%s as a bad request", async (allowEdits) => { + expect(errorCode(await invoke("mcp.setAllowEdits", { allowEdits }))).toBe("INVALID_REQUEST"); + expect(service.mcpSetAllowEdits).not.toHaveBeenCalled(); + }); + + it("passes valid flags on", async () => { + await invoke("mcp.setEnabled", { enabled: true }); + await invoke("mcp.setAllowEdits", { allowEdits: false }); + expect(service.mcpSetEnabled).toHaveBeenCalledWith(true); + expect(service.mcpSetAllowEdits).toHaveBeenCalledWith(false); + }); +}); diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index e25ea048b..82e7c15a1 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -13,6 +13,7 @@ import { import type { ChatEventSink } from "../ai-edition/chat-service"; import type { DocumentService } from "../ai-edition/document-service"; import { StylePresetError, type StylePresetService } from "../ai-edition/style-preset-service"; +import { isValidMcpPort } from "../mcp/mcp-settings-store"; import { type CursorTelemetryLoadResult, TelemetryCursorAdapter, @@ -62,6 +63,8 @@ export interface NativeBridgeContext { /** The one shared style preset service — it serialises writes per instance. */ getStylePresets: () => StylePresetService; getAiEditionLlmConfig: () => import("../ai-edition/llm-config-store").LlmConfigStore; + /** The local MCP server's controller. Absent in the headless CLI. */ + getMcpController?: () => import("../mcp/mcp-controller").McpController; runAiEditionChat: ( projectId: string, sessionId: string, @@ -234,6 +237,7 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { // Passed uncalled on purpose — invoking it here would build the store (and // hit the macOS Keychain) while wiring the bridge at startup. llmConfig: context.getAiEditionLlmConfig, + mcp: context.getMcpController?.(), runChat: context.runAiEditionChat, undoLastToolBatch: context.undoAiEditionToolBatch, rewindToMessage: context.rewindToMessage, @@ -380,18 +384,23 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { compositorViewService.setRect(request.payload.id, request.payload.rect); return createSuccessResponse(requestId, { ok: true }); case "readFrame": { - // The renderer polls this every rAF tick (~30fps). It passes the - // generation it last painted as `sinceGen`; native returns `null` when - // nothing newer exists (idle path — no buffer copy). On a new frame it - // returns `{ gen, width, height, data }`. The response wrapper does NOT - // JSON-stringify — `ipcMain.handle` round-trips via structured clone, - // which preserves the nested `Buffer` in `.data` as binary. - const frame = compositorViewService.readFrame( + // The renderer polls this every rAF tick. It passes the generation it + // last painted as `sinceGen`; native returns `null` when nothing newer + // exists (idle path — no buffer copy). On a new frame it returns + // `{ gen, width, height, data }`, or — for a view on shared textures — + // sends the texture to the asking frame and returns its receipt. The + // response wrapper does NOT JSON-stringify — `ipcMain.handle` round-trips + // via structured clone, which preserves the nested `Buffer` as binary. + const frame = await compositorViewService.readFrame( request.payload.id, request.payload.sinceGen, + event.senderFrame, ); return createSuccessResponse(requestId, frame); } + case "stopSharedFrames": + compositorViewService.stopSharedFrames(request.payload.id); + return createSuccessResponse(requestId, { ok: true }); case "setParam": compositorViewService.setParam( request.payload.id, @@ -416,6 +425,7 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { request.payload.webcamOffsetSec, request.payload.clipIndex, request.payload.sourceTimeSec, + request.payload.additionalCameras ?? [], ); return createSuccessResponse(requestId, { ok: true }); case "destroyView": @@ -573,6 +583,48 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { requestId, await aiEditionService.llmListProviderModels(request.payload.providerId), ); + case "mcp.getStatus": + return createSuccessResponse(requestId, await aiEditionService.mcpGetStatus()); + // Renderer input: check the types here, so a bad value is reported as + // the bad request it is rather than failing later as an internal error. + case "mcp.setEnabled": + if (typeof request.payload?.enabled !== "boolean") { + return createErrorResponse( + requestId, + "INVALID_REQUEST", + "Invalid MCP enabled flag.", + ); + } + return createSuccessResponse( + requestId, + await aiEditionService.mcpSetEnabled(request.payload.enabled), + ); + case "mcp.setPort": + if (!isValidMcpPort(request.payload?.port)) { + return createErrorResponse( + requestId, + "INVALID_REQUEST", + "MCP port must be a whole number between 1024 and 65535.", + ); + } + return createSuccessResponse( + requestId, + await aiEditionService.mcpSetPort(request.payload.port), + ); + case "mcp.setAllowEdits": + if (typeof request.payload?.allowEdits !== "boolean") { + return createErrorResponse( + requestId, + "INVALID_REQUEST", + "Invalid MCP edit permission.", + ); + } + return createSuccessResponse( + requestId, + await aiEditionService.mcpSetAllowEdits(request.payload.allowEdits), + ); + case "mcp.regenerateToken": + return createSuccessResponse(requestId, await aiEditionService.mcpRegenerateToken()); case "chat.run": { const sessionId = request.payload.sessionId; const sink = buildChatEventSink(event.sender, sessionId); diff --git a/electron/ipc/nativeMacMidCaptureErrorWatch.test.ts b/electron/ipc/nativeMacMidCaptureErrorWatch.test.ts index c6958cd4b..6eaec1ef3 100644 --- a/electron/ipc/nativeMacMidCaptureErrorWatch.test.ts +++ b/electron/ipc/nativeMacMidCaptureErrorWatch.test.ts @@ -6,7 +6,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("fires on an error raised after recording started", () => { const onError = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onError); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onError, vi.fn()); watch({ event: "ready" }); watch({ event: "recording-started" }); @@ -17,7 +17,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("leaves an error raised before recording started to the start wait", () => { const onError = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onError); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onError, vi.fn()); watch(error); @@ -26,7 +26,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("ignores events that are not errors", () => { const onError = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onError); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onError, vi.fn()); watch({ event: "recording-started" }); watch({ event: "warning", code: "stop-capture-failed" }); @@ -38,7 +38,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("ignores a helper that is no longer the current process", () => { const onError = vi.fn(); let current = true; - const watch = createNativeMacMidCaptureErrorWatch(() => current, onError); + const watch = createNativeMacMidCaptureErrorWatch(() => current, onError, vi.fn()); watch({ event: "recording-started" }); current = false; @@ -50,7 +50,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { /** Killed or crashed: no error line ever comes, only the process closing. */ it("fires when the helper exits in the middle of a take", () => { const onTakeEnded = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onTakeEnded); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onTakeEnded, vi.fn()); watch({ event: "recording-started" }); watch.exited(); @@ -60,7 +60,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("leaves an exit before recording started to the start wait", () => { const onTakeEnded = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onTakeEnded); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onTakeEnded, vi.fn()); watch.exited(); @@ -71,7 +71,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("ignores the exit of a take that is already being stopped", () => { const onTakeEnded = vi.fn(); let live = true; - const watch = createNativeMacMidCaptureErrorWatch(() => live, onTakeEnded); + const watch = createNativeMacMidCaptureErrorWatch(() => live, onTakeEnded, vi.fn()); watch({ event: "recording-started" }); live = false; @@ -79,4 +79,34 @@ describe("createNativeMacMidCaptureErrorWatch", () => { expect(onTakeEnded).not.toHaveBeenCalled(); }); + + it("surfaces a system-audio warning once without ending the take", () => { + const onTakeEnded = vi.fn(); + const onSystemAudioUnavailable = vi.fn(); + const watch = createNativeMacMidCaptureErrorWatch( + () => true, + onTakeEnded, + onSystemAudioUnavailable, + ); + + watch({ event: "recording-started" }); + watch({ event: "warning", code: "system-audio-unavailable" }); + watch({ event: "warning", code: "system-audio-unavailable" }); + + expect(onSystemAudioUnavailable).toHaveBeenCalledOnce(); + expect(onTakeEnded).not.toHaveBeenCalled(); + }); + + it("does not surface the system-audio warning for a take that is no longer live", () => { + const onSystemAudioUnavailable = vi.fn(); + const watch = createNativeMacMidCaptureErrorWatch( + () => false, + vi.fn(), + onSystemAudioUnavailable, + ); + + watch({ event: "warning", code: "system-audio-unavailable" }); + + expect(onSystemAudioUnavailable).not.toHaveBeenCalled(); + }); }); diff --git a/electron/ipc/nativeMacMidCaptureErrorWatch.ts b/electron/ipc/nativeMacMidCaptureErrorWatch.ts index 6c786d3f3..48f862888 100644 --- a/electron/ipc/nativeMacMidCaptureErrorWatch.ts +++ b/electron/ipc/nativeMacMidCaptureErrorWatch.ts @@ -15,12 +15,23 @@ export function createNativeMacMidCaptureErrorWatch( isLiveTake: () => boolean, onTakeEnded: () => void, + onSystemAudioUnavailable: () => void, ) { let recordingStarted = false; + let reportedSystemAudioUnavailable = false; const watch = (event: Record) => { if (event.event === "recording-started") { recordingStarted = true; } + if ( + event.event === "warning" && + event.code === "system-audio-unavailable" && + !reportedSystemAudioUnavailable && + isLiveTake() + ) { + reportedSystemAudioUnavailable = true; + onSystemAudioUnavailable(); + } if (event.event === "error" && recordingStarted && isLiveTake()) { onTakeEnded(); } diff --git a/electron/ipc/recordingPrefs.test.ts b/electron/ipc/recordingPrefs.test.ts index fe8cdce9d..7d2d056a1 100644 --- a/electron/ipc/recordingPrefs.test.ts +++ b/electron/ipc/recordingPrefs.test.ts @@ -19,6 +19,8 @@ const defaults: RecordingPrefs = { camEnabled: false, camDeviceId: null, camDeviceName: null, + camAdditionalDevices: [], + camQuality: "2160p", systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, diff --git a/electron/lingeringOverlay.test.ts b/electron/lingeringOverlay.test.ts new file mode 100644 index 000000000..37700033f --- /dev/null +++ b/electron/lingeringOverlay.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; +import { isOnlyLingeringOverlay } from "./lingeringOverlay"; + +const window = (destroyed = false) => ({ isDestroyed: () => destroyed }); + +describe("isOnlyLingeringOverlay", () => { + it("is true when the hidden overlay is the last window open", () => { + const overlay = window(); + expect(isOnlyLingeringOverlay([overlay], overlay)).toBe(true); + expect(isOnlyLingeringOverlay([overlay, window(true)], overlay)).toBe(true); + }); + + it("is false while the HUD or the editor is still open", () => { + const overlay = window(); + expect(isOnlyLingeringOverlay([window(), overlay], overlay)).toBe(false); + }); + + it("is false without an overlay to close", () => { + expect(isOnlyLingeringOverlay([], null)).toBe(false); + const gone = window(true); + expect(isOnlyLingeringOverlay([gone], gone)).toBe(false); + }); +}); diff --git a/electron/lingeringOverlay.ts b/electron/lingeringOverlay.ts new file mode 100644 index 000000000..35d64e64d --- /dev/null +++ b/electron/lingeringOverlay.ts @@ -0,0 +1,18 @@ +// The countdown overlay is created on the first take and only hidden between takes, so it is +// ready to paint the next countdown at once. A hidden window still counts for Electron, so +// once the HUD and the editor were gone `window-all-closed` never fired: OpenScreen stayed +// up with no window and only the tray icon left (issue #961). `main.ts` closes the overlay +// when it is the last window standing. + +/** The slice of `BrowserWindow` this needs, so the rule can be tested without Electron. */ +export type LingeringWindow = { isDestroyed(): boolean }; + +/** True when `overlay` is the only window still open, so it alone holds the app up. */ +export function isOnlyLingeringOverlay( + open: readonly LingeringWindow[], + overlay: LingeringWindow | null, +): boolean { + if (!overlay || overlay.isDestroyed()) return false; + const alive = open.filter((window) => !window.isDestroyed()); + return alive.length === 1 && alive[0] === overlay; +} diff --git a/electron/main.ts b/electron/main.ts index de3e3bbd7..e525aa064 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -58,6 +58,7 @@ import { getSelectedDesktopSource, registerIpcHandlers, } from "./ipc/handlers"; +import { isOnlyLingeringOverlay } from "./lingeringOverlay"; import { installMainProcessErrorGuards } from "./main-process-errors"; import { showMessageBoxOver } from "./messageBox"; import { @@ -65,6 +66,7 @@ import { showPermissionsWindow, showPermissionsWindowIfNeeded, } from "./permissions"; +import { setDisplaySleepBlocked } from "./recording/displaySleepBlocker"; import { offersStarPrompt, REPO_URL, storeReviewUrl } from "./star-prompt"; import { registerSttIpc, shutdownStt } from "./stt"; import { checkLatestRelease } from "./update-checker"; @@ -1088,6 +1090,17 @@ if (!cliCommand) { app.on("window-all-closed", () => { app.quit(); }); + // The countdown overlay hides between takes instead of closing, so it would keep + // `window-all-closed` from ever firing (#961). Close it with the last other window. Every + // HUD/editor switch opens the next window before the previous one's `closed` arrives, so + // this only fires when the user really closed the last one. + app.on("browser-window-created", (_, win) => { + win.once("closed", () => { + if (isOnlyLingeringOverlay(BrowserWindow.getAllWindows(), countdownOverlayWindow)) { + countdownOverlayWindow?.close(); + } + }); + }); } app.on("activate", () => { @@ -1360,7 +1373,7 @@ appReady?.then(async () => { showMainWindow(); } - registerIpcHandlers( + const { mcpController } = registerIpcHandlers( createEditorWindowWrapper, createSourceSelectorWindowWrapper, createCountdownOverlayWindowWrapper, @@ -1372,6 +1385,7 @@ appReady?.then(async () => { (recording: boolean, sourceName: string) => { selectedSourceName = sourceName; isRecording = recording; + setDisplaySleepBlocked(recording); if (!tray) createTray(); updateTrayMenu(recording); // `canOfferUpdateCheck()` now answers "not mid-take" too, and the app/Help menus are @@ -1404,6 +1418,9 @@ appReady?.then(async () => { } createWindow(); + // Off unless the user turned it on in Settings → AI. Started here rather than + // in registerIpcHandlers so neither the headless CLI nor a bench run binds it. + void mcpController.startIfEnabled(); void showPermissionsWindowIfNeeded().catch((error) => console.warn("[permissions] could not read the permissions at launch:", error), ); diff --git a/electron/mcp/editor-document-host.test.ts b/electron/mcp/editor-document-host.test.ts new file mode 100644 index 000000000..5b11cd19a --- /dev/null +++ b/electron/mcp/editor-document-host.test.ts @@ -0,0 +1,126 @@ +import { EventEmitter } from "node:events"; +import type { IpcMain, WebContents } from "electron"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createEmptyDocument } from "../../src/lib/ai-edition/schema"; +import { + AI_EDITION_MCP_HOST_CHANNEL, + AI_EDITION_MCP_REQUEST_CHANNEL, + AI_EDITION_MCP_RESPONSE_CHANNEL, + type AiEditionMcpHostRequest, +} from "../../src/native/contracts"; +import { EditorDocumentHost } from "./editor-document-host"; + +/** A webContents that records what main sends it and can be destroyed. */ +class FakeWebContents extends EventEmitter { + sent: AiEditionMcpHostRequest[] = []; + destroyed = false; + send(channel: string, payload: AiEditionMcpHostRequest) { + if (channel === AI_EDITION_MCP_REQUEST_CHANNEL) this.sent.push(payload); + } + isDestroyed() { + return this.destroyed; + } + destroy() { + this.destroyed = true; + this.emit("destroyed"); + } +} + +function setup() { + const ipc = new EventEmitter(); + const host = new EditorDocumentHost(ipc as unknown as IpcMain); + const fromRenderer = (sender: FakeWebContents, channel: string, payload: unknown) => + ipc.emit(channel, { sender: sender as unknown as WebContents }, payload); + return { host, fromRenderer }; +} + +/** Answers the next request the editor receives. */ +async function answer( + editor: FakeWebContents, + fromRenderer: ReturnType["fromRenderer"], + result: unknown, + sender = editor, +) { + await vi.waitFor(() => expect(editor.sent.length).toBeGreaterThan(0)); + const request = editor.sent.shift(); + fromRenderer(sender, AI_EDITION_MCP_RESPONSE_CHANNEL, { requestId: request?.requestId, result }); +} + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("EditorDocumentHost", () => { + it("answers null while no editor has registered", async () => { + const { host } = setup(); + await expect(host.snapshot()).resolves.toBeNull(); + }); + + it("asks the registered editor for its snapshot", async () => { + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const pending = host.snapshot(); + await answer(editor, fromRenderer, { document: { id: "d" }, revision: 3 }); + await expect(pending).resolves.toEqual({ document: { id: "d" }, revision: 3 }); + }); + + it("treats a snapshot without a numeric revision as no snapshot", async () => { + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + // A missing revision would reach the apply guard as `undefined` and skip it. + for (const revision of [undefined, "3", Number.NaN]) { + const pending = host.snapshot(); + await answer(editor, fromRenderer, { document: { id: "d" }, revision }); + await expect(pending).resolves.toBeNull(); + } + }); + + it("passes the apply verdict through", async () => { + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const document = createEmptyDocument({ projectId: "p", title: "t" }); + const pending = host.apply(document, 7); + await vi.waitFor(() => expect(editor.sent).toHaveLength(1)); + expect(editor.sent[0]).toMatchObject({ op: "apply", expectedRevision: 7 }); + await answer(editor, fromRenderer, "conflict"); + await expect(pending).resolves.toBe("conflict"); + }); + + it("ignores a reply from any other window", async () => { + vi.useFakeTimers(); + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const pending = host.snapshot(); + await answer(editor, fromRenderer, { document: {}, revision: 1 }, new FakeWebContents()); + await vi.advanceTimersByTimeAsync(30_000); + await expect(pending).resolves.toBeNull(); + }); + + it("forgets an editor that unregisters or is destroyed", async () => { + const { host, fromRenderer } = setup(); + const first = new FakeWebContents(); + fromRenderer(first, AI_EDITION_MCP_HOST_CHANNEL, true); + fromRenderer(first, AI_EDITION_MCP_HOST_CHANNEL, false); + await expect(host.snapshot()).resolves.toBeNull(); + + const second = new FakeWebContents(); + fromRenderer(second, AI_EDITION_MCP_HOST_CHANNEL, true); + second.destroy(); + await expect(host.snapshot()).resolves.toBeNull(); + expect(second.sent).toHaveLength(0); + }); + + it("reports an unanswered apply as a timeout, not as a failure", async () => { + vi.useFakeTimers(); + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const pending = host.apply(createEmptyDocument({ projectId: "p", title: "t" }), 1); + await vi.advanceTimersByTimeAsync(30_000); + await expect(pending).resolves.toBe("timeout"); + }); +}); diff --git a/electron/mcp/editor-document-host.ts b/electron/mcp/editor-document-host.ts new file mode 100644 index 000000000..55d6d6156 --- /dev/null +++ b/electron/mcp/editor-document-host.ts @@ -0,0 +1,107 @@ +// The MCP server's door onto the live document, which the editor window owns. +// +// The open project lives in the renderer's store (with the revision that guards +// agent writes), not in the main process, so each MCP call asks the editor for +// a snapshot and hands the result back to the editor's own revision-guarded +// apply — the same `applyAgentDocumentIfCurrent` an in-app chat turn uses. +// +// The editor announces itself on AI_EDITION_MCP_HOST_CHANNEL when it starts +// answering, and again with `false` when it stops; a destroyed window counts as +// stopped. Only that one webContents is ever asked, and only its replies count. + +import { randomUUID } from "node:crypto"; +import type { IpcMain, IpcMainEvent, WebContents } from "electron"; +import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; +import { + AI_EDITION_MCP_HOST_CHANNEL, + AI_EDITION_MCP_REQUEST_CHANNEL, + AI_EDITION_MCP_RESPONSE_CHANNEL, + type AiEditionMcpHostRequest, + type AiEditionMcpHostResponse, + type AiEditionMcpHostSnapshot, +} from "../../src/native/contracts"; +import type { McpApplyResult, McpDocumentHost } from "./openscreen-mcp-server"; + +// Long enough for a save of a large project to land, short enough that a hung +// renderer does not hold the client's call open indefinitely. +const REQUEST_TIMEOUT_MS = 30_000; + +const TIMED_OUT = Symbol("timed-out"); + +interface Pending { + resolve: (value: unknown) => void; + timer: ReturnType; +} + +type RequestBody = + | { op: "snapshot" } + | { op: "apply"; document: unknown; expectedRevision: number }; + +export class EditorDocumentHost implements McpDocumentHost { + private editor: WebContents | null = null; + private readonly pending = new Map(); + + constructor(ipcMain: IpcMain) { + ipcMain.on(AI_EDITION_MCP_HOST_CHANNEL, (event: IpcMainEvent, active: unknown) => { + if (active === true) { + this.editor = event.sender; + event.sender.once("destroyed", () => { + if (this.editor === event.sender) this.editor = null; + }); + } else if (this.editor === event.sender) { + this.editor = null; + } + }); + ipcMain.on(AI_EDITION_MCP_RESPONSE_CHANNEL, (event: IpcMainEvent, message: unknown) => { + if (event.sender !== this.editor) return; + const response = message as AiEditionMcpHostResponse | null; + if (!response || typeof response.requestId !== "string") return; + const pending = this.pending.get(response.requestId); + if (!pending) return; + clearTimeout(pending.timer); + this.pending.delete(response.requestId); + pending.resolve(response.result); + }); + } + + private request(body: RequestBody): Promise { + const editor = this.editor; + if (!editor || editor.isDestroyed()) return Promise.resolve(undefined); + const requestId = randomUUID(); + return new Promise((resolve) => { + const timer = setTimeout(() => { + this.pending.delete(requestId); + resolve(TIMED_OUT); + }, REQUEST_TIMEOUT_MS); + this.pending.set(requestId, { resolve, timer }); + try { + editor.send(AI_EDITION_MCP_REQUEST_CHANNEL, { + requestId, + ...body, + } satisfies AiEditionMcpHostRequest); + } catch { + clearTimeout(timer); + this.pending.delete(requestId); + resolve(undefined); + } + }); + } + + async snapshot(): Promise { + const result = await this.request({ op: "snapshot" }); + if (!result || result === TIMED_OUT || typeof result !== "object") return null; + // The revision is the apply guard: one that is missing or not a number would + // reach `applyAgentDocumentIfCurrent` as `undefined`, which skips the + // stale-edit check entirely. Treat such a reply as no snapshot at all. + const snapshot = result as Partial; + if (typeof snapshot.revision !== "number" || !Number.isFinite(snapshot.revision)) return null; + return snapshot as AiEditionMcpHostSnapshot; + } + + async apply(document: AxcutDocument, expectedRevision: number): Promise { + const result = await this.request({ op: "apply", document, expectedRevision }); + if (result === TIMED_OUT) return "timeout"; + if (typeof result !== "string") return "no-editor"; + return result as McpApplyResult; + } +} diff --git a/electron/mcp/mcp-controller.test.ts b/electron/mcp/mcp-controller.test.ts new file mode 100644 index 000000000..3a23bae09 --- /dev/null +++ b/electron/mcp/mcp-controller.test.ts @@ -0,0 +1,165 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { createServer, type Server } from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { McpController } from "./mcp-controller"; +import { DEFAULT_MCP_PORT, McpSettingsStore, type McpTokenCrypto } from "./mcp-settings-store"; +import type { McpToolDeps } from "./openscreen-mcp-server"; + +// Reversible and visibly not plaintext, so a test can tell the token was encrypted. +const fakeCrypto: McpTokenCrypto = { + isEncryptionAvailable: () => true, + encryptString: (plain) => Buffer.from(`enc:${plain}`), + decryptString: (encrypted) => encrypted.toString().replace(/^enc:/, ""), +}; + +const deps: McpToolDeps = { + host: { snapshot: async () => null, apply: async () => "no-editor" }, + editsAllowed: () => true, + version: "0.0.0", +}; + +let dir: string; +let controller: McpController | null = null; +let blocker: Server | null = null; + +beforeEach(() => { + dir = mkdtempSync(path.join(os.tmpdir(), "openscreen-mcp-")); +}); + +afterEach(async () => { + await controller?.stop(); + controller = null; + await new Promise((resolve) => (blocker ? blocker.close(() => resolve()) : resolve())); + blocker = null; + rmSync(dir, { recursive: true, force: true }); +}); + +/** A free port, found by binding one and letting it go. */ +async function freePort(): Promise { + const server = createServer(); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as { port: number }; + await new Promise((resolve) => server.close(() => resolve())); + return port; +} + +async function initializeStatus(url: string, token: string) { + const response = await fetch(url, { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + Accept: "application/json, text/event-stream", + }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "t", version: "0" }, + }, + }), + }); + return response.status; +} + +describe("McpSettingsStore", () => { + it("is off by default, on the default port", () => { + const store = new McpSettingsStore(dir, fakeCrypto); + expect(store.getSettings()).toEqual({ + enabled: false, + port: DEFAULT_MCP_PORT, + allowEdits: false, + }); + }); + + it("keeps MCP edits off unless they were turned on explicitly", async () => { + expect(new McpSettingsStore(dir, fakeCrypto).getSettings().allowEdits).toBe(false); + await new McpSettingsStore(dir, fakeCrypto).setSettings({ enabled: true }); + expect(new McpSettingsStore(dir, fakeCrypto).getSettings().allowEdits).toBe(false); + await new McpSettingsStore(dir, fakeCrypto).setSettings({ allowEdits: true }); + expect(new McpSettingsStore(dir, fakeCrypto).getSettings().allowEdits).toBe(true); + }); + + it("keeps the token encrypted on disk and stable across instances", async () => { + const token = await new McpSettingsStore(dir, fakeCrypto).getToken(); + expect(readFileSync(path.join(dir, "mcp-token.enc"), "utf8")).toBe(`enc:${token}`); + expect(await new McpSettingsStore(dir, fakeCrypto).getToken()).toBe(token); + }); + + it("refuses a port outside the unprivileged range", async () => { + const store = new McpSettingsStore(dir, fakeCrypto); + await expect(store.setSettings({ port: 80 })).rejects.toThrow(/1024/); + }); + + it("will not mint a token without OS encryption", async () => { + const store = new McpSettingsStore(dir, { ...fakeCrypto, isEncryptionAvailable: () => false }); + await expect(store.getToken()).rejects.toThrow(/safeStorage/); + }); +}); + +describe("McpController", () => { + it("does not listen until enabled", async () => { + controller = new McpController(new McpSettingsStore(dir, fakeCrypto), deps); + await controller.startIfEnabled(); + const status = await controller.getStatus(); + expect(status.running).toBe(false); + expect(status.token).toBeNull(); + }); + + it("starts on enable and stops on disable", async () => { + const store = new McpSettingsStore(dir, fakeCrypto); + await store.setSettings({ port: await freePort() }); + controller = new McpController(store, deps); + + const on = await controller.setEnabled(true); + expect(on.running).toBe(true); + expect(await initializeStatus(on.url, on.token ?? "")).toBe(200); + + const off = await controller.setEnabled(false); + expect(off.running).toBe(false); + await expect(initializeStatus(on.url, on.token ?? "")).rejects.toThrow(); + }); + + it("explains a port that is already taken", async () => { + const port = await freePort(); + blocker = createServer(); + await new Promise((resolve) => blocker?.listen(port, "127.0.0.1", resolve)); + const store = new McpSettingsStore(dir, fakeCrypto); + await store.setSettings({ port }); + controller = new McpController(store, deps); + + const status = await controller.setEnabled(true); + expect(status.running).toBe(false); + expect(status.error).toContain(String(port)); + }); + + it("saves the edit permission without restarting the server", async () => { + const store = new McpSettingsStore(dir, fakeCrypto); + await store.setSettings({ port: await freePort() }); + controller = new McpController(store, deps); + const on = await controller.setEnabled(true); + expect(on.allowEdits).toBe(false); + + const allowed = await controller.setAllowEdits(true); + expect(allowed.allowEdits).toBe(true); + expect(allowed.token).toBe(on.token); + expect(await initializeStatus(allowed.url, allowed.token ?? "")).toBe(200); + }); + + it("locks out the old token once it is regenerated", async () => { + const store = new McpSettingsStore(dir, fakeCrypto); + await store.setSettings({ port: await freePort() }); + controller = new McpController(store, deps); + const before = await controller.setEnabled(true); + const after = await controller.regenerateToken(); + + expect(after.token).not.toBe(before.token); + expect(await initializeStatus(after.url, before.token ?? "")).toBe(401); + expect(await initializeStatus(after.url, after.token ?? "")).toBe(200); + }); +}); diff --git a/electron/mcp/mcp-controller.ts b/electron/mcp/mcp-controller.ts new file mode 100644 index 000000000..49eb1e538 --- /dev/null +++ b/electron/mcp/mcp-controller.ts @@ -0,0 +1,119 @@ +// Owns the MCP server's lifecycle: start it when enabled, restart it when its +// port or token changes, stop it when disabled, and report status to settings. +// +// Off by default. While it is off nothing listens and the token is never read, +// so a user who never turns it on never meets a Keychain prompt for it. + +import type { AiEditionMcpStatus } from "../../src/native/contracts"; +import type { McpSettingsStore } from "./mcp-settings-store"; +import { + MCP_ENDPOINT_PATH, + type McpToolDeps, + type RunningMcpServer, + startMcpHttpServer, +} from "./openscreen-mcp-server"; + +export class McpController { + private running: RunningMcpServer | null = null; + private error: string | null = null; + /** Serialises start/stop so a quick toggle cannot leave two servers bound. */ + private transition: Promise = Promise.resolve(); + + constructor( + private readonly store: McpSettingsStore, + private readonly deps: McpToolDeps, + ) {} + + private serially(task: () => Promise): Promise { + const next = this.transition.then(task, task); + this.transition = next.catch(() => undefined); + return next; + } + + private async stopNow(): Promise { + const running = this.running; + this.running = null; + await running?.close(); + } + + /** (Re)start from the stored settings — or stop, if they say disabled. */ + private async applySettings(): Promise { + await this.stopNow(); + this.error = null; + const { enabled, port } = this.store.getSettings(); + if (!enabled) return; + try { + const token = await this.store.getToken(); + this.running = await startMcpHttpServer({ port, token, deps: this.deps }); + } catch (error) { + const code = (error as NodeJS.ErrnoException | null)?.code; + this.error = + code === "EADDRINUSE" + ? `Port ${port} is already in use. Choose another port.` + : error instanceof Error + ? error.message + : String(error); + } + } + + startIfEnabled(): Promise { + return this.serially(() => this.applySettings()); + } + + stop(): Promise { + return this.serially(() => this.stopNow()); + } + + async getStatus(): Promise { + await this.transition; + const { enabled, port, allowEdits } = this.store.getSettings(); + let token: string | null = null; + if (enabled) { + try { + token = await this.store.getToken(); + } catch { + // Surfaced through `error` already, from the failed start. + } + } + return { + enabled, + port, + allowEdits, + running: this.running !== null, + url: `http://127.0.0.1:${port}${MCP_ENDPOINT_PATH}`, + token, + error: this.error, + }; + } + + async setEnabled(enabled: boolean): Promise { + await this.serially(async () => { + await this.store.setSettings({ enabled }); + await this.applySettings(); + }); + return this.getStatus(); + } + + async setPort(port: number): Promise { + await this.serially(async () => { + await this.store.setSettings({ port }); + await this.applySettings(); + }); + return this.getStatus(); + } + + /** No restart needed: the server reads it on every call. */ + async setAllowEdits(allowEdits: boolean): Promise { + await this.serially(() => this.store.setSettings({ allowEdits })); + return this.getStatus(); + } + + /** New token; the running server is restarted so the old one stops working at once. */ + async regenerateToken(): Promise { + await this.serially(async () => { + await this.store.regenerateToken(); + await this.applySettings(); + }); + return this.getStatus(); + } +} diff --git a/electron/mcp/mcp-settings-store.ts b/electron/mcp/mcp-settings-store.ts new file mode 100644 index 000000000..4d8aaaa47 --- /dev/null +++ b/electron/mcp/mcp-settings-store.ts @@ -0,0 +1,109 @@ +// Settings for the local MCP server: whether it runs, on which port, and the +// bearer token a client must present. +// +// Same split as LlmConfigStore: the non-secret part is plain JSON, the token is +// a credential and goes through the OS keychain (`safeStorage`), never plain +// JSON. The encryption is injected rather than imported so this file stays +// testable without Electron. + +import { randomBytes } from "node:crypto"; +import { readFileSync } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; + +export const DEFAULT_MCP_PORT = 47821; + +export interface McpSettings { + enabled: boolean; + port: number; + /** + * Whether MCP clients may run the tools that change the project. Off by + * default and separate from the in-app agent's "Project edits": turning the + * server on gives a client read access, and writing is a second decision. + */ + allowEdits: boolean; +} + +export interface McpTokenCrypto { + isEncryptionAvailable(): boolean; + encryptString(plain: string): Buffer; + decryptString(encrypted: Buffer): string; +} + +export function isValidMcpPort(port: unknown): port is number { + return typeof port === "number" && Number.isInteger(port) && port >= 1024 && port <= 65535; +} + +export class McpSettingsStore { + private readonly settingsPath: string; + private readonly tokenPath: string; + private settings: McpSettings; + private token: string | null = null; + + constructor( + userDataPath: string, + private readonly crypto: McpTokenCrypto, + ) { + this.settingsPath = path.join(userDataPath, "mcp-server.json"); + this.tokenPath = path.join(userDataPath, "mcp-token.enc"); + this.settings = this.readSettings(); + } + + private readSettings(): McpSettings { + try { + const raw = JSON.parse(readFileSync(this.settingsPath, "utf8")) as Partial; + return { + enabled: raw.enabled === true, + port: isValidMcpPort(raw.port) ? raw.port : DEFAULT_MCP_PORT, + allowEdits: raw.allowEdits === true, + }; + } catch { + return { enabled: false, port: DEFAULT_MCP_PORT, allowEdits: false }; + } + } + + getSettings(): McpSettings { + return { ...this.settings }; + } + + async setSettings(patch: Partial): Promise { + if (patch.port !== undefined && !isValidMcpPort(patch.port)) { + throw new Error("Port must be a whole number between 1024 and 65535."); + } + this.settings = { ...this.settings, ...patch }; + await fs.writeFile(this.settingsPath, JSON.stringify(this.settings, null, 2), "utf8"); + return this.getSettings(); + } + + /** + * The token, created on first use. Read lazily: on macOS the decrypt is a + * Keychain access, which must not happen for users who never turn MCP on. + */ + async getToken(): Promise { + if (this.token) return this.token; + try { + const encrypted = readFileSync(this.tokenPath); + if (this.crypto.isEncryptionAvailable()) { + const token = this.crypto.decryptString(encrypted); + if (token) { + this.token = token; + return token; + } + } + } catch { + // No token yet, or unreadable — mint a fresh one below. + } + return this.regenerateToken(); + } + + /** Replace the token. Every client configured with the old one stops working. */ + async regenerateToken(): Promise { + if (!this.crypto.isEncryptionAvailable()) { + throw new Error("safeStorage is not available on this platform."); + } + const token = randomBytes(32).toString("base64url"); + await fs.writeFile(this.tokenPath, this.crypto.encryptString(token)); + this.token = token; + return token; + } +} diff --git a/electron/mcp/openscreen-mcp-server.test.ts b/electron/mcp/openscreen-mcp-server.test.ts new file mode 100644 index 000000000..f57b22c98 --- /dev/null +++ b/electron/mcp/openscreen-mcp-server.test.ts @@ -0,0 +1,252 @@ +// End-to-end over real HTTP: the SDK's own client against the server on an +// ephemeral port, with an in-memory stand-in for the editor window. + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; +import { afterEach, describe, expect, it } from "vitest"; +import { + type AxcutDocument, + createEmptyDocument, + documentSchema, +} from "../../src/lib/ai-edition/schema"; +import { OPENSCREEN_TOOL_NAMES } from "../ai-edition/agent-tools"; +import { TOOL_DESCRIPTIONS } from "../ai-edition/deep-agent/service"; +import { + MCP_ENDPOINT_PATH, + type McpApplyResult, + type McpDocumentHost, + type RunningMcpServer, + startMcpHttpServer, +} from "./openscreen-mcp-server"; + +const TOKEN = "test-token-0123456789"; + +function fixtureDocument(): AxcutDocument { + const base = createEmptyDocument({ + title: "Test", + projectId: "proj_1", + createdAt: "2026-01-01T00:00:00.000Z", + }); + return documentSchema.parse({ + ...base, + project: { ...base.project, primaryAssetId: "asset_1" }, + assets: [ + { + id: "asset_1", + kind: "video", + label: "Recording", + originalPath: "/tmp/rec.mp4", + durationSec: 30, + }, + ], + timeline: { + ...base.timeline, + clips: [ + { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 30, + timelineStartSec: 0, + timelineEndSec: 30, + wordRefs: [], + origin: "user", + reason: "", + }, + ], + }, + }); +} + +/** The editor window, reduced to a document and the revision guarding it. */ +class FakeEditor implements McpDocumentHost { + document: AxcutDocument | null = fixtureDocument(); + revision = 1; + applied: AxcutDocument[] = []; + /** Runs between the snapshot and the apply — where a user edit would land. */ + onSnapshot?: () => void; + + async snapshot() { + if (!this.document) return null; + const snapshot = { document: structuredClone(this.document), revision: this.revision }; + this.onSnapshot?.(); + return snapshot; + } + + async apply(document: AxcutDocument, expectedRevision: number): Promise { + if (!this.document) return "no-live-document"; + if (expectedRevision !== this.revision) return "conflict"; + this.document = document; + this.revision += 1; + this.applied.push(document); + return "applied"; + } +} + +let running: RunningMcpServer | null = null; +let client: Client | null = null; + +afterEach(async () => { + await client?.close(); + await running?.close(); + client = null; + running = null; +}); + +async function connect( + editor: FakeEditor, + options: { editsAllowed?: boolean; headers?: Record } = {}, +) { + running = await startMcpHttpServer({ + port: 0, + token: TOKEN, + deps: { host: editor, editsAllowed: () => options.editsAllowed ?? true, version: "0.0.0" }, + }); + client = new Client({ name: "test", version: "0.0.0" }); + const url = new URL(`http://127.0.0.1:${running.port}${MCP_ENDPOINT_PATH}`); + await client.connect( + new StreamableHTTPClientTransport(url, { + requestInit: { headers: options.headers ?? { Authorization: `Bearer ${TOKEN}` } }, + }), + ); + return client; +} + +function resultText(result: Awaited>): string { + const content = result.content as Array<{ type: string; text?: string }>; + return content.map((part) => part.text ?? "").join(""); +} + +describe("the MCP tool surface", () => { + it("is exactly the in-app agent's tools, with its descriptions", async () => { + const mcp = await connect(new FakeEditor()); + const { tools } = await mcp.listTools(); + expect(tools.map((t) => t.name)).toEqual([...OPENSCREEN_TOOL_NAMES]); + for (const tool of tools) { + expect(tool.description).toBe(TOOL_DESCRIPTIONS[tool.name]); + expect(tool.inputSchema.type).toBe("object"); + } + // The zod schemas survive the trip to JSON Schema with their fields intact. + const addTrim = tools.find((t) => t.name === "addTrim"); + expect(Object.keys(addTrim?.inputSchema.properties ?? {})).toEqual( + expect.arrayContaining(["startSec", "endSec"]), + ); + }); + + it("marks reads read-only and deletions destructive", async () => { + const mcp = await connect(new FakeEditor()); + const { tools } = await mcp.listTools(); + const byName = new Map(tools.map((t) => [t.name, t.annotations])); + expect(byName.get("getCurrentDocument")?.readOnlyHint).toBe(true); + expect(byName.get("addTrim")?.readOnlyHint).toBe(false); + expect(byName.get("removeClip")?.destructiveHint).toBe(true); + }); + + it("hands the client the in-app agent's guidance as server instructions", async () => { + const mcp = await connect(new FakeEditor()); + expect(mcp.getInstructions()).toContain("Time-bases (do not mix them up)"); + }); +}); + +describe("calling a tool", () => { + it("reads the live editor document", async () => { + const mcp = await connect(new FakeEditor()); + const result = await mcp.callTool({ name: "getCurrentDocument", arguments: {} }); + expect(result.isError).toBeFalsy(); + expect(resultText(result)).toContain("clip_1"); + }); + + it("applies an edit to the editor, against the revision it read", async () => { + const editor = new FakeEditor(); + const mcp = await connect(editor); + const result = await mcp.callTool({ + name: "addTrim", + arguments: { assetId: "asset_1", startSec: 5, endSec: 6 }, + }); + expect(result.isError).toBeFalsy(); + expect(editor.applied).toHaveLength(1); + expect(editor.document?.timeline.trimRanges).toHaveLength(1); + expect(editor.revision).toBe(2); + }); + + it("does not touch the editor for a read", async () => { + const editor = new FakeEditor(); + const mcp = await connect(editor); + await mcp.callTool({ name: "getTranscript", arguments: {} }); + expect(editor.applied).toHaveLength(0); + }); + + it("refuses a write when the user has turned project edits off", async () => { + const editor = new FakeEditor(); + const mcp = await connect(editor, { editsAllowed: false }); + const result = await mcp.callTool({ + name: "addTrim", + arguments: { assetId: "asset_1", startSec: 5, endSec: 6 }, + }); + expect(result.isError).toBe(true); + expect(editor.applied).toHaveLength(0); + }); + + it("reports a conflict instead of overwriting a user edit made mid-call", async () => { + const editor = new FakeEditor(); + editor.onSnapshot = () => { + editor.revision += 1; + }; + const mcp = await connect(editor); + const result = await mcp.callTool({ + name: "addTrim", + arguments: { assetId: "asset_1", startSec: 5, endSec: 6 }, + }); + expect(result.isError).toBe(true); + expect(resultText(result)).toContain("NOT applied"); + expect(editor.applied).toHaveLength(0); + }); + + it("says so when no project is open", async () => { + const editor = new FakeEditor(); + editor.document = null; + const mcp = await connect(editor); + const result = await mcp.callTool({ name: "getCurrentDocument", arguments: {} }); + expect(result.isError).toBe(true); + expect(resultText(result)).toContain("No project is open"); + }); +}); + +describe("the HTTP guard", () => { + async function post(headers: Record, path = MCP_ENDPOINT_PATH) { + running = await startMcpHttpServer({ + port: 0, + token: TOKEN, + deps: { host: new FakeEditor(), editsAllowed: () => true, version: "0.0.0" }, + }); + return fetch(`http://127.0.0.1:${running.port}${path}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json, text/event-stream", + ...headers, + }, + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list" }), + }); + } + + it("rejects a request without the token", async () => { + expect((await post({})).status).toBe(401); + }); + + it("rejects a wrong token", async () => { + expect((await post({ Authorization: "Bearer nope" })).status).toBe(401); + }); + + it("rejects a browser origin", async () => { + const response = await post({ + Authorization: `Bearer ${TOKEN}`, + Origin: "https://evil.example", + }); + expect(response.status).toBe(403); + }); + + it("rejects any path but the endpoint", async () => { + expect((await post({ Authorization: `Bearer ${TOKEN}` }, "/other")).status).toBe(404); + }); +}); diff --git a/electron/mcp/openscreen-mcp-server.ts b/electron/mcp/openscreen-mcp-server.ts new file mode 100644 index 000000000..1e24700b9 --- /dev/null +++ b/electron/mcp/openscreen-mcp-server.ts @@ -0,0 +1,255 @@ +// The OpenScreen MCP server: the in-app agent's tools, offered to any MCP client +// (Claude Code, Codex, Cursor…) that the user runs themselves. +// +// Nothing here is a second implementation. The tool list, argument schemas, +// descriptions and guidance are the in-app agent's own (`TOOL_ARG_SCHEMAS`, +// `TOOL_DESCRIPTIONS`, `buildSystemPrompt`), and every call runs through +// `runDocumentTool` — the same executor, consent gate and cursor read the +// in-app agent uses. The one difference is where the document comes from: the +// in-app agent is handed a snapshot per chat turn, while here each call reads +// the live document from the editor window and writes the result back through +// the same revision-guarded apply, so a user edit landing mid-call is never +// overwritten and every edit is one undo step. +// +// The HTTP layer is local-only: bound to 127.0.0.1, a bearer token on every +// request, and a Host/Origin check so a web page cannot reach it by DNS +// rebinding. + +import { timingSafeEqual } from "node:crypto"; +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { type AxcutDocument, documentSchema } from "../../src/lib/ai-edition/schema"; +import { isMutatingTool } from "../ai-edition/agent-tools"; +import { + buildSystemPrompt, + type CursorTelemetryReader, + probeCursorTelemetry, + runDocumentTool, + TOOL_ARG_SCHEMAS, + TOOL_DESCRIPTIONS, +} from "../ai-edition/deep-agent/service"; + +export const MCP_SERVER_NAME = "openscreen"; +export const MCP_ENDPOINT_PATH = "/mcp"; + +export interface McpDocumentSnapshot { + document: unknown; + /** The editor's revision when the snapshot was taken — the apply guard. */ + revision: number; +} + +/** Mirrors the renderer's `AgentDocumentApplyResult`, plus the editor having gone + * away or not answering in time. */ +export type McpApplyResult = + | "applied" + | "conflict" + | "save-failed" + | "no-live-document" + | "no-editor" + | "timeout"; + +/** The live document, as held by the editor window. */ +export interface McpDocumentHost { + /** `null` when no editor window is open or it has no project loaded. */ + snapshot(): Promise; + apply(document: AxcutDocument, expectedRevision: number): Promise; +} + +export interface McpToolDeps { + host: McpDocumentHost; + /** The "Project edits" setting — the same one the in-app agent obeys. */ + editsAllowed(): boolean; + cursor?: CursorTelemetryReader; + version: string; +} + +const NO_PROJECT_MESSAGE = + "No project is open in the OpenScreen editor. Ask the user to open one in OpenScreen, then retry."; + +const APPLY_FAILURE_MESSAGES: Record, string> = { + conflict: + "The edit was NOT applied: the project changed in the editor while this call ran. Call getCurrentDocument to re-read it, then retry.", + "save-failed": "The edit was NOT applied: OpenScreen could not save the project.", + "no-live-document": NO_PROJECT_MESSAGE, + "no-editor": NO_PROJECT_MESSAGE, + // Not "NOT applied": the editor may have saved it and only the answer was lost. + timeout: + "OpenScreen did not confirm this edit in time. Call getCurrentDocument to see whether it landed before retrying.", +}; + +const DESTRUCTIVE_TOOLS: ReadonlySet = new Set([ + "replaceTimeline", + "removeTrim", + "removeModifier", + "removeClip", +]); + +const MCP_PREAMBLE = [ + "These tools act on the project currently open in the OpenScreen editor. Every edit is saved straight away and appears in the editor, where the user can undo it with Ctrl/Cmd+Z. Nothing here records, exports or imports media.", + "", +].join("\n"); + +function textResult(text: string, isError: boolean): CallToolResult { + return { content: [{ type: "text", text }], ...(isError ? { isError: true } : {}) }; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * Runs tool calls one at a time. Each call is snapshot → execute → apply, and + * two of those interleaving would make the second apply against a revision the + * first just moved — a spurious conflict at best. + */ +export function createToolRunner(deps: McpToolDeps) { + let queue: Promise = Promise.resolve(); + + async function run(name: string, args: unknown): Promise { + const snapshot = await deps.host.snapshot(); + if (!snapshot) return textResult(NO_PROJECT_MESSAGE, true); + const parsed = documentSchema.safeParse(snapshot.document); + if (!parsed.success) { + return textResult("The project open in the editor could not be read.", true); + } + const document = parsed.data; + const availableByAssetId = await probeCursorTelemetry(document, deps.cursor); + const execution = await runDocumentTool(document, name, args, deps.editsAllowed(), { + cursor: deps.cursor, + availableByAssetId, + }); + if (execution.document) { + const applied = await deps.host.apply(execution.document, snapshot.revision); + if (applied !== "applied") return textResult(APPLY_FAILURE_MESSAGES[applied], true); + } + return textResult(execution.resultJson, !execution.ok); + } + + return (name: string, args: unknown): Promise => { + const next = queue.then( + () => run(name, args), + () => run(name, args), + ); + queue = next.catch(() => undefined); + return next.catch((error) => textResult(`Tool failed: ${errorMessage(error)}`, true)); + }; +} + +export function createOpenScreenMcpServer( + deps: McpToolDeps, + runTool: (name: string, args: unknown) => Promise, +): McpServer { + const server = new McpServer( + { name: MCP_SERVER_NAME, version: deps.version }, + { instructions: MCP_PREAMBLE + buildSystemPrompt({ editsAllowed: deps.editsAllowed() }) }, + ); + for (const [name, schema] of TOOL_ARG_SCHEMAS) { + const mutating = isMutatingTool(name); + server.registerTool( + name, + { + description: TOOL_DESCRIPTIONS[name], + inputSchema: schema, + annotations: { + readOnlyHint: !mutating, + destructiveHint: DESTRUCTIVE_TOOLS.has(name), + openWorldHint: false, + }, + }, + (args: unknown) => runTool(name, args), + ); + } + return server; +} + +function isAllowedHost(hostHeader: string | undefined, port: number): boolean { + return hostHeader === `127.0.0.1:${port}` || hostHeader === `localhost:${port}`; +} + +function isAllowedOrigin(origin: string | undefined, port: number): boolean { + // MCP clients are not browsers and send no Origin; a browser always does. + if (origin === undefined) return true; + return origin === `http://127.0.0.1:${port}` || origin === `http://localhost:${port}`; +} + +function hasValidToken(authorization: string | undefined, token: string): boolean { + const presented = authorization?.startsWith("Bearer ") ? authorization.slice(7).trim() : ""; + const a = Buffer.from(presented); + const b = Buffer.from(token); + return a.length === b.length && timingSafeEqual(a, b); +} + +function reject(res: ServerResponse, status: number, message: string, headers = {}): void { + res.writeHead(status, { "Content-Type": "application/json", ...headers }); + res.end(JSON.stringify({ jsonrpc: "2.0", error: { code: -32000, message }, id: null })); +} + +export interface RunningMcpServer { + port: number; + close(): Promise; +} + +export async function startMcpHttpServer(options: { + port: number; + token: string; + deps: McpToolDeps; +}): Promise { + const { token, deps } = options; + const runTool = createToolRunner(deps); + let boundPort = options.port; + + const handle = async (req: IncomingMessage, res: ServerResponse) => { + const pathname = new URL(req.url ?? "/", "http://127.0.0.1").pathname; + if (pathname !== MCP_ENDPOINT_PATH) return reject(res, 404, "Not found"); + if (!isAllowedHost(req.headers.host, boundPort)) return reject(res, 403, "Forbidden host"); + if (!isAllowedOrigin(req.headers.origin, boundPort)) { + return reject(res, 403, "Forbidden origin"); + } + if (!hasValidToken(req.headers.authorization, token)) { + return reject(res, 401, "Unauthorized", { "WWW-Authenticate": "Bearer" }); + } + + // Stateless: one server + transport per request, the SDK's documented shape + // for a server that keeps no per-session state. Tools read the live editor + // on every call, so there is nothing a session would need to remember. + const server = createOpenScreenMcpServer(deps, runTool); + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined, + enableJsonResponse: true, + }); + res.on("close", () => { + void transport.close(); + void server.close(); + }); + try { + await server.connect(transport); + await transport.handleRequest(req, res); + } catch (error) { + if (!res.headersSent) reject(res, 500, errorMessage(error)); + } + }; + + const httpServer: Server = createServer((req, res) => { + void handle(req, res); + }); + await new Promise((resolve, reject) => { + httpServer.once("error", reject); + httpServer.listen(options.port, "127.0.0.1", () => { + httpServer.off("error", reject); + resolve(); + }); + }); + boundPort = (httpServer.address() as AddressInfo).port; + + return { + port: boundPort, + close: () => + new Promise((resolve) => { + httpServer.closeAllConnections(); + httpServer.close(() => resolve()); + }), + }; +} diff --git a/electron/media/mediaLinksRegistry.test.ts b/electron/media/mediaLinksRegistry.test.ts index 7c76a7580..0aae45269 100644 --- a/electron/media/mediaLinksRegistry.test.ts +++ b/electron/media/mediaLinksRegistry.test.ts @@ -187,6 +187,29 @@ describe("mediaLinksRegistry", () => { } }); + it("round-trips additional cameras and resolves old records without them", async () => { + const screenPath = path.join(tempDir, "multi.webm"); + const oldScreenPath = path.join(tempDir, "old.webm"); + await writeFileOfSize(screenPath, 5000, "m"); + await writeFileOfSize(oldScreenPath, 5000, "o"); + const additionalWebcams = [ + { path: path.join(tempDir, "multi-webcam-2.mp4"), label: "Desk" }, + { path: path.join(tempDir, "multi-webcam-3.mp4"), label: "" }, + ]; + await registerMediaLinks(tempDir, screenPath, { + webcamVideoPath: path.join(tempDir, "multi-webcam.mp4"), + additionalWebcams, + }); + await registerMediaLinks(tempDir, oldScreenPath, { + webcamVideoPath: path.join(tempDir, "old-webcam.mp4"), + }); + + const resolved = await findMediaLinksByFingerprint(tempDir, screenPath); + expect(resolved?.additionalWebcams).toEqual(additionalWebcams); + const old = await findMediaLinksByFingerprint(tempDir, oldScreenPath); + expect(old).not.toHaveProperty("additionalWebcams"); + }); + it("returns null when there is no matching fingerprint", async () => { const unknownPath = path.join(tempDir, "unknown.webm"); await writeFileOfSize(unknownPath, 1000, "z"); diff --git a/electron/media/mediaLinksRegistry.ts b/electron/media/mediaLinksRegistry.ts index ec0ad76de..56bd7d57f 100644 --- a/electron/media/mediaLinksRegistry.ts +++ b/electron/media/mediaLinksRegistry.ts @@ -19,7 +19,11 @@ import fs from "node:fs/promises"; import path from "node:path"; -import type { CursorCaptureMode } from "../../src/lib/recordingSession"; +import { + type AdditionalWebcam, + type CursorCaptureMode, + normalizeAdditionalWebcams, +} from "../../src/lib/recordingSession"; // ponytail: `baseDir` is passed in by every caller (RECORDINGS_DIR in // electron/ipc/handlers.ts) rather than imported here, so this module has no @@ -40,6 +44,7 @@ export interface MediaLinkEntry { fingerprint: MediaFingerprint; webcamVideoPath?: string; webcamOffsetMs?: number; + additionalWebcams?: AdditionalWebcam[]; cursorTelemetryPath?: string; cursorCaptureMode?: CursorCaptureMode; updatedAt: string; @@ -98,6 +103,7 @@ function normalizeEntry(candidate: unknown): MediaLinkEntry | null { if (!candidate || typeof candidate !== "object") return null; const raw = candidate as Partial; const fp = raw.fingerprint; + const additionalWebcams = normalizeAdditionalWebcams(raw.additionalWebcams); if ( typeof raw.lastKnownPath !== "string" || !fp || @@ -116,6 +122,7 @@ function normalizeEntry(candidate: unknown): MediaLinkEntry | null { }, ...(typeof raw.webcamVideoPath === "string" ? { webcamVideoPath: raw.webcamVideoPath } : {}), ...(typeof raw.webcamOffsetMs === "number" ? { webcamOffsetMs: raw.webcamOffsetMs } : {}), + ...(additionalWebcams.length > 0 ? { additionalWebcams } : {}), ...(typeof raw.cursorTelemetryPath === "string" ? { cursorTelemetryPath: raw.cursorTelemetryPath } : {}), @@ -220,6 +227,7 @@ async function updateRegistry( export interface MediaLinksToRegister { webcamVideoPath?: string; webcamOffsetMs?: number; + additionalWebcams?: AdditionalWebcam[]; cursorTelemetryPath?: string; cursorCaptureMode?: CursorCaptureMode; } @@ -235,7 +243,12 @@ export async function registerMediaLinks( videoPath: string, links: MediaLinksToRegister, ): Promise { + // Extras alone register nothing. That relies on R6: additional cameras are + // recorded only while camera 1 is on, so a take with extras but no camera 1 + // is one whose camera 1 file came out empty — a rare loss accepted here. if (!links.webcamVideoPath && !links.cursorTelemetryPath) return; + const { additionalWebcams: rawAdditionalWebcams, ...linksWithoutAdditional } = links; + const additionalWebcams = normalizeAdditionalWebcams(rawAdditionalWebcams); const fingerprint = await computeFingerprint(videoPath); await updateRegistry(baseDir, (file) => { const existingIndex = file.entries.findIndex((e) => @@ -244,7 +257,8 @@ export async function registerMediaLinks( const entry: MediaLinkEntry = { lastKnownPath: videoPath, fingerprint, - ...links, + ...linksWithoutAdditional, + ...(additionalWebcams.length > 0 ? { additionalWebcams } : {}), updatedAt: new Date().toISOString(), }; const entries = @@ -258,6 +272,7 @@ export async function registerMediaLinks( export interface MediaLinksLookup { webcamVideoPath?: string; webcamOffsetMs?: number; + additionalWebcams?: AdditionalWebcam[]; cursorTelemetryPath?: string; cursorCaptureMode?: CursorCaptureMode; } @@ -314,6 +329,7 @@ export async function findRelocatedMediaByStoredPath( screenVideoPath: match.lastKnownPath, ...(match.webcamVideoPath ? { webcamVideoPath: match.webcamVideoPath } : {}), ...(typeof match.webcamOffsetMs === "number" ? { webcamOffsetMs: match.webcamOffsetMs } : {}), + ...(match.additionalWebcams?.length ? { additionalWebcams: match.additionalWebcams } : {}), ...(match.cursorTelemetryPath ? { cursorTelemetryPath: match.cursorTelemetryPath } : {}), ...(match.cursorCaptureMode ? { cursorCaptureMode: match.cursorCaptureMode } : {}), }; @@ -358,6 +374,7 @@ export async function findMediaLinksByFingerprint( return { ...(match.webcamVideoPath ? { webcamVideoPath: match.webcamVideoPath } : {}), ...(typeof match.webcamOffsetMs === "number" ? { webcamOffsetMs: match.webcamOffsetMs } : {}), + ...(match.additionalWebcams?.length ? { additionalWebcams: match.additionalWebcams } : {}), ...(match.cursorTelemetryPath ? { cursorTelemetryPath: match.cursorTelemetryPath } : {}), ...(match.cursorCaptureMode ? { cursorCaptureMode: match.cursorCaptureMode } : {}), }; diff --git a/electron/media/projectMediaRelinker.test.ts b/electron/media/projectMediaRelinker.test.ts index 2b52d32b5..505a430f4 100644 --- a/electron/media/projectMediaRelinker.test.ts +++ b/electron/media/projectMediaRelinker.test.ts @@ -62,6 +62,54 @@ describe("relinkProjectMedia", () => { expect(logged.join("\n")).toContain(currentWebcamPath); }); + it("relinks additional cameras by index and leaves absent ones alone", async () => { + const currentScreenPath = path.join(tempDir, "recording-43.mp4"); + const currentWebcamPath = path.join(tempDir, "recording-43-webcam.mp4"); + const currentExtra2 = path.join(tempDir, "recording-43-webcam-2.mp4"); + const currentExtra3 = path.join(tempDir, "recording-43-webcam-3.mp4"); + await fs.writeFile(currentScreenPath, "screen bytes"); + await fs.writeFile(currentWebcamPath, "webcam bytes"); + await fs.writeFile(currentExtra2, "extra 2"); + await fs.writeFile(currentExtra3, "extra 3"); + await registerMediaLinks(tempDir, currentScreenPath, { + webcamVideoPath: currentWebcamPath, + additionalWebcams: [ + { path: currentExtra2, label: "Desk" }, + { path: currentExtra3, label: "Wide" }, + ], + }); + + const project = { + assets: [ + { + id: "asset-1", + originalPath: "C:\\Users\\demo\\recording-43.mp4", + sizeBytes: Buffer.byteLength("screen bytes"), + cameraTrack: { + sourcePath: "C:\\Users\\demo\\recording-43-webcam.mp4", + startMs: 0, + offsetMs: 0, + visible: true, + }, + additionalCameraTracks: [ + { sourcePath: "C:\\Users\\demo\\recording-43-webcam-2.mp4", label: "Desk" }, + { sourcePath: "C:\\Users\\demo\\recording-43-webcam-3.mp4", label: "Wide" }, + ], + }, + ], + }; + + const relinked = (await relinkProjectMedia(project, tempDir)) as typeof project; + + expect(relinked.assets[0].cameraTrack.sourcePath).toBe(currentWebcamPath); + expect(relinked.assets[0].additionalCameraTracks.map((t) => t.sourcePath)).toEqual([ + currentExtra2, + currentExtra3, + ]); + expect(relinked.assets[0].additionalCameraTracks.map((t) => t.label)).toEqual(["Desk", "Wide"]); + expect(project.assets[0].additionalCameraTracks[0].sourcePath).toContain("demo"); + }); + it("refuses to relink an asset the document recorded no size for", async () => { // A same-named recording exists and is registered with its webcam, so a // basename match would resolve — that is exactly what must not happen. The diff --git a/electron/media/projectMediaRelinker.ts b/electron/media/projectMediaRelinker.ts index 0c0b8c259..bef7dad02 100644 --- a/electron/media/projectMediaRelinker.ts +++ b/electron/media/projectMediaRelinker.ts @@ -41,11 +41,23 @@ async function resolveAssetMedia( isRecord(cameraTrack) && typeof cameraTrack.sourcePath === "string" && cameraTrack.sourcePath ? cameraTrack.sourcePath : null; + const additionalTracks = Array.isArray(asset.additionalCameraTracks) + ? asset.additionalCameraTracks + : []; + const additionalMissing = await Promise.all( + additionalTracks.map( + async (track) => + isRecord(track) && + typeof track.sourcePath === "string" && + track.sourcePath !== "" && + !(await fileExists(track.sourcePath)), + ), + ); const screenExists = await fileExists(originalPath); const cameraMissing = cameraPath !== null && !(await fileExists(cameraPath)); // Nothing to repair, and this runs on every project open — don't fingerprint // (i.e. open and read) every asset just to confirm what the stats already say. - if (screenExists && !cameraMissing) return asset; + if (screenExists && !cameraMissing && !additionalMissing.some(Boolean)) return asset; let links: RelocatedMediaLookup | null = null; if (screenExists) { @@ -81,10 +93,25 @@ async function resolveAssetMedia( nextCameraTrack = { ...cameraTrack, sourcePath: links.webcamVideoPath }; } + // Extras are matched by index, the same order the recording registered them in. + let additionalChanged = false; + const nextAdditionalTracks = await Promise.all( + additionalTracks.map(async (track, index) => { + const replacement = links.additionalWebcams?.[index]?.path; + if (!additionalMissing[index] || !replacement || !(await fileExists(replacement))) { + return track; + } + console.log(`[media-relink] additional webcam ${index + 2} -> ${replacement}`); + additionalChanged = true; + return { ...track, sourcePath: replacement }; + }), + ); + return { ...asset, originalPath: links.screenVideoPath, ...(nextCameraTrack === cameraTrack ? {} : { cameraTrack: nextCameraTrack }), + ...(additionalChanged ? { additionalCameraTracks: nextAdditionalTracks } : {}), }; } diff --git a/electron/native-bridge/screen/macPickerSession.test.ts b/electron/native-bridge/screen/macPickerSession.test.ts index c5945bedf..adac0495b 100644 --- a/electron/native-bridge/screen/macPickerSession.test.ts +++ b/electron/native-bridge/screen/macPickerSession.test.ts @@ -199,6 +199,57 @@ describe("MacPickerSession", () => { expect(helper.commands.at(-1)).toBe("stop"); }); + it("forgets a screen pick once an app window it does not leave out must be (#965)", async () => { + const { session } = await pickDisplay(); + + // The windows the picker was told about, in any order, or fewer of them: still good. + session.forgetSelectionUnlessExcluding([8, 7]); + session.forgetSelectionUnlessExcluding([7]); + expect(session.getSelection()).not.toBeNull(); + + // The HUD came back from the editor as a new window: the pick would record it. + session.forgetSelectionUnlessExcluding([9]); + expect(session.getSelection()).toBeNull(); + }); + + it("keeps the exclusions of the pick, not of a picker that was cancelled", async () => { + const { helper, session } = await pickDisplay(); + const cancelled = session.present([9]); + await flush(); + helper.say({ event: "picker-cancelled" }); + expect(await cancelled).toBeNull(); + + session.forgetSelectionUnlessExcluding([7, 8]); + expect(session.getSelection()).not.toBeNull(); + session.forgetSelectionUnlessExcluding([9]); + expect(session.getSelection()).toBeNull(); + }); + + it("keeps a window pick whatever the app's windows are", async () => { + const { helper, session } = await readySession(); + const pick = session.present([7]); + await flush(); + helper.say({ ...DISPLAY_PICK, kind: "window", windowId: 42 }); + expect(await pick).not.toBeNull(); + + session.forgetSelectionUnlessExcluding([9]); + expect(session.getSelection()).toMatchObject({ kind: "window" }); + }); + + it("keeps the pick while a take is running from it", async () => { + const { helper, session } = await pickDisplay(); + session.startTake({}); + + // The notes window opened mid-take: the running take still reads the pick's frame. + session.forgetSelectionUnlessExcluding([7, 8, 9]); + expect(session.getSelection()).not.toBeNull(); + + helper.say({ event: "take-ended" }); + await flush(); + session.forgetSelectionUnlessExcluding([7, 8, 9]); + expect(session.getSelection()).toBeNull(); + }); + it("ends a running take and forgets the pick when the session dies", async () => { const { helper, session } = await pickDisplay(); const take = session.startTake({}); diff --git a/electron/native-bridge/screen/macPickerSession.ts b/electron/native-bridge/screen/macPickerSession.ts index 0f0e5b186..7b7aea758 100644 --- a/electron/native-bridge/screen/macPickerSession.ts +++ b/electron/native-bridge/screen/macPickerSession.ts @@ -186,6 +186,10 @@ export class MacPickerSession { private proc: ChildProcessWithoutNullStreams | null = null; private starting: Promise | null = null; private selection: MacPickerSelection | null = null; + /** The windows the picker was told to leave out, as of the last `present`. */ + private presentedExclusions: readonly number[] = []; + /** The windows the retained pick leaves out: baked into its filter, fixed until the next pick. */ + private selectionExclusions: readonly number[] = []; private take: TakeProcess | null = null; private pendingPick: ((selection: MacPickerSelection | null) => void) | null = null; private lineBuffer = ""; @@ -199,6 +203,26 @@ export class MacPickerSession { return this.selection; } + /** + * Forgets a screen pick that does not leave out every one of `windowIds`. + * + * The windows a screen pick leaves out are the ones the picker was told about when it + * was shown, baked into the pick's filter. A take cannot add one: a fresh filter needs + * the Screen Recording grant this picker exists to avoid. So once the HUD or the notes + * window is not one the pick knows -- the HUD is a new window after every trip through + * the editor -- the pick would record it, and only picking again leaves it out (#965). + * A window pick records that one window and is never affected, and neither is a take + * already running: its filter is set, and its cursor telemetry still reads the pick. + */ + forgetSelectionUnlessExcluding(windowIds: readonly number[]) { + if (this.selection?.kind !== "display" || (this.take && !this.take.ended)) { + return; + } + if (!windowIds.every((windowId) => this.selectionExclusions.includes(windowId))) { + this.selection = null; + } + } + /** Starts the helper if it is not running. Resolves false when it cannot. */ start(): Promise { if (this.proc) { @@ -286,6 +310,7 @@ export class MacPickerSession { resolve(selection); }; }); + this.presentedExclusions = [...excludedWindowIds]; this.send({ command: "present", excludedWindowIds, @@ -335,6 +360,7 @@ export class MacPickerSession { const selection = parsePickerSelection(event); if (selection) { this.selection = selection; + this.selectionExclusions = this.presentedExclusions; } this.resolvePick(selection); return; diff --git a/electron/native-bridge/services/aiEditionService.ts b/electron/native-bridge/services/aiEditionService.ts index 90088781a..579ba59a3 100644 --- a/electron/native-bridge/services/aiEditionService.ts +++ b/electron/native-bridge/services/aiEditionService.ts @@ -13,6 +13,7 @@ import type { AiEditionLlmConfig, AiEditionLlmDisconnectResult, AiEditionLlmSnapshot, + AiEditionMcpStatus, AiEditionProjectSummary, } from "../../../src/native/contracts"; import { @@ -31,6 +32,7 @@ import { probeMiniMaxModels, } from "../../ai-edition/llm-provider-auth"; import { PROVIDER_DEFINITIONS } from "../../ai-edition/provider-registry"; +import type { McpController } from "../../mcp/mcp-controller"; export interface AiEditionServiceOptions { documents: DocumentService; @@ -80,6 +82,8 @@ export interface AiEditionServiceOptions { title: string, ) => AiEditionChatSessionSummary | null; deleteSession: (projectId: string, sessionId: string) => boolean; + /** The local MCP server. Absent where no server is wired (tests, headless CLI). */ + mcp?: McpController; } export class AiEditionService { @@ -271,6 +275,31 @@ export class AiEditionService { } } + private get mcp(): McpController { + if (!this.options.mcp) throw new Error("The MCP server is not available in this build."); + return this.options.mcp; + } + + mcpGetStatus(): Promise { + return this.mcp.getStatus(); + } + + mcpSetEnabled(enabled: boolean): Promise { + return this.mcp.setEnabled(enabled); + } + + mcpSetPort(port: number): Promise { + return this.mcp.setPort(port); + } + + mcpSetAllowEdits(allowEdits: boolean): Promise { + return this.mcp.setAllowEdits(allowEdits); + } + + mcpRegenerateToken(): Promise { + return this.mcp.regenerateToken(); + } + async chatRun( projectId: string, sessionId: string, diff --git a/electron/native-bridge/services/compositorViewService.test.ts b/electron/native-bridge/services/compositorViewService.test.ts index eee8fe305..4a099a7f9 100644 --- a/electron/native-bridge/services/compositorViewService.test.ts +++ b/electron/native-bridge/services/compositorViewService.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import type { WebFrameMain } from "electron"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { CURSOR_THEMES, DEFAULT_CURSOR_SPRITES } from "../../../src/lib/cursor/cursorThemes"; import type { CompositorViewAddon, GifExportStats } from "../../native/compositor-view/addon"; @@ -58,6 +59,166 @@ describe("native GIF cancellation capability", () => { }); }); +describe("CompositorViewService frames handed over as shared GPU textures", () => { + const target = { frameTreeNodeId: 1 } as unknown as WebFrameMain; + const sharedFrame = { + gen: 3, + slot: 1, + handle: Buffer.from([1, 2, 3, 4, 5, 6, 7, 8]), + width: 4, + height: 2, + footage: null, + footageProjective: false, + }; + + function setup(options: { addon?: Partial; gpuCompositing?: boolean } = {}) { + const addon = { + createView: vi.fn(() => 7), + setSharedFrames: vi.fn(() => true), + readSharedFrame: vi.fn(() => sharedFrame), + readFrame: vi.fn(() => null), + releaseSharedFrame: vi.fn(), + destroyView: vi.fn(), + ...options.addon, + }; + const imported = { release: vi.fn() }; + let allReferencesReleased: (() => void) | undefined; + const sharedTexture = { + importSharedTexture: vi.fn((importOptions: Electron.ImportSharedTextureOptions) => { + allReferencesReleased = importOptions.allReferencesReleased as () => void; + return imported as unknown as Electron.SharedTextureImported; + }), + sendSharedTexture: vi.fn(async () => undefined), + }; + const service = new CompositorViewService({ + addon: addon as unknown as CompositorViewAddon, + sharedTexture, + gpuCompositing: () => options.gpuCompositing ?? true, + }); + const id = service.createView({ x: 0, y: 0, width: 4, height: 2 }); + return { + addon, + id, + imported, + service, + sharedTexture, + releaseEverywhere: () => allReferencesReleased?.(), + }; + } + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("sends the texture to the frame that asked, and answers with its receipt", async () => { + const { addon, id, imported, service, sharedTexture } = setup(); + expect(addon.setSharedFrames).toHaveBeenCalledWith(id, true); + + const receipt = await service.readFrame(id, 2, target); + + expect(addon.readSharedFrame).toHaveBeenCalledWith(id, 2); + expect(sharedTexture.importSharedTexture).toHaveBeenCalledWith( + expect.objectContaining({ + textureInfo: { + pixelFormat: "rgba", + codedSize: { width: 4, height: 2 }, + handle: { ntHandle: sharedFrame.handle }, + }, + }), + ); + const meta = { + viewId: id, + gen: 3, + width: 4, + height: 2, + footage: null, + footageProjective: false, + }; + expect(sharedTexture.sendSharedTexture).toHaveBeenCalledWith( + { frame: target, importedSharedTexture: imported }, + meta, + ); + expect(receipt).toEqual({ ...meta, shared: true }); + // This process's reference goes at once: the renderer holds its own. + expect(imported.release).toHaveBeenCalled(); + expect(addon.releaseSharedFrame).not.toHaveBeenCalled(); + }); + + it("hands the slot back once Chromium has let go of the texture everywhere", async () => { + const { addon, id, releaseEverywhere, service } = setup(); + await service.readFrame(id, 0, target); + + releaseEverywhere(); + + expect(addon.releaseSharedFrame).toHaveBeenCalledWith(id, 1, 3); + }); + + it("goes back to read-back when a delivery fails", async () => { + const { addon, id, service, sharedTexture } = setup(); + sharedTexture.sendSharedTexture.mockRejectedValueOnce(new Error("timed out after 1000ms")); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + try { + expect(await service.readFrame(id, 0, target)).toBeNull(); + } finally { + warn.mockRestore(); + } + expect(addon.setSharedFrames).toHaveBeenLastCalledWith(id, false); + + vi.mocked(addon.readSharedFrame).mockClear(); + await service.readFrame(id, 3, target); + expect(addon.readSharedFrame).not.toHaveBeenCalled(); + expect(addon.readFrame).toHaveBeenCalledWith(id, 3); + }); + + it("frees the slot itself when the texture never got imported", async () => { + const { addon, id, service, sharedTexture } = setup(); + sharedTexture.importSharedTexture.mockImplementationOnce(() => { + throw new TypeError("Invalid ntHandle value"); + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + try { + await service.readFrame(id, 0, target); + } finally { + warn.mockRestore(); + } + expect(addon.releaseSharedFrame).toHaveBeenCalledWith(id, 1, 3); + }); + + it("reads pixels for a view whose render thread went back to read-back on its own", async () => { + const packet = { gen: 4, width: 4, height: 2, data: Buffer.alloc(32) }; + const { id, service, sharedTexture } = setup({ + addon: { readSharedFrame: vi.fn(() => null), readFrame: vi.fn(() => packet) }, + }); + + expect(await service.readFrame(id, 3, target)).toBe(packet); + expect(sharedTexture.sendSharedTexture).not.toHaveBeenCalled(); + }); + + it("keeps a view on read-back when Chromium does not composite on the GPU", async () => { + const { addon, id, service } = setup({ gpuCompositing: false }); + expect(addon.setSharedFrames).not.toHaveBeenCalled(); + + await service.readFrame(id, 0, target); + expect(addon.readSharedFrame).not.toHaveBeenCalled(); + expect(addon.readFrame).toHaveBeenCalledWith(id, 0); + }); + + it("keeps a view on read-back when OPENSCREEN_PREVIEW_READBACK=1", () => { + vi.stubEnv("OPENSCREEN_PREVIEW_READBACK", "1"); + const { addon } = setup(); + expect(addon.setSharedFrames).not.toHaveBeenCalled(); + }); + + it("keeps a view on read-back when the addon cannot share (not Windows, software backend)", async () => { + const { id, service, sharedTexture } = setup({ + addon: { setSharedFrames: vi.fn(() => false) }, + }); + + await service.readFrame(id, 0, target); + expect(sharedTexture.importSharedTexture).not.toHaveBeenCalled(); + }); +}); + /** A source checkout's `crates/.cargo/config.toml`, with `FFMPEG_DIR` written as `body`. */ function writeCargoConfig(root: string, body: string): void { const cargoDir = path.join(root, "crates", ".cargo"); diff --git a/electron/native-bridge/services/compositorViewService.ts b/electron/native-bridge/services/compositorViewService.ts index 4a765db2b..550d67dcb 100644 --- a/electron/native-bridge/services/compositorViewService.ts +++ b/electron/native-bridge/services/compositorViewService.ts @@ -2,14 +2,19 @@ import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { app } from "electron"; +import { app, sharedTexture, type WebFrameMain } from "electron"; import { type CursorKind, readCursorAsArrow, resolveCursorSprites, } from "../../../src/lib/cursor/cursorThemes"; +import type { + CompositorSharedFrameMeta, + CompositorSharedFrameReceipt, +} from "../../../src/native/contracts"; import type { GifExportJob } from "../../ipc/gifExportJobs"; import type { + ClipCameraInput, ClipInput, CompositorBackend, CompositorParamValue, @@ -20,6 +25,7 @@ import type { GifExportStats, GifParamsInput, NativeFramePacket, + NativeSharedFramePacket, RemuxStats, SegmentationSupport, } from "../../native/compositor-view/addon"; @@ -215,6 +221,26 @@ export interface CompositorViewServiceOptions { */ appRoot?: string; isPackaged?: boolean; + /** Electron's `sharedTexture` module, injectable for tests. `null` keeps every view on + * read-back. */ + sharedTexture?: SharedTextureApi | null; + /** Whether Chromium composites on the GPU, where a shared texture is imported. Injectable + * for tests; defaults to `app.getGPUFeatureStatus()`. */ + gpuCompositing?: () => boolean; +} + +/** The two calls a shared preview frame needs from Electron's `sharedTexture` module. */ +export type SharedTextureApi = Pick< + Electron.SharedTexture, + "importSharedTexture" | "sendSharedTexture" +>; + +function defaultGpuCompositing(): boolean { + try { + return String(app.getGPUFeatureStatus().gpu_compositing).startsWith("enabled"); + } catch { + return false; + } } function defaultAppRoot(): string { @@ -467,6 +493,8 @@ function tryLoadAddon(candidates: string[]): CompositorViewAddon | null { export class CompositorViewService { private readonly options: CompositorViewServiceOptions; private readonly rects = new Map(); + /** Views whose frames go out as shared GPU textures rather than RAM pixels. */ + private readonly sharedViews = new Set(); private addon: CompositorViewAddon | null = null; private loadAttempted = false; private syntheticIdCounter = 0; @@ -593,9 +621,41 @@ export class CompositorViewService { } const id = addon.createView(rect, paths?.screenPath, paths?.webcamPath, paths?.cursorPath); this.rects.set(id, rect); + this.shareFrames(addon, id); return id; } + /** Hands the view's frames over as shared GPU textures when this host can, instead of + * copying them through IPC: measured, that transport alone kept 37 to 55 % of the + * renderer's main thread busy at 30 fps (rendering-performance.md). Anything missing — + * the Electron API, GPU compositing, Windows' hardware backend native-side — leaves the + * view on read-back. `OPENSCREEN_PREVIEW_READBACK=1` forces read-back, to compare. */ + private shareFrames(addon: CompositorViewAddon, id: number): void { + if (process.env.OPENSCREEN_PREVIEW_READBACK === "1" || !this.sharedTextureApi()) { + return; + } + if (!(this.options.gpuCompositing ?? defaultGpuCompositing)()) { + return; + } + if (addon.setSharedFrames?.(id, true)) { + this.sharedViews.add(id); + } + } + + private sharedTextureApi(): SharedTextureApi | null { + const api = + this.options.sharedTexture === undefined ? sharedTexture : this.options.sharedTexture; + return typeof api?.importSharedTexture === "function" ? api : null; + } + + /** Takes the view back to read-back frames: a delivery failed, or the renderer saw a shared + * frame land as nothing (Chromium could not open the texture). The render thread + * republishes the current frame, so the canvas is not left empty. */ + stopSharedFrames(id: number): void { + this.sharedViews.delete(id); + this.ensureAddon()?.setSharedFrames?.(id, false); + } + setRect(id: number, rect: CompositorViewRect): void { const addon = this.ensureAddon(); this.rects.set(id, rect); @@ -605,19 +665,78 @@ export class CompositorViewService { addon.setRect(id, rect); } - /** Reads the most recently rendered frame for `id` as a self-describing packet - * (`{ gen, width, height, data }`), but only if its generation is newer than - * `sinceGen`. Returns `null` when the addon is absent, no frame is ready yet, - * OR the caller already holds the current generation — the idle path, where - * `null` comes back without any buffer copy. Byte order is RGBA. */ - readFrame(id: number, sinceGen: number): NativeFramePacket | null { + /** Reads the most recently rendered frame for `id`, but only if its generation is newer + * than `sinceGen`. Returns `null` when the addon is absent, no frame is ready yet, OR the + * caller already holds the current generation — the idle path, where nothing is copied. + * + * A view on shared textures sends the frame to `target` as a GPU texture, and answers with + * its receipt (`shared: true`, no pixels): the texture reaches the renderer before this + * reply does. Any other view answers with the RGBA pixels themselves. */ + async readFrame( + id: number, + sinceGen: number, + target?: WebFrameMain | null, + ): Promise { const addon = this.ensureAddon(); if (!addon) { return null; } + if (target && this.sharedViews.has(id)) { + const frame = addon.readSharedFrame?.(id, sinceGen); + if (frame) { + return this.sendSharedFrame(addon, id, frame, target); + } + } + // Also the answer for a view whose render thread went back to read-back on its own. return addon.readFrame(id, sinceGen); } + private async sendSharedFrame( + addon: CompositorViewAddon, + id: number, + frame: NativeSharedFramePacket, + target: WebFrameMain, + ): Promise { + const meta: CompositorSharedFrameMeta = { + viewId: id, + gen: frame.gen, + width: frame.width, + height: frame.height, + footage: frame.footage ?? null, + footageProjective: frame.footageProjective ?? false, + clipIndex: frame.clipIndex, + sourceTimeSec: frame.sourceTimeSec, + }; + const api = this.sharedTextureApi(); + let imported: Electron.SharedTextureImported | undefined; + try { + if (!api) { + throw new Error("the sharedTexture API is gone"); + } + imported = api.importSharedTexture({ + textureInfo: { + pixelFormat: "rgba", + codedSize: { width: frame.width, height: frame.height }, + handle: { ntHandle: frame.handle }, + }, + // Chromium is done with the texture in every process: the slot may be written again. + allReferencesReleased: () => addon.releaseSharedFrame?.(id, frame.slot, frame.gen), + }); + await api.sendSharedTexture({ frame: target, importedSharedTexture: imported }, meta); + return { ...meta, shared: true }; + } catch (error) { + console.warn("[compositor-view] shared frame not delivered; reading frames back:", error); + this.stopSharedFrames(id); + if (!imported) { + addon.releaseSharedFrame?.(id, frame.slot, frame.gen); + } + return null; + } finally { + // This process's reference only: the renderer holds its own until it has drawn. + imported?.release(); + } + } + setParam(id: number, key: string, value: CompositorParamValue): void { const addon = this.ensureAddon(); if (!addon) { @@ -657,17 +776,27 @@ export class CompositorViewService { webcamOffsetSec: number, clipIndex: number, sourceTimeSec: number, + additionalCameras: ClipCameraInput[] = [], ): void { const addon = this.ensureAddon(); if (!addon) { return; } - addon.setActiveClip(id, screenPath, webcamPath, webcamOffsetSec, clipIndex, sourceTimeSec); + addon.setActiveClip( + id, + screenPath, + webcamPath, + webcamOffsetSec, + clipIndex, + sourceTimeSec, + additionalCameras, + ); } destroyView(id: number): void { const addon = this.ensureAddon(); this.rects.delete(id); + this.sharedViews.delete(id); if (!addon) { return; } diff --git a/electron/native/README.md b/electron/native/README.md index 16013c4bd..a9c6f8b83 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -85,6 +85,17 @@ Current V2 JSON shape: The current helper implementation supports display/window video capture, system audio loopback, selected-microphone capture, Media Foundation webcam capture, and a DirectShow webcam fallback for virtual cameras that are not exposed through Media Foundation. Webcam frames are currently composed into the primary MP4 as a bottom-right picture-in-picture overlay. Browser `deviceId` values do not always map to Media Foundation symbolic links or WASAPI endpoint IDs, so the renderer passes both browser IDs and user-visible device names. For microphones, the helper tries the requested WASAPI endpoint ID first, then resolves an active capture endpoint by `microphoneDeviceName`, then falls back to the default endpoint. For webcams, Electron resolves a matching DirectShow filter CLSID for the selected label; the helper uses Media Foundation first, then that exact DirectShow filter when the requested camera is absent from Media Foundation. +Several cameras: a `webcams` list records up to four cameras, each into its own MP4, all on the same T0 as the screen and audio. Each entry takes `camDeviceId`, `camDeviceName`, `camClsid` (the DirectShow filter CLSID), `camWidth`, `camHeight`, `camFps` and `camPath`; an entry without `camPath` is skipped. When the list is present and non-empty it replaces the legacy `webcam*` fields; without it those fields still describe one camera, which writes to `webcamPath` when one is given and is drawn into the screen as the inline picture-in-picture otherwise. + +```json +"webcams": [ + { "camDeviceId": "…", "camDeviceName": "Camera A", "camClsid": "{…}", "camWidth": 1920, "camHeight": 1080, "camFps": 30, "camPath": "C:\\path\\recording-123-webcam.mp4" }, + { "camDeviceId": "…", "camDeviceName": "Camera B", "camClsid": "", "camWidth": 1280, "camHeight": 720, "camFps": 30, "camPath": "C:\\path\\recording-123-webcam-2.mp4" } +] +``` + +Every per-camera event carries the camera's `index`, its position in the list after entries without `camPath` have been skipped: `webcam-format` (`{"event":"webcam-format","schemaVersion":2,"index":0,"width":…,"height":…,"fps":…,"deviceName":"…"}`) for each camera that opened, and `{"event":"warning","code":"webcam-unavailable","index":1,"deviceName":"…","message":"…"}` for each that did not (`code` comes before `index` so older substring readers still match). A camera that cannot be opened, or whose encoder or capture will not start (the message says which), is dropped and the rest of the take goes on; a camera whose encoder rejects a sample mid-take is disabled on its own, without stopping the screen or the other cameras. A camera unplugged mid-take (or whose reads keep failing for a second) is disabled the same way: its file ends at the loss, is still finalized at stop, and is left out of `webcamPaths`. `recording-stopped` keeps `webcamPath` (camera 0, when it recorded) and adds `webcamPaths`, the files of every camera still recording at stop, in index order — an empty list when every camera that wrote a file of its own was disabled mid-take, so a reader can tell "stopped early" from an older helper that never sends the key. Both are printed before the camera files are finalized (see the stop sequence), so a camera whose finalize fails is reported on stderr and by a non-zero exit, not removed from the list. Each camera finalizes in its own `[stop-timing]` step, `webcam-encoder-finalize-`. + Container: recordings are written as fragmented MP4 (`MFCreateFMPEG4MediaSink` + `MFCreateSinkWriterFromMediaSink`, `MF_MPEG4SINK_MIN_FRAGMENT_DURATION` = 1s) rather than plain MP4. A plain MP4 has no index until `IMFSinkWriter::Finalize()` writes `moov` at the very end, so when the shutdown watchdog force-exits a wedged helper the file on disk holds every frame and no way to read them — that is why issues #252 / #292 / #327 cost the whole recording rather than the frozen tail of it. A fragmented MP4 writes its index up front and its samples in self-describing `moof`+`mdat` pairs, so the same kill leaves a file that plays up to the last complete fragment. This does not fix the freeze; it removes the data loss the freeze causes. Because the fragmented sink needs both output media types at construction, the sink writer is built from a media sink instead of from a URL, and the helper reads the video/audio stream positions back off the sink rather than assuming them. If any of that is unavailable on a machine, the helper retries with the plain container and says so — `container` in the `encoder-selection` event is `fragmented-mp4` or `mp4`, and it reports what was used, not what was asked for. Encoder selection: by default the helper keeps the existing sink-writer path first. If that path fails while setting up H.264, it retries with the Microsoft software H.264 encoder (`mfh264enc.dll`). The key of this retry is registering that encoder locally in the helper process via `MFTRegisterLocalByCLSID`, which makes a software H.264 encoder available even when the machine's hardware encoders are missing or broken; hardware transforms are disabled for the retry only as a secondary guard so the sink writer prefers the locally registered software encoder, not as the fallback mechanism itself. Set `preferSoftwareEncoder: true` in the helper JSON, or set `OPENSCREEN_WGC_PREFER_SOFTWARE_ENCODER=true` before launching Electron, to force the software path from the first attempt. @@ -97,6 +108,13 @@ At startup the helper also emits `capture-adapter`, naming the GPU its D3D devic Desktop icons: `"hideDesktopIcons": true` on a display capture puts a borderless window, painted with the wallpaper by `PaintDesktop`, on the captured monitor directly above the window that hosts the icons (`SHELLDLL_DefView`'s top-level parent) and below every app window (`src/desktop_icon_cover.cpp`). WGC cannot exclude another process's window, so covering is the only route that changes no setting. The window belongs to the helper, so it goes away with the recording or with a crashed helper; there is nothing to restore. Windows notifications cannot be kept out: toasts are drawn above every app window, and the only switch is Do Not Disturb, a user setting the helper does not touch. +Menus and popups in window recordings: a window source is captured per HWND, and a context menu or dropdown is a window of its own, so it used to be missing from the video (#894). On Windows 11 24H2 (build 26100) and later the helper sets `GraphicsCaptureSession.IncludeSecondaryWindows` (`IGraphicsCaptureSession6`) on window sources, and DWM draws the popups belonging to the captured window into the same texture: the frame size, DPI handling, occlusion immunity and the fixed frame pool are unchanged. Monitor sources never set it, since they already contain everything on the screen. Four things to know: + +- **Popups are clipped at the window's edge.** The texture is the window's size, so the part of a menu that overhangs it is not recorded (measured: a 276 px menu opened 138 px inside the right edge shows 138 px, cut at the last column). +- **It needs 24H2.** Older runtimes keep capturing the window alone, as before. The helper reports the outcome as `{"event":"secondary-windows","applied":true|false}` (absent for monitor sources and when the option is disabled); the app logs it and shows nothing. Set `OPENSCREEN_WGC_DISABLE_SECONDARY_WINDOWS=1` to turn the option off and get the window-only capture back. +- **What DWM lets in was measured, because Microsoft's rule is style and z-order, not process.** It draws windows with `WS_POPUP` or `WS_EX_TOOLWINDOW` that overlap the window and have only other such windows between them and it. On build 26200, three unrelated TopMost popups from other processes (`WS_POPUP`, `WS_POPUP` plus `WS_EX_TOOLWINDOW`, and `WS_EX_TOOLWINDOW` alone), all overlapping the window with nothing else above it, stayed out of the recording, as did Explorer's own Win+X menu; the window's own owned menu was in. `npm run test:wgc-window-popup:win` reproduces the measurement (menu present, strangers absent, overhang clipped, kill switch removes the menu) and refuses to pass if a non-qualifying window sits above its target, since a stranger missing for that reason would prove nothing. +- **The menu shows up only when no ordinary window sits above the recorded window.** If another window (neither a popup nor a tool window) overlaps it, the menu is missing from the first frame to the last, which is expected from how DWM selects secondary windows (#910). Measured on a second machine (Windows 11 build 26200, AMD, 46 runs of the fixture): the menu was captured in all 25 runs with nothing ordinary above the window, and absent in all 21 runs with one above it (Claude Desktop, Notepad); in the 20 valid runs at 2000 ms and 5000 ms no unrelated popup leaked (0 px). That is 0 leaks in 20 runs, which does not exclude a failure rate up to about 15 percent, and the fixture is synthetic WinForms, not a real application's menus. + Encoder diagnostic on final sink-writer failure: when the final sink-writer attempt fails (`MFCreateSinkWriterFromMediaSink` on the fragmented container, `MFCreateSinkWriterFromURL` on the plain one; the message names which), the helper logs the registered H.264 video encoder MFT count (via `MFTEnumEx`), the registered AAC encoder count when audio was requested, and the hex HRESULT. If no H.264 encoder is registered, it additionally emits the four-bullet actionable error (missing Media Feature Pack / GPU driver registration / empty `HKLM:\SOFTWARE\Microsoft\Windows Media Foundation\Transforms` / reboot). If an H.264 encoder IS registered but the sink writer still failed, it logs a hint pointing at invalid output path, missing MP4 mux, or GPU driver incompatibility. There is still no fail-fast pre-flight gate because `MFTEnumEx` and the sink writer can disagree about which H.264 encoders are available in non-interactive / Session 0 contexts. Smoke-test the helper with: @@ -104,6 +122,7 @@ Smoke-test the helper with: ```powershell npm run test:wgc-helper:win npm run test:wgc-window:win +npm run test:wgc-window-popup:win npm run test:wgc-audio:win npm run test:wgc-mic:win npm run test:wgc-mixed-audio:win @@ -118,6 +137,12 @@ npm run test:wgc-webcam:win Remove-Item Env:OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_NAME ``` +To check that a camera which cannot be opened costs only itself, record the real camera next to a nonexistent second one listed through `webcams`: + +```powershell +npm run test:wgc-helper:win -- --webcam --missing-second-webcam +``` + To validate a specific native microphone manually: ```powershell diff --git a/electron/native/compositor-view/addon.d.ts b/electron/native/compositor-view/addon.d.ts index 1365846c7..cbc1b8d79 100644 --- a/electron/native/compositor-view/addon.d.ts +++ b/electron/native/compositor-view/addon.d.ts @@ -34,6 +34,33 @@ export interface NativeFramePacket { width: number; height: number; data: Buffer; + /** The footage in this frame: its TL, TR, BR, BL corners (x, y as fractions of the frame, + * eight numbers), where a privacy blur's gimbal goes. Absent before anything is composed. */ + footage?: number[] | null; + /** The footage maps from those corners by their homography (real camera), not bilinearly. */ + footageProjective?: boolean; + /** Where the view was when it composed this frame: the active clip in `scene.clips` and the + * screen frame's time in its source file. Absent from an older `.node`. */ + clipIndex?: number; + sourceTimeSec?: number; +} + +/** A preview frame left in a shared GPU texture instead of copied into RAM (Windows, hardware + * backend — see `setSharedFrames`). `handle` is the texture's NT handle the way Electron's + * `sharedTexture.importSharedTexture` takes it: 8 bytes, little-endian, valid in this process + * only. Its `slot` stays reserved until `releaseSharedFrame(id, slot, gen)`. */ +export interface NativeSharedFramePacket { + gen: number; + slot: number; + handle: Buffer; + width: number; + height: number; + footage?: number[] | null; + footageProjective?: boolean; + /** Where the view was when it composed this frame: the active clip in `scene.clips` and the + * screen frame's time in its source file. Absent from an older `.node`. */ + clipIndex?: number; + sourceTimeSec?: number; } export interface ExportStats { @@ -91,6 +118,14 @@ export interface ExportParamsInput { bitrate?: number; } +/** An additional camera (2-4) of a clip, for `setActiveClip` and the export clip list. + * = napi `ClipCameraInput`. */ +export interface ClipCameraInput { + path: string; + /** Camera source time = screen source time - this. */ + offsetSec: number; +} + /** One timeline clip for the native multiclip export (screen + webcam files + source trim). */ export interface ClipInput { screenPath: string; @@ -99,6 +134,8 @@ export interface ClipInput { sourceEndSec: number; /** webcam source time = screen source time − this. */ webcamOffsetSec: number; + /** Cameras 2-4 (index k-1 = camera k). Only those a layout region shows are decoded. */ + additionalCameras?: ClipCameraInput[]; } /** Which backend the compositor will run on. `"cpu"` = WARP rasterisation + software @@ -153,6 +190,17 @@ export interface CompositorViewAddon { * still frame): `null` comes back WITHOUT cloning the buffer or crossing IPC. * Pass `sinceGen = 0` to force delivery of the current frame. */ readFrame(id: number, sinceGen: number): NativeFramePacket | null; + /** Deliver this view's frames as shared GPU textures (`readSharedFrame`) rather than RAM + * pixels (`readFrame`). Returns `false` where that cannot work (not Windows, software + * backend): the view keeps reading back. Optional: an older `.node` predates it. */ + setSharedFrames?(id: number, enabled: boolean): boolean; + /** The latest frame left in the view's shared texture ring, if newer than `sinceGen`. + * `null` too while the view reads back to RAM. Throws the render thread's fatal error, + * like `readFrame`. */ + readSharedFrame?(id: number, sinceGen: number): NativeSharedFramePacket | null; + /** Chromium let go of frame `gen` in `slot`, in every process: the render thread may write + * that slot again. A no-op for a destroyed view. */ + releaseSharedFrame?(id: number, slot: number, gen: number): void; setParam(id: number, key: string, value: CompositorParamValue): void; setPlaying(id: number, playing: boolean): void; /** Seeks the view to source-media `seconds` for the active clip. */ @@ -160,6 +208,8 @@ export interface CompositorViewAddon { /** Installs the app scene (JSON `SceneDescription`) — layout preset etc. drive the render * instead of the fixture. Invalid JSON is ignored native-side. */ setScene(id: number, sceneJson: string): void; + /** `additionalCameras`: cameras 2-4 of the clip (index k-1 = camera k, empty `path` = none). + * An addon built before it ignores the argument. */ setActiveClip( id: number, screenPath: string, @@ -167,6 +217,7 @@ export interface CompositorViewAddon { webcamOffsetSec: number, clipIndex: number, sourceTimeSec: number, + additionalCameras?: ClipCameraInput[], ): void; destroyView(id: number): void; /** Renders the fixture to `outPath` (C8), auto-pausing live previews. `onProgress` diff --git a/electron/native/pipewire-capture/src/dmabuf_import.rs b/electron/native/pipewire-capture/src/dmabuf_import.rs index 4dc60412f..59e900a84 100644 --- a/electron/native/pipewire-capture/src/dmabuf_import.rs +++ b/electron/native/pipewire-capture/src/dmabuf_import.rs @@ -266,8 +266,10 @@ impl DmabufImporter { // Output size = the recorded (out) size. For a monitor that equals the // source; for a window it is the crop rectangle, and the per-frame crop // fields set in `import` pick which region of the source is scaled into it. + // BT.709 studio range out, as `ensure_sws` converts the CPU path and as + // the compositor decodes; left to the driver it is not guaranteed (#926). let scale_args = std::ffi::CString::new(format!( - "w={}:h={}:format=nv12", + "w={}:h={}:format=nv12:out_color_matrix=bt709:out_range=limited: out_color_primaries=bt709:out_color_transfer=bt709", self.out_width, self.out_height )) .map_err(|_| "scale_vaapi args contained a NUL".to_owned())?; diff --git a/electron/native/pipewire-capture/src/encoder.rs b/electron/native/pipewire-capture/src/encoder.rs index a2e041bfe..05191862a 100644 --- a/electron/native/pipewire-capture/src/encoder.rs +++ b/electron/native/pipewire-capture/src/encoder.rs @@ -355,6 +355,12 @@ impl VideoEncoder { (*codec_ctx).framerate = ff::AVRational { num: params.fps, den: 1 }; (*codec_ctx).pix_fmt = backend.codec_pixel_format(); (*codec_ctx).bit_rate = params.bitrate; + // What the samples are (see `ensure_sws` and the dmabuf VPP), said in + // the stream so no player has to guess from the frame size (#926). + (*codec_ctx).color_range = ff::AVCOL_RANGE_MPEG; + (*codec_ctx).colorspace = ff::AVCOL_SPC_BT709; + (*codec_ctx).color_primaries = ff::AVCOL_PRI_BT709; + (*codec_ctx).color_trc = ff::AVCOL_TRC_BT709; // Half a second, not the two seconds a streaming preset would use. // // This file is an EDITING SOURCE, and the editor scrubs it. Seeking @@ -787,6 +793,17 @@ impl VideoEncoder { ff::err_to_string(initialised) )); } + // BT.709 studio range, which is what the compositor decodes every + // recording as. swscale's default matrix is BT.601, so left alone every + // Linux take played back with shifted hue and saturation + // (getopenscreen/openscreen#926). Full range in: the source is RGB. + let bt709 = ff::sws_getCoefficients(ff::SWS_CS_ITU709 as i32); + let unity = 1 << 16; + if ff::sws_setColorspaceDetails(self.sws, bt709, 1, bt709, 0, 0, unity, unity) < 0 { + return Err(format!( + "swscale refused a BT.709 conversion from pixel format {src_format} to {dst_format}" + )); + } // Read back rather than assume: libswscale is free to ignore a thread // count it cannot use for a given conversion, and a request that was // silently dropped looks exactly like threading that did not help. @@ -1053,6 +1070,40 @@ struct MuxTrack { stream_time_base: ff::AVRational, } +/// The mov muxer options that make the output fragmented. +/// +/// A fragment is cut at a video keyframe once it holds at least one second, the +/// same floor the Windows sink uses (`kFragmentDurationHns` in +/// wgc-capture/src/mf_encoder.cpp). The GOP is half a second (see +/// `VideoEncoder::open_backend`), so a fragment is two GOPs and a kill loses at +/// most about a second. `frag_keyframe` alone would cut every half second; +/// `frag_duration` alone would cut mid-GOP, leaving fragments that open on a +/// frame nothing can decode without the previous one. `min_frag_duration` is +/// the floor that waits for the keyframe. `default_base_moof` makes each moof's +/// data offsets relative to that moof rather than to the file, so a fragment +/// parses on its own. +/// +/// `flush_packets` hands every closed fragment to the kernel at once. Without +/// it the moov and the fragments wait in avio's 32 KB buffer, and a kill loses +/// that too: measured, a low-bitrate take killed after three seconds left a +/// 28-byte file. It costs one write per fragment, since the muxer holds each +/// fragment's samples until it closes. +/// +/// `delay_moov` is for the audio track. AAC's first packet sits 1024 samples +/// before zero (encoder priming), and an empty moov written with the header +/// cannot know that, so libavformat drops the edit list and shifts every track +/// 21 ms later instead: measured, the first video frame then lasts 55 ms and +/// every later one lands 21 ms after its capture time, against a cursor track +/// that does not move. Delaying the moov to the first fragment lets it carry +/// the same edit list the plain MP4 had; timestamps match it packet for packet. +/// The price is that a take killed in its first second keeps nothing, as every +/// killed take did before. +const FRAGMENT_OPTIONS: [(&CStr, &CStr); 3] = [ + (c"movflags", c"+frag_keyframe+empty_moov+default_base_moof+delay_moov"), + (c"min_frag_duration", c"1000000"), + (c"flush_packets", c"1"), +]; + /// An MP4 writer, taking however many streams [`Self::add_stream`] is called for. /// /// WHAT A RECORDING ACTUALLY USES IS TWO: one H.264 video stream and one AAC @@ -1067,11 +1118,15 @@ struct MuxTrack { /// and it also removes the question of reconciling two capture clocks in the /// container: there is only one audio timeline now, built in `AudioMix::pump`. /// +/// The file is a FRAGMENTED MP4, like the Windows and macOS helpers write +/// (#338): a moov with no samples in front, then self-describing moof+mdat pairs. A plain +/// MP4 has no index until the trailer, so a killed helper, a crash or a power +/// loss used to leave gigabytes of frames and no way to read them (#944). Now it +/// plays up to the last complete fragment. See [`FRAGMENT_OPTIONS`]. +/// /// `+faststart` is not used: it rewrites the whole file on close, which on a -/// long recording means copying gigabytes. The moov atom is written at the end -/// as usual, and the app reads these files locally, where a trailing moov costs -/// nothing. This is the difference from the WebM path, which had NO index at all -/// and could not be seeked even locally (see electron/recording/webm-seek-index.ts). +/// long recording means copying gigabytes, and a fragmented file has its moov +/// in front anyway. pub struct Muxer { fmt: *mut ff::AVFormatContext, tracks: Vec, @@ -1153,9 +1208,21 @@ impl Muxer { if self.tracks.is_empty() { return Err("the output has no streams".to_owned()); } - // SAFETY: `fmt` is ours and every stream was added through add_stream. + // SAFETY: `fmt` is ours and every stream was added through add_stream; + // the dictionary is ours and freed before leaving the block. unsafe { - let header = ff::avformat_write_header(self.fmt, ptr::null_mut()); + let mut options: *mut ff::AVDictionary = ptr::null_mut(); + for (key, value) in FRAGMENT_OPTIONS { + ff::av_dict_set(&mut options, key.as_ptr(), value.as_ptr(), 0); + } + let header = ff::avformat_write_header(self.fmt, &mut options); + // The muxer takes out every option it knows; one left over is a typo + // that would silently bring back the unfragmented file. + let unknown = ff::av_dict_count(options); + ff::av_dict_free(&mut options); + if header >= 0 && unknown > 0 { + return Err(format!("the mp4 muxer ignored {unknown} fragment option(s)")); + } if header < 0 { return Err(format!( "avformat_write_header: {}", @@ -1217,7 +1284,7 @@ impl Drop for Muxer { fn drop(&mut self) { // SAFETY: `fmt` is either null or ours. `header_written` is still true // only on the error path — `finish` clears it — and a file left without - // its trailer would be unplayable, so write one on the way out. + // its trailer would lose its last fragment, so write one on the way out. unsafe { if self.fmt.is_null() { return; @@ -1423,6 +1490,111 @@ mod tests { assert!(size > 1024, "the muxed file is {size} bytes, which cannot be 120 frames"); } + #[test] + fn software_path_converts_and_tags_bt709() { + // The compositor decodes every recording as BT.709 studio range; swscale's + // default is BT.601, which puts pure red at Y 81, Cb 90 (#926). + let (width, height) = (64, 32); + let mut encoder = VideoEncoder::open( + VideoParams { width, height, fps: 30, bitrate: 2_000_000 }, + Some(Backend::Software), + |_, _| {}, + ) + .expect("the software encoder always opens"); + let stride = width as usize * 4; + let mut frame = vec![0u8; stride * height as usize]; + for pixel in frame.chunks_exact_mut(4) { + pixel.copy_from_slice(&[0, 0, 255, 255]); // BGRA pure red + } + encoder.stage(&frame, stride, ff::AV_PIX_FMT_BGRA).expect("stage"); + // SAFETY: `stage` just filled `sw_frame`, a YUV420P frame of this size. + let (y, cb, cr, ctx) = unsafe { + let staged = encoder.sw_frame; + (*(*staged).data[0], *(*staged).data[1], *(*staged).data[2], &*encoder.codec_ctx) + }; + let close = |got: u8, want: i32| (i32::from(got) - want).abs() <= 1; + assert!(close(y, 63) && close(cb, 102) && close(cr, 240), "red came out Y {y} Cb {cb} Cr {cr}"); + assert_eq!(ctx.color_range, ff::AVCOL_RANGE_MPEG); + assert_eq!(ctx.colorspace, ff::AVCOL_SPC_BT709); + assert_eq!(ctx.color_primaries, ff::AVCOL_PRI_BT709); + assert_eq!(ctx.color_trc, ff::AVCOL_TRC_BT709); + } + + #[test] + fn a_killed_muxer_leaves_a_file_that_opens_and_holds_its_frames() { + // Three seconds at 30 fps with audio, then a simulated kill: no trailer, + // no flush of the write buffer, no interleave queue drained (#944). + let (width, height, fps) = (64, 32, 30); + // Per process, so two test runs on one machine never share the file. + let output = std::env::temp_dir() + .join(format!("openscreen-killed-muxer-{}.mp4", std::process::id())); + let _ = std::fs::remove_file(&output); + let mut video = VideoEncoder::open( + VideoParams { width, height, fps, bitrate: 1_000_000 }, + Some(Backend::Software), + |_, _| {}, + ) + .expect("the software encoder always opens"); + let mut audio = AudioEncoder::open(128_000).expect("aac"); + let mut muxer = Muxer::create(&output).expect("muxer"); + let video_track = muxer.add_stream(video.codec_context()).expect("video track"); + let audio_track = muxer.add_stream(audio.codec_context()).expect("audio track"); + muxer.write_header().expect("header"); + + let stride = width as usize * 4; + let mut frame = vec![0u8; stride * height as usize]; + let samples = vec![0.0f32; (AUDIO_SAMPLE_RATE / fps) as usize * AUDIO_CHANNELS]; + for pts in 0..3 * i64::from(fps) { + frame.fill(pts as u8); + video + .submit(&frame, stride, ff::AV_PIX_FMT_BGRA, pts, |p| muxer.write(video_track, p)) + .expect("video"); + audio.push(&samples, |p| muxer.write(audio_track, p)).expect("audio"); + } + std::mem::forget(muxer); + + // SAFETY: ffmpeg's own demuxer on the file just written; the context is + // closed before the block ends. + let (streams, video_frames, second_frame_at) = unsafe { + let path = CString::new(output.as_os_str().as_encoded_bytes()).unwrap(); + let mut input: *mut ff::AVFormatContext = ptr::null_mut(); + let opened = + ff::avformat_open_input(&mut input, path.as_ptr(), ptr::null(), ptr::null_mut()); + assert!(opened >= 0, "a killed take must open: {}", ff::err_to_string(opened)); + let probed = ff::avformat_find_stream_info(input, ptr::null_mut()); + assert!(probed >= 0, "stream info: {}", ff::err_to_string(probed)); + let packet = ff::av_packet_alloc(); + let mut video_frames = 0; + let mut second_frame_at = 0.0; + while ff::av_read_frame(input, packet) >= 0 { + let stream = *(*input).streams.add((*packet).stream_index as usize); + if (*(*stream).codecpar).codec_type == ff::AVMEDIA_TYPE_VIDEO { + if video_frames == 1 { + let base = (*stream).time_base; + second_frame_at = (*packet).pts as f64 * f64::from(base.num) + / f64::from(base.den); + } + video_frames += 1; + } + ff::av_packet_unref(packet); + } + let mut packet = packet; + ff::av_packet_free(&mut packet); + let streams = (*input).nb_streams; + ff::avformat_close_input(&mut input); + (streams, video_frames, second_frame_at) + }; + assert_eq!(streams, 2, "both tracks are declared up front"); + // Fragments close at the keyframes at 1 s and 2 s; only the one in + // flight, the last second, is lost. + assert!(video_frames >= 2 * fps, "only {video_frames} frames survived the kill"); + // Without an edit list for AAC's priming, every frame after the first + // slides 21 ms off its capture time (see FRAGMENT_OPTIONS). + let one_frame = 1.0 / f64::from(fps); + assert!((second_frame_at - one_frame).abs() < 1e-3, "frame 1 at {second_frame_at} s"); + let _ = std::fs::remove_file(&output); + } + #[test] fn radv_perftest_is_added_without_clobbering_an_existing_value() { std::env::set_var("RADV_PERFTEST", "gpl"); diff --git a/electron/native/pipewire-capture/src/main.rs b/electron/native/pipewire-capture/src/main.rs index 44d96ae7d..13c987ae9 100644 --- a/electron/native/pipewire-capture/src/main.rs +++ b/electron/native/pipewire-capture/src/main.rs @@ -662,8 +662,31 @@ fn run( // means the import never works, which would otherwise record nothing — see // MAX_CONSECUTIVE_IMPORT_FAILURES. let mut consecutive_drops: u32 = 0; + // When the capture clock last advanced. See the heartbeat at the top of the loop. + let mut last_advance = Instant::now(); loop { + // THE HEARTBEAT, CHECKED ON EVERY PASS, NOT ONLY ON A TIMEOUT. `advance` + // holds the last picture forward and drains the audio rings, and it used + // to run only on a new frame or when `recv_timeout` expired. Any other + // message restarts that timeout, and cursor messages arrive at the + // compositor's rate while the mouse moves: a static screen with a moving + // mouse never timed out, so nothing drained and the 2 s rings dropped + // their oldest audio (getopenscreen/openscreen#936). + if heartbeat_due(last_advance, Instant::now(), config.tick) { + last_advance = Instant::now(); + if let Some(capture) = capture.as_mut() { + if let Err(message) = capture.advance() { + let _ = emitter.emit(&Event::Error { + code: "encode-failed".to_owned(), + message, + }); + exit_code = 1; + break; + } + } + } + // Return PipeWire buffers whose dmabuf imports completed last iteration // (or that were superseded on the capture thread). This MUST run on this // loop, not the PipeWire thread — `Session::requeue` takes the thread-loop @@ -918,6 +941,7 @@ fn run( } } if let Some(capture) = capture.as_mut() { + last_advance = Instant::now(); if let Err(message) = capture.advance() { let _ = emitter.emit(&Event::Error { code: "encode-failed".to_owned(), @@ -1239,19 +1263,8 @@ fn run( emit_sample(emitter, &cursor, content_rect(&capture, size), &mut pending_asset, None); last_emit = Instant::now(); } - // The heartbeat that keeps the output at a constant frame rate - // while the screen is static: no frame arrived, but the clock - // moved, so the last picture is held forward. - if let Some(capture) = capture.as_mut() { - if let Err(message) = capture.advance() { - let _ = emitter.emit(&Event::Error { - code: "encode-failed".to_owned(), - message, - }); - exit_code = 1; - break; - } - } + // No advance here: a timeout means a tick has passed, so the + // heartbeat at the top of the next pass holds the picture forward. } Err(RecvTimeoutError::Disconnected) => break, @@ -1271,9 +1284,9 @@ fn run( /// Writes the trailer and reports what the recording cost. /// -/// Runs even when the loop broke on an error: a file whose moov atom was never -/// written is unplayable, and a partial recording is worth more to the user than -/// none. +/// Runs even when the loop broke on an error: a file whose trailer was never +/// written loses its last fragment, and a partial recording is worth more to the +/// user than none. fn finish_capture( emitter: &mut Emitter, capture: Capture, @@ -1349,6 +1362,11 @@ fn finish_capture( /// only rectangle the file shows. Before then, and for a cursor-only session /// that opens no encoder at all, the whole negotiated stream, which is what the /// consumer of a cursor-only recording is compositing over. +/// Whether a tick has passed since the capture clock last advanced. +fn heartbeat_due(last_advance: Instant, now: Instant, tick: Duration) -> bool { + now.saturating_duration_since(last_advance) >= tick +} + fn content_rect(capture: &Option, size: Option<(i32, i32)>) -> Option { match capture { Some(capture) if capture.started() => Some(capture.content_rect()), @@ -1563,6 +1581,28 @@ mod cursor_sample_tests { assert!(buffer.is_empty(), "emitted {}", String::from_utf8_lossy(&buffer)); } + /// #936: messages arriving faster than the tick must not hold the heartbeat + /// off. The loop checks it on every pass, so what matters is that it comes + /// due once a tick has passed, however many passes happened in between. + #[test] + fn the_heartbeat_comes_due_every_tick_whatever_else_arrives() { + let tick = Duration::from_millis(16); + let start = Instant::now(); + let mut last_advance = start; + let mut advances = 0; + // A cursor message every 7 ms (144 Hz) for one second of loop passes. + for pass in 0..=(1000 / 7) { + let now = start + Duration::from_millis(pass * 7); + if heartbeat_due(last_advance, now, tick) { + last_advance = now; + advances += 1; + } + } + // Every 21 ms at worst (three 7 ms passes): about 47 a second, never zero. + assert!(advances >= 1000 / 21, "{advances} advances in a second of cursor traffic"); + assert!(!heartbeat_due(start, start + Duration::from_millis(15), tick)); + } + /// A cursor-only session opens no encoder, so it falls back to the whole /// stream; once pixels are flowing the encoder's rectangle wins. #[test] diff --git a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/CaptureSizing.swift b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/CaptureSizing.swift index 9741d9559..0b69fedfe 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/CaptureSizing.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/CaptureSizing.swift @@ -72,3 +72,24 @@ private func evenFloor(_ value: Int) -> Int { let clamped = max(2, value) return clamped - (clamped % 2) } + +/// The video bitrate for the size the stream actually got, the rule the Linux helper uses +/// (`default_bitrate` in `pipewire-capture/src/capture.rs`): 0.1 bit per pixel per frame, +/// about 12 Mbit/s at 1920×1080/60, kept between 2 and 60 Mbit/s. +/// +/// The renderer used to send `computeBitrate(TARGET_WIDTH, TARGET_HEIGHT)`, the 4K ceiling with +/// its high-frame-rate boost: 76.5 Mbit/s for every take, a 1080p one included, 4.25× what +/// Windows spends on the same capture (getopenscreen/openscreen#924). It cannot know the real +/// size, which `captureOutputSize` settles here, so it now sends none. +public func defaultVideoBitrate(width: Int, height: Int, fps: Int) -> Int { + let bitsPerPixel = 0.1 + let pixelsPerSecond = Double(max(1, width)) * Double(max(1, height)) * Double(max(1, fps)) + return min(max(Int(pixelsPerSecond * bitsPerPixel), 2_000_000), 60_000_000) +} + +/// The H.264 keyframe interval, in frames: one second of video, the interval Windows uses. +/// Left to VideoToolbox it had no bound, and the editor's scrub cost is the GOP length: a +/// seek decodes from the keyframe before it (getopenscreen/openscreen#937). +public func videoKeyFrameInterval(fps: Int) -> Int { + max(1, fps) +} diff --git a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/VideoColor.swift b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/VideoColor.swift new file mode 100644 index 000000000..35b30eed0 --- /dev/null +++ b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/VideoColor.swift @@ -0,0 +1,22 @@ +import AVFoundation +import CoreGraphics + +// The colour every recording is decoded as: BT.709 primaries, transfer and matrix, studio +// range. The compositor assumes it for every file, and the Windows (#929) and Linux (#932) +// helpers write it. Left to the defaults, ScreenCaptureKit hands over the display's own colour +// space (P3 on most Macs) and VideoToolbox picks the matrix and the tags +// (getopenscreen/openscreen#943). +// +// Range has no key here: it comes from the pixel format, and the stream captures `420v`, the +// studio-range one. + +/// The matrix ScreenCaptureKit converts to YCbCr with. It has to be the one the file is tagged +/// with in `videoColorProperties`, which `VideoColorTests` pins. +public let captureYCbCrMatrix: CFString = CGDisplayStream.yCbCrMatrix_ITU_R_709_2 + +/// `AVVideoColorPropertiesKey` for the H.264 writer input. +public let videoColorProperties: [String: String] = [ + AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_709_2, + AVVideoTransferFunctionKey: AVVideoTransferFunction_ITU_R_709_2, + AVVideoYCbCrMatrixKey: AVVideoYCbCrMatrix_ITU_R_709_2, +] diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift index 4e0723f55..f08c447fd 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift @@ -152,6 +152,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { private var totalPausedDuration = CMTime.zero /// Sample queue only. See `VideoTimestampGate` for the failure it exists to prevent. private var videoTimestampGate = VideoTimestampGate() + /// Sample queue only. Frames the writer input was not ready for, or refused, reported in + /// `recording-stopped` so a macOS drop rate can be measured at all (#937). + private var droppedVideoFrames = 0 private var nativeMicrophoneEnabled = false private var outputWidth = 1920 private var outputHeight = 1080 @@ -454,8 +457,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { "captureBounds": captureBoundsPayload(), ]) } else if !appended { + droppedVideoFrames += 1 reportWriterFailure("video append") } + } else { + droppedVideoFrames += 1 } } @@ -701,7 +707,15 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(max(1, request.video.fps))) configuration.queueDepth = 6 configuration.showsCursor = !request.video.hideSystemCursor - configuration.pixelFormat = kCVPixelFormatType_32BGRA + // Studio-range BT.709 YCbCr from ScreenCaptureKit itself, which the encoder takes as is + // and the compositor decodes (#943). BGRA left the matrix to VideoToolbox. Nothing here + // reads the pixels: a frame goes through `retimedSampleBuffer` (timing only) to + // `append`, and `isCompleteFrame` reads attachments. 1.5 bytes a pixel instead of 4, + // times `queueDepth`. + configuration.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + configuration.colorMatrix = captureYCbCrMatrix + // Unset, the buffers carry the display's colour space: P3 on most Macs. + configuration.colorSpaceName = CGColorSpace.sRGB configuration.sampleRate = 48_000 configuration.channelCount = 2 configuration.excludesCurrentProcessAudio = true @@ -754,8 +768,12 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: outputWidth, AVVideoHeightKey: outputHeight, + // BT.709 tags, the colour the stream is captured in (#943). + AVVideoColorPropertiesKey: videoColorProperties, AVVideoCompressionPropertiesKey: [ - AVVideoAverageBitRateKey: request.video.bitrate ?? 18_000_000, + // From the size this stream really got. The renderer sends none (#924). + AVVideoAverageBitRateKey: request.video.bitrate + ?? defaultVideoBitrate(width: outputWidth, height: outputHeight, fps: request.video.fps), AVVideoExpectedSourceFrameRateKey: request.video.fps, // Without this the encoder defaults to B-frames, and a reordered // stream needs a composition offset per sample. AVAssetWriter emits @@ -784,6 +802,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { // particular run happened to die. Reordering off is 3/3 clean across // both rates, and a SIGKILL at 25s still leaves 27 readable `moof`. AVVideoAllowFrameReorderingKey: false, + // A keyframe every second, by frame count and by time: a still screen + // delivers fewer frames than the fps, and the count alone would then + // stretch the GOP, which is the editor's scrub cost (#937). + AVVideoMaxKeyFrameIntervalKey: videoKeyFrameInterval(fps: request.video.fps), + AVVideoMaxKeyFrameIntervalDurationKey: 1, ], ] let input = AVAssetWriterInput(mediaType: .video, outputSettings: settings) @@ -906,7 +929,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } } - let refusedVideoFrames = sampleQueue.sync { videoTimestampGate.rejectedCount } + let (refusedVideoFrames, droppedFrames) = sampleQueue.sync { + (videoTimestampGate.rejectedCount, droppedVideoFrames) + } if refusedVideoFrames > 1 { emit([ "event": "warning", @@ -929,6 +954,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { emit([ "event": "recording-stopped", "screenPath": request.outputs.screenPath, + "droppedVideoFrames": droppedFrames, ]) } else { emitError( diff --git a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/CaptureSizingTests.swift b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/CaptureSizingTests.swift index 53029a31a..0d318585f 100644 --- a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/CaptureSizingTests.swift +++ b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/CaptureSizingTests.swift @@ -134,4 +134,31 @@ final class CaptureSizingTests: XCTestCase { XCTAssertEqual(result.height, 1080, "scale \(scale)") } } + + // MARK: - Bitrate from the real size (#924) + + func testBitrateFollowsTheCapturedSizeNotTheCeiling() { + // 1080p60 used to be asked for 76.5 Mbit/s, the 4K ceiling's figure. + XCTAssertEqual(defaultVideoBitrate(width: 1920, height: 1080, fps: 60), 12_441_600) + XCTAssertEqual(defaultVideoBitrate(width: 2560, height: 1440, fps: 60), 22_118_400) + XCTAssertEqual(defaultVideoBitrate(width: 3840, height: 2160, fps: 60), 49_766_400) + } + + func testBitrateStaysWithinItsFloorAndCeiling() { + // A small window still gets enough bits to stay sharp; 4K at 120 fps does not ask for + // more than any disk wants. + XCTAssertEqual(defaultVideoBitrate(width: 320, height: 240, fps: 30), 2_000_000) + XCTAssertEqual(defaultVideoBitrate(width: 3840, height: 2160, fps: 120), 60_000_000) + } + + // MARK: - Keyframe interval (#937) + + func testKeyFrameIntervalIsOneSecondOfFrames() { + XCTAssertEqual(videoKeyFrameInterval(fps: 60), 60) + XCTAssertEqual(videoKeyFrameInterval(fps: 30), 30) + } + + func testKeyFrameIntervalNeverFallsBelowOneFrame() { + XCTAssertEqual(videoKeyFrameInterval(fps: 0), 1) + } } diff --git a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/VideoColorTests.swift b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/VideoColorTests.swift new file mode 100644 index 000000000..2a423d2b9 --- /dev/null +++ b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/VideoColorTests.swift @@ -0,0 +1,18 @@ +import AVFoundation +import XCTest + +import OpenScreenCaptureCore + +/// The file has to say what its samples are: a matrix that converts with one standard and tags +/// another is the shifted-hue defect #943 exists to remove. +final class VideoColorTests: XCTestCase { + func testCaptureMatrixIsTheOneTheFileIsTaggedWith() { + XCTAssertEqual(captureYCbCrMatrix as String, videoColorProperties[AVVideoYCbCrMatrixKey]) + } + + func testFileIsTaggedBT709() { + XCTAssertEqual(videoColorProperties[AVVideoColorPrimariesKey], "ITU_R_709_2") + XCTAssertEqual(videoColorProperties[AVVideoTransferFunctionKey], "ITU_R_709_2") + XCTAssertEqual(videoColorProperties[AVVideoYCbCrMatrixKey], "ITU_R_709_2") + } +} diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt index c0921486f..009a04677 100644 --- a/electron/native/wgc-capture/CMakeLists.txt +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -39,8 +39,15 @@ add_executable(wgc-capture src/audio_sample_utils.cpp src/audio_sample_utils.h src/desktop_icon_cover.cpp + src/device_selection.cpp + src/device_selection.h src/desktop_icon_cover.h src/dpi_awareness.h + src/realtime_scheduling.h + src/frame_visibility.cpp + src/frame_visibility.h + src/json_fields.cpp + src/json_fields.h src/dshow_webcam_capture.cpp src/dshow_webcam_capture.h src/main.cpp @@ -55,6 +62,12 @@ add_executable(wgc-capture src/wasapi_render_keepalive.cpp src/wasapi_render_keepalive.h src/webcam_capture.cpp + src/webcam_loss.cpp + src/webcam_loss.h + src/webcam_format.cpp + src/webcam_format.h + src/webcam_config.cpp + src/webcam_config.h src/webcam_capture.h src/wgc_session.cpp src/wgc_session.h @@ -69,6 +82,7 @@ target_compile_definitions(wgc-capture PRIVATE target_compile_options(wgc-capture PRIVATE /EHsc /W4 /utf-8) target_link_libraries(wgc-capture PRIVATE + avrt d3d11 dxgi mf @@ -77,11 +91,13 @@ target_link_libraries(wgc-capture PRIVATE mfuuid runtimeobject windowsapp + winmm ) add_executable(cursor-sampler src/cursor-sampler.cpp src/dpi_awareness.h + src/realtime_scheduling.h ) target_compile_definitions(cursor-sampler PRIVATE @@ -92,8 +108,10 @@ target_compile_definitions(cursor-sampler PRIVATE target_compile_options(cursor-sampler PRIVATE /EHsc /W4 /utf-8) target_link_libraries(cursor-sampler PRIVATE + avrt gdi32 gdiplus + winmm ) add_executable(audio_sample_utils_test @@ -111,9 +129,124 @@ target_compile_definitions(audio_sample_utils_test PRIVATE target_compile_options(audio_sample_utils_test PRIVATE /EHsc /W4 /utf-8) target_link_libraries(audio_sample_utils_test PRIVATE + avrt mf mfplat mfreadwrite mfuuid ole32 + winmm ) + +add_executable(webcam_format_test + src/webcam_format.cpp + src/webcam_format.h + src/webcam_format_test.cpp +) + +target_compile_definitions(webcam_format_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(webcam_format_test PRIVATE /EHsc /W4 /utf-8) + +add_executable(frame_visibility_test + src/frame_visibility.cpp + src/frame_visibility.h + src/frame_visibility_test.cpp +) + +target_compile_definitions(frame_visibility_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(frame_visibility_test PRIVATE /EHsc /W4 /utf-8) + +add_executable(webcam_snapshot_test + src/dshow_webcam_capture.h + src/webcam_snapshot_test.cpp +) + +target_compile_definitions(webcam_snapshot_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(webcam_snapshot_test PRIVATE /EHsc /W4 /utf-8) + +add_executable(mf_encoder_color_test + src/audio_sample_utils.cpp + src/audio_sample_utils.h + src/mf_encoder.cpp + src/mf_encoder.h + src/mf_encoder_color_test.cpp +) + +target_compile_definitions(mf_encoder_color_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(mf_encoder_color_test PRIVATE /EHsc /W4 /utf-8) + +target_link_libraries(mf_encoder_color_test PRIVATE + avrt + d3d11 + dxgi + mf + mfplat + mfreadwrite + mfuuid + ole32 + propsys +) + +add_executable(webcam_config_test + src/json_fields.cpp + src/json_fields.h + src/webcam_config.cpp + src/webcam_config.h + src/webcam_config_test.cpp +) + +target_compile_definitions(webcam_config_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(webcam_config_test PRIVATE /EHsc /W4 /utf-8) + +add_executable(device_selection_test + src/device_selection.cpp + src/device_selection.h + src/device_selection_test.cpp +) + +target_compile_definitions(device_selection_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(device_selection_test PRIVATE /EHsc /W4 /utf-8) + +add_executable(webcam_loss_test + src/webcam_loss.cpp + src/webcam_loss.h + src/webcam_loss_test.cpp +) + +target_compile_definitions(webcam_loss_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(webcam_loss_test PRIVATE /EHsc /W4 /utf-8) diff --git a/electron/native/wgc-capture/src/audio_sample_utils.cpp b/electron/native/wgc-capture/src/audio_sample_utils.cpp index 54e77e40f..4ffe3ff50 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils.cpp +++ b/electron/native/wgc-capture/src/audio_sample_utils.cpp @@ -1,4 +1,5 @@ #include "audio_sample_utils.h" +#include "realtime_scheduling.h" #include @@ -478,6 +479,18 @@ void mixAudioInPlace( } } +namespace { + +// How far behind real time AudioMixer writes; see mixLoop. Wider than the capture +// threads' poll jitter with room for a stall, and harmless beyond that: the +// output timeline does not move with it. +// ponytail: absorbs jitter, not clock drift. A device slower than the steady clock +// spends the cushion over a long take, then leaves one gap as it re-anchors; +// resampling each source to the mixer clock is the upgrade if that is ever heard. +constexpr uint32_t MixerCushionMs = 100; + +} // namespace + AudioMixer::AudioMixer( const AudioInputFormat& format, const AudioInputFormat& systemFormat, @@ -510,34 +523,59 @@ bool AudioMixer::start() { systemDecimator_.reset(); microphoneDecimator_.reset(); thread_ = std::thread([this] { + const MmcssThread mmcss(L"Pro Audio"); mixLoop(); }); return true; } +void AudioMixer::resetSources() { + systemQueue_.clear(); + microphoneQueue_.clear(); + systemDecimator_.reset(); + microphoneDecimator_.reset(); + systemStarved_ = false; + microphoneStarved_ = false; +} + void AudioMixer::beginTimeline() { { std::scoped_lock lock(mutex_); - systemQueue_.clear(); - microphoneQueue_.clear(); - systemDecimator_.reset(); - microphoneDecimator_.reset(); + resetSources(); emittedFrames_ = 0; + mixedFrames_ = 0; timelineStarted_ = true; + // Here, not when mixLoop next wakes: the video's T0 is taken right after + // this call, and every millisecond the mixer took to wake would have + // placed the whole track that much early -- tens of them on a busy machine. + clockStart_ = std::chrono::steady_clock::now(); + clockAnchored_ = true; } cv_.notify_all(); } void AudioMixer::setPaused(bool paused) { { - std::scoped_lock lock(mutex_); - paused_ = paused; - if (paused_) { - systemQueue_.clear(); - microphoneQueue_.clear(); - systemDecimator_.reset(); - microphoneDecimator_.reset(); + std::unique_lock lock(mutex_); + if (paused && !paused_) { + // The queues still hold what the cushion kept back; mixLoop writes it + // up to this instant, and the resume clears whatever is left. + pausedAt_ = std::chrono::steady_clock::now(); + pauseFlushed_ = !timelineStarted_; + } else if (!paused && paused_) { + // A resume right behind the pause would otherwise clear the cushion + // before mixLoop has written it: the last 100 ms of voice before the + // pause, gone (measured, 80 to 90 ms). + cv_.wait(lock, [this] { return pauseFlushed_ || stopRequested_.load(); }); + resetSources(); + // Resumed at this instant, where the pause left off: the flush has + // run, so `emittedFrames_` is the pause point and mixLoop is idle. + clockStart_ = std::chrono::steady_clock::now() - + std::chrono::duration_cast(std::chrono::duration( + static_cast(emittedFrames_) / format_.sampleRate)); + clockAnchored_ = true; } + paused_ = paused; } cv_.notify_all(); } @@ -560,7 +598,7 @@ void AudioMixer::pushSystem(const BYTE* data, DWORD byteCount) { if (paused_) { return; } - append(systemQueue_, data, byteCount, systemFormat_, 1.0, systemDecimator_); + append(systemQueue_, systemStarved_, data, byteCount, systemFormat_, 1.0, systemDecimator_); } cv_.notify_all(); } @@ -581,13 +619,16 @@ void AudioMixer::pushMicrophone(const BYTE* data, DWORD byteCount) { // in: the queue would hold a flat-topped signal and the mix would add // further distortion on top of it, with whatever headroom the opposite // polarity of the system stream offered already destroyed. - append(microphoneQueue_, data, byteCount, microphoneFormat_, 1.0, microphoneDecimator_); + append( + microphoneQueue_, microphoneStarved_, data, byteCount, microphoneFormat_, 1.0, + microphoneDecimator_); } cv_.notify_all(); } void AudioMixer::append( std::vector& queue, + bool& starved, const BYTE* data, DWORD byteCount, const AudioInputFormat& sourceFormat, @@ -598,16 +639,35 @@ void AudioMixer::append( } convertAudioWithGain(data, byteCount, sourceFormat, format_, gain, gainBuffer_, decimator); + // A source that ran dry is coming back: loopback after a silence, or a device + // that stalled for longer than the cushion. Its packet belongs at now, which + // is as far ahead of what mixLoop has taken as real time is -- the cushion + // plus however late mixLoop is running, which on a loaded machine is tens of + // milliseconds. Queued at the front, it would land that much early. + if (starved) { + uint64_t lagFrames = 0; + if (clockAnchored_) { + const double elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - clockStart_).count(); + const auto nowFrames = static_cast(std::max(0.0, elapsed) * format_.sampleRate); + lagFrames = nowFrames > mixedFrames_ ? nowFrames - mixedFrames_ : 0; + } + queue.assign(static_cast(lagFrames) * format_.blockAlign, 0); + starved = false; + } queue.insert(queue.end(), gainBuffer_.begin(), gainBuffer_.end()); } -bool AudioMixer::pop(std::vector& queue, std::vector& chunk, size_t byteCount) { +bool AudioMixer::pop( + std::vector& queue, bool& starved, std::vector& chunk, size_t byteCount) { + chunk.assign(byteCount, 0); + if (queue.size() < byteCount) { + starved = true; + } if (queue.empty()) { - chunk.assign(byteCount, 0); return false; } - chunk.assign(byteCount, 0); const size_t copiedBytes = std::min(byteCount, queue.size()); std::memcpy(chunk.data(), queue.data(), copiedBytes); queue.erase(queue.begin(), queue.begin() + static_cast(copiedBytes)); @@ -631,72 +691,60 @@ bool AudioMixer::pop(std::vector& queue, std::vector& chunk, size_t * to cause a system-audio desync it merely stopped concealing * (getopenscreen/openscreen#406). * - * `audioClockStart` is anchored so that `emittedFrames_` always describes the + * A cushion behind real time (getopenscreen/openscreen#911). The capture threads + * poll WASAPI and push whatever has piled up -- every 15.6 ms at the default + * timer resolution, and later than that on a busy machine -- so a packet can + * reach its queue after the chunk it belongs to is due. Mixing at real time + * zero-filled that chunk and the packet played one chunk late: a hole of up to + * 10 ms in the middle of a continuous voice, heard as crackle. Measured on real + * takes: ten or more such holes in 25 s. Writing `MixerCushionMs` behind real time + * gives a late packet that long to arrive. The timestamps do not move -- they + * still come from `emittedFrames_` -- so the cushion only delays the writing, + * and a pause or a stop writes what it still holds up to that instant. + * + * `clockStart_` is anchored so that `emittedFrames_` always describes the * time elapsed since the timeline began; re-deriving it on resume is what lets a * pause interrupt the clock without shifting everything recorded after it. */ void AudioMixer::mixLoop() { const uint32_t chunkFrames = std::max(1, format_.sampleRate / 100); const size_t chunkBytes = static_cast(chunkFrames) * format_.blockAlign; + const uint64_t cushionFrames = static_cast(format_.sampleRate) * MixerCushionMs / 1000; std::vector mixedChunk; std::vector sourceChunk; - std::chrono::steady_clock::time_point audioClockStart; - bool audioClockAnchored = false; + // This loop's copy of `clockStart_`, taken under the lock at the top of each + // pass: beginTimeline and a resume move it from other threads. + std::chrono::steady_clock::time_point clockStart; const auto framesToDuration = [&](uint64_t frames) { return std::chrono::duration_cast( std::chrono::duration(static_cast(frames) / format_.sampleRate)); }; + const auto framesAt = [&](std::chrono::steady_clock::time_point time) { + const double elapsed = std::chrono::duration(time - clockStart).count(); + return static_cast(std::max(0.0, elapsed) * format_.sampleRate); + }; - while (true) { - { - std::unique_lock lock(mutex_); - cv_.wait_for(lock, std::chrono::milliseconds(20), [&] { - return stopRequested_.load() || (timelineStarted_ && !paused_); - }); - - if (stopRequested_) { - break; - } - if (!timelineStarted_ || paused_) { - // A pause stops the clock rather than resetting it: the anchor is - // re-derived from `emittedFrames_` on resume, so what follows keeps - // the position it would have had. - audioClockAnchored = false; - continue; - } - } - - const auto now = std::chrono::steady_clock::now(); - if (!audioClockAnchored) { - audioClockStart = now - framesToDuration(emittedFrames_); - audioClockAnchored = true; - } - - // How much of the timeline real time has covered. Emitting up to here -- - // from the queues where they have data, from silence where they do not -- - // is what keeps the audio clock pinned to the take rather than to whether - // anything happened to be playing. - const auto elapsed = std::chrono::duration(now - audioClockStart).count(); - const uint64_t targetFrames = static_cast(elapsed * format_.sampleRate); - + // Writes every chunk that ends by `targetFrames` -- from the queues where they + // have data, from silence where they do not, which is what keeps the audio + // clock pinned to the take rather than to whether anything happened to be + // playing. False once the output refuses a chunk. + const auto emitUntil = [&](uint64_t targetFrames) { while (emittedFrames_ + chunkFrames <= targetFrames) { { std::scoped_lock lock(mutex_); - if (stopRequested_ || !timelineStarted_ || paused_) { - break; - } mixedChunk.assign(chunkBytes, 0); if (includeSystem_) { - pop(systemQueue_, sourceChunk, chunkBytes); + pop(systemQueue_, systemStarved_, sourceChunk, chunkBytes); mixAudioInPlace(mixedChunk, sourceChunk.data(), static_cast(sourceChunk.size()), format_); } if (includeMicrophone_) { - pop(microphoneQueue_, sourceChunk, chunkBytes); + pop(microphoneQueue_, microphoneStarved_, sourceChunk, chunkBytes); mixAudioInPlace( mixedChunk, sourceChunk.data(), static_cast(sourceChunk.size()), format_, microphoneGain_); } + mixedFrames_ = emittedFrames_ + chunkFrames; } const int64_t timestampHns = @@ -705,15 +753,70 @@ void AudioMixer::mixLoop() { static_cast((static_cast(chunkFrames) * HnsPerSecond) / format_.sampleRate); if (!output_(mixedChunk.data(), static_cast(mixedChunk.size()), timestampHns, durationHns)) { stopRequested_ = true; - break; + return false; } emittedFrames_ += chunkFrames; } + return true; + }; - if (stopRequested_) { + while (true) { + bool stopping = false; + bool running = false; + bool flush = false; + std::chrono::steady_clock::time_point pausedAt; + { + std::unique_lock lock(mutex_); + cv_.wait_for(lock, std::chrono::milliseconds(20), [&] { + return stopRequested_.load() || (timelineStarted_ && !paused_); + }); + stopping = stopRequested_; + running = timelineStarted_ && !paused_; + pausedAt = pausedAt_; + clockStart = clockStart_; + if (stopping || !running) { + flush = clockAnchored_; + clockAnchored_ = false; + } + } + + if (stopping || !running) { + // A pause stops the clock rather than resetting it: the anchor is + // re-derived from `emittedFrames_` on resume, so what follows keeps + // the position it would have had -- once the cushion is written up to + // the pause, which is where that position is. A stop writes it up to + // now, unless it lands on a pause this loop has not handled yet: the + // paused span is out of the video, so it stays out of the audio. + const bool written = + !flush || emitUntil(framesAt(running ? std::chrono::steady_clock::now() : pausedAt)); + { + std::scoped_lock lock(mutex_); + pauseFlushed_ = true; + } + cv_.notify_all(); + if (!written || stopping) { + break; + } + continue; + } + + const uint64_t realFrames = framesAt(std::chrono::steady_clock::now()); + if (!emitUntil(realFrames > cushionFrames ? realFrames - cushionFrames : 0)) { break; } - std::this_thread::sleep_until(audioClockStart + framesToDuration(emittedFrames_ + chunkFrames)); + // Woken early by a pause or a stop, so that one writes the cushion out at once. + std::unique_lock lock(mutex_); + cv_.wait_until( + lock, clockStart + framesToDuration(emittedFrames_ + chunkFrames + cushionFrames), + [&] { return stopRequested_.load() || paused_; }); } + // Whatever ended the loop, a resume waiting on the pause flush must not wait + // for a loop that is gone. Set under the lock, so the wakeup cannot slip + // between the waiter's check and its sleep. + { + std::scoped_lock lock(mutex_); + pauseFlushed_ = true; + } + cv_.notify_all(); } diff --git a/electron/native/wgc-capture/src/audio_sample_utils.h b/electron/native/wgc-capture/src/audio_sample_utils.h index f869d5e94..81d99d021 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils.h +++ b/electron/native/wgc-capture/src/audio_sample_utils.h @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -103,12 +104,14 @@ class AudioMixer { private: void append( std::vector& queue, + bool& starved, const BYTE* data, DWORD byteCount, const AudioInputFormat& sourceFormat, double gain, AudioDecimatorState& decimator); - bool pop(std::vector& queue, std::vector& chunk, size_t byteCount); + bool pop(std::vector& queue, bool& starved, std::vector& chunk, size_t byteCount); + void resetSources(); void mixLoop(); AudioInputFormat format_{}; @@ -124,10 +127,23 @@ class AudioMixer { std::vector microphoneQueue_; AudioDecimatorState systemDecimator_; AudioDecimatorState microphoneDecimator_; + // Set when pop() runs a source dry, cleared by its next packet (see append). + bool systemStarved_ = false; + bool microphoneStarved_ = false; std::vector gainBuffer_; std::thread thread_; std::atomic stopRequested_ = false; bool timelineStarted_ = false; bool paused_ = false; + std::chrono::steady_clock::time_point pausedAt_{}; + // Set by mixLoop once it has written the cushion up to `pausedAt_`; a resume + // waits for it, or it would clear audio that belongs before the pause. + bool pauseFlushed_ = true; + // mixLoop's clock, shared so a source coming back after running dry can be + // placed at real time (see append). Written by mixLoop under `mutex_`. + std::chrono::steady_clock::time_point clockStart_{}; + bool clockAnchored_ = false; + // Frames mixLoop has taken from the queues so far, under `mutex_`. + uint64_t mixedFrames_ = 0; uint64_t emittedFrames_ = 0; }; diff --git a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp index b28f738c8..73995bd46 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp +++ b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp @@ -1,4 +1,5 @@ #include "audio_sample_utils.h" +#include "realtime_scheduling.h" #include #include @@ -215,6 +216,38 @@ HRESULT trySetAacPcmRateWithVideo(UINT32 sampleRate) { } // namespace int main() { + // --- Timer resolution (getopenscreen/openscreen#921) ---------------------- + // + // First, before anything in this process could have raised it: a 5 ms sleep + // is what the WASAPI poll asks for, and at the default 15.625 ms tick it + // lasts a whole tick. Median of 20, so one preempted sleep cannot decide it. + { + const auto medianSleepMs = [] { + std::vector samples; + for (int i = 0; i < 20; i += 1) { + const auto start = std::chrono::steady_clock::now(); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + samples.push_back( + std::chrono::duration(std::chrono::steady_clock::now() - start).count()); + } + std::sort(samples.begin(), samples.end()); + return samples[samples.size() / 2]; + }; + const double before = medianSleepMs(); + const HighResolutionTiming timing; + // Load only ever lengthens a sleep, so a median under 8 ms cannot be + // faked by a busy host, and at the default tick it cannot happen at all. + // A failing round is retried, so a transient load does not fail the build. + double after = medianSleepMs(); + for (int retry = 0; retry < 2 && after >= 8.0; retry += 1) { + after = std::min(after, medianSleepMs()); + } + char detail[96]{}; + sprintf_s(detail, "5 ms sleep: median %.2f ms before, %.2f ms after", before, after); + std::cout << "TIMER_RAW " << detail << std::endl; + expect("timer-5ms-sleep-lasts-about-5ms", after < 8.0, detail); + } + const AudioInputFormat diagnostic = makeFormat(MFAudioFormat_Float, 96000, 8, 32); const AudioInputFormat snapped = makeAacCompatibleAudioFormat(diagnostic); expect( @@ -1446,6 +1479,187 @@ int main() { } } + // --- Capture jitter: the mixer's cushion (getopenscreen/openscreen#911) --- + // + // The capture threads poll WASAPI every 5 ms -- 15.6 ms at the default timer + // resolution -- and push whatever packets have piled up, while the mixer emits + // on its own clock. With no margin between the two, every packet that arrived + // just after the mixer's tick was zero-filled: holes of up to 10 ms in the + // middle of the voice, heard as a crackle in the recording itself. `capture` + // below is that loop without WASAPI: a 10 ms packet becomes available every + // 10 ms from `from`, and a poller that sleeps 5 ms pushes what has piled up. + { + using std::chrono::milliseconds; + using Clock = std::chrono::steady_clock; + const AudioInputFormat f32 = makeFormat(MFAudioFormat_Float, 48000, 2, 32); + const auto dcPacket = [&](float value) { + std::vector bytes(480 * f32.blockAlign, 0); + auto* samples = reinterpret_cast(bytes.data()); + for (size_t i = 0; i < 480 * 2; i += 1) { + samples[i] = value; + } + return bytes; + }; + const auto capture = [](const auto& push, Clock::time_point from, int packets, + int stallAtPacket, int stallMs) { + int pushed = 0; + bool stalled = false; + while (pushed < packets) { + const auto now = Clock::now(); + const int available = now < from + ? 0 + : std::min(packets, static_cast((now - from) / milliseconds(10))); + for (; pushed < available; pushed += 1) { + push(); + } + if (!stalled && pushed >= stallAtPacket) { + stalled = true; + std::this_thread::sleep_for(milliseconds(stallMs)); + } + std::this_thread::sleep_for(milliseconds(5)); + } + }; + // Left channel of everything the mixer writes, from beginTimeline to stop. + const auto runTake = [&](bool includeSystem, bool includeMic, const auto& drive) { + std::mutex guard; + std::vector left; + AudioMixer mixer( + target48k, f32, f32, includeSystem, includeMic, 1.0, + [&](const BYTE* data, DWORD byteCount, int64_t, int64_t) { + std::scoped_lock lock(guard); + const auto* samples = reinterpret_cast(data); + for (size_t i = 0; i < byteCount / target48k.blockAlign; i += 1) { + left.push_back(samples[i * 2]); + } + return true; + }); + expect("jitter-mixer-start", mixer.start(), ""); + mixer.beginTimeline(); + drive(mixer, Clock::now()); + mixer.stop(); + std::scoped_lock lock(guard); + return left; + }; + const auto msAt = [](size_t frame) { return static_cast(frame) * 1000.0 / 48000.0; }; + + // (1) A microphone streaming a steady DC through ordinary poll jitter and one + // 40 ms stall: every sample between its first and its last must be voiced. + { + const auto packet = dcPacket(0.5f); + const auto left = runTake(false, true, [&](AudioMixer& mixer, Clock::time_point t0) { + capture( + [&] { mixer.pushMicrophone(packet.data(), static_cast(packet.size())); }, + t0, 120, 50, 40); + }); + const auto voiced = [](int16_t s) { return s != 0; }; + const auto first = std::find_if(left.begin(), left.end(), voiced); + const auto last = std::find_if(left.rbegin(), left.rend(), voiced).base(); + const size_t span = first < last ? static_cast(last - first) : 0; + const size_t holes = first < last ? static_cast(std::count(first, last, int16_t{0})) : 0; + char detail[128]{}; + sprintf_s(detail, "span=%zu holes=%zu of 57600 pushed", span, holes); + std::cout << "JITTER_RAW streaming " << detail << std::endl; + expect("mixer-streaming-source-has-no-holes", span >= 110 * 480 && holes == 0, detail); + } + + // (2) The cushion must not move a source that starts after silence -- loopback + // delivers nothing while nothing plays. Played from 400 ms, it lands there. + { + const auto packet = dcPacket(0.5f); + const auto left = runTake(true, false, [&](AudioMixer& mixer, Clock::time_point t0) { + capture( + [&] { mixer.pushSystem(packet.data(), static_cast(packet.size())); }, + t0 + milliseconds(400), 30, 1000, 0); + }); + const auto first = std::find_if(left.begin(), left.end(), [](int16_t s) { return s != 0; }); + const double at = msAt(static_cast(first - left.begin())); + char detail[96]{}; + sprintf_s(detail, "first sound at %.1f ms, played at 400 ms", at); + std::cout << "JITTER_RAW late-source " << detail << std::endl; + expect("mixer-late-source-lands-when-it-played", first != left.end() && std::abs(at - 400.0) <= 40.0, detail); + } + + // (3) Nor across a pause: what the cushion still holds at the pause is written + // before it, so what follows the resume starts where the pause began. + { + const auto beforePause = dcPacket(0.5f); + const auto afterPause = dcPacket(-0.5f); + double pausedAtMs = 0.0; + const auto left = runTake(false, true, [&](AudioMixer& mixer, Clock::time_point t0) { + capture( + [&] { mixer.pushMicrophone(beforePause.data(), static_cast(beforePause.size())); }, + t0, 40, 1000, 0); + pausedAtMs = std::chrono::duration(Clock::now() - t0).count(); + mixer.setPaused(true); + std::this_thread::sleep_for(milliseconds(200)); + mixer.setPaused(false); + capture( + [&] { mixer.pushMicrophone(afterPause.data(), static_cast(afterPause.size())); }, + Clock::now(), 30, 1000, 0); + }); + const auto first = std::find_if(left.begin(), left.end(), [](int16_t s) { return s < 0; }); + const double at = msAt(static_cast(first - left.begin())); + char detail[96]{}; + sprintf_s(detail, "resumed sound at %.1f ms, paused at %.1f ms", at, pausedAtMs); + std::cout << "JITTER_RAW pause " << detail << std::endl; + expect("mixer-resume-continues-at-the-pause", first != left.end() && std::abs(at - pausedAtMs) <= 40.0, detail); + } + + // (5) A pause followed by a stop, as the helper ends every take (#942): + // the track ends at the pause, not at the stop that comes later. + { + const auto packet = dcPacket(0.5f); + double pausedAtMs = 0.0; + const auto left = runTake(false, true, [&](AudioMixer& mixer, Clock::time_point t0) { + capture( + [&] { mixer.pushMicrophone(packet.data(), static_cast(packet.size())); }, + t0, 40, 1000, 0); + pausedAtMs = std::chrono::duration(Clock::now() - t0).count(); + mixer.setPaused(true); + std::this_thread::sleep_for(milliseconds(300)); + }); + const double lengthMs = msAt(left.size()); + char detail[96]{}; + sprintf_s(detail, "track %.1f ms long, paused at %.1f ms", lengthMs, pausedAtMs); + std::cout << "JITTER_RAW stop-after-pause " << detail << std::endl; + expect("mixer-track-ends-at-the-pause-before-a-stop", std::abs(lengthMs - pausedAtMs) <= 20.0, detail); + } + + // (4) The same with a resume that follows the pause at once, before the + // mixer has had a chance to see the pause: the resume must wait for the + // cushion to be written, or it throws the cushion away and what follows + // lands a cushion early. + { + const auto beforePause = dcPacket(0.5f); + const auto afterPause = dcPacket(-0.5f); + double pausedAtMs = 0.0; + const auto left = runTake(false, true, [&](AudioMixer& mixer, Clock::time_point t0) { + capture( + [&] { mixer.pushMicrophone(beforePause.data(), static_cast(beforePause.size())); }, + t0, 40, 1000, 0); + pausedAtMs = std::chrono::duration(Clock::now() - t0).count(); + mixer.setPaused(true); + mixer.setPaused(false); + capture( + [&] { mixer.pushMicrophone(afterPause.data(), static_cast(afterPause.size())); }, + Clock::now(), 30, 1000, 0); + }); + const auto first = std::find_if(left.begin(), left.end(), [](int16_t s) { return s < 0; }); + const double at = msAt(static_cast(first - left.begin())); + // 40 packets were pushed before the pause: 400 ms of voice, all of it + // due before the pause, none of it to be thrown away with the cushion. + const auto lastBefore = std::find_if(left.rbegin(), left.rend(), [](int16_t s) { return s > 0; }); + const double voiceEnd = msAt(static_cast(left.rend() - lastBefore)); + char detail[128]{}; + sprintf_s( + detail, "voice before the pause ends at %.1f ms of 400, resumed at %.1f ms, paused at %.1f ms", + voiceEnd, at, pausedAtMs); + std::cout << "JITTER_RAW instant-resume " << detail << std::endl; + expect("mixer-instant-resume-keeps-the-voice-before-it", voiceEnd >= 390.0, detail); + expect("mixer-instant-resume-continues-at-the-pause", first != left.end() && std::abs(at - pausedAtMs) <= 40.0, detail); + } + } + // --- mixAudioInPlace, per output format ----------------------------------- // // The gain rides in mixAudioInPlace on every format branch, but the diff --git a/electron/native/wgc-capture/src/cursor-sampler.cpp b/electron/native/wgc-capture/src/cursor-sampler.cpp index aa58a1cd7..cf4947c55 100644 --- a/electron/native/wgc-capture/src/cursor-sampler.cpp +++ b/electron/native/wgc-capture/src/cursor-sampler.cpp @@ -3,6 +3,7 @@ #include #include "dpi_awareness.h" +#include "realtime_scheduling.h" #include #include @@ -429,6 +430,7 @@ int main(int argc, char* argv[]) { std::cerr << "ERROR: Could not enable per-monitor-v2 DPI awareness" << std::endl; return 1; } + const HighResolutionTiming highResolutionTiming; if (argc < 2) { std::cerr << "Usage: cursor-sampler [windowHandle]" << std::endl; diff --git a/electron/native/wgc-capture/src/device_selection.cpp b/electron/native/wgc-capture/src/device_selection.cpp new file mode 100644 index 000000000..3211cbd1d --- /dev/null +++ b/electron/native/wgc-capture/src/device_selection.cpp @@ -0,0 +1,158 @@ +#include "device_selection.h" + +#include +#include + +namespace { + +/** + * Does one of these appear inside the other as WHOLE WORDS? + * + * Plain containment answered for devices that merely share a spelling: a + * requested "Logi" is inside "Logitech", and "Micro" inside "Microphone", + * neither of them as a word. Matching on that resolved a camera nobody asked + * for -- and resolving one is exactly what stops the request reaching the + * DirectShow fallback, where the cameras Media Foundation cannot enumerate live. + * + * Both sides arrive normalized, so a boundary is the start of the string, its + * end, or a space. + */ +bool containsAsWords(const std::wstring& haystack, const std::wstring& needle) { + if (haystack.empty() || needle.empty()) { + return false; + } + size_t pos = haystack.find(needle); + while (pos != std::wstring::npos) { + const bool startsOnBoundary = pos == 0 || haystack[pos - 1] == L' '; + const size_t after = pos + needle.size(); + const bool endsOnBoundary = after == haystack.size() || haystack[after] == L' '; + if (startsOnBoundary && endsOnBoundary) { + return true; + } + pos = haystack.find(needle, pos + 1); + } + return false; +} + +bool containsInsensitive(const std::wstring& haystack, const std::wstring& needle) { + return containsAsWords(haystack, needle) || containsAsWords(needle, haystack); +} + +std::wstring normalizeDeviceName(const std::wstring& value) { + std::wstring normalized; + normalized.reserve(value.size()); + bool lastWasSpace = true; + for (const wchar_t ch : value) { + if (std::iswalnum(ch)) { + normalized.push_back(static_cast(std::towlower(ch))); + lastWasSpace = false; + continue; + } + if (!lastWasSpace) { + normalized.push_back(L' '); + lastWasSpace = true; + } + } + while (!normalized.empty() && normalized.back() == L' ') { + normalized.pop_back(); + } + return normalized; +} + +std::wstring toLower(const std::wstring& value) { + std::wstring lowered; + lowered.reserve(value.size()); + for (const wchar_t ch : value) { + lowered.push_back(static_cast(std::towlower(ch))); + } + return lowered; +} + +} // namespace + +bool DeviceClaims::contains(const std::wstring& identity) const { + const std::wstring normalized = normalizeDeviceIdentity(identity); + return !normalized.empty() && + std::find(identities_.begin(), identities_.end(), normalized) != identities_.end(); +} + +void DeviceClaims::add(const std::wstring& identity) { + const std::wstring normalized = normalizeDeviceIdentity(identity); + if (!normalized.empty() && + std::find(identities_.begin(), identities_.end(), normalized) == identities_.end()) { + identities_.push_back(normalized); + } +} + +std::wstring normalizeDeviceIdentity(const std::wstring& identity) { + const std::wstring lowered = toLower(identity); + const size_t interfaceClass = lowered.rfind(L"#{"); + if (interfaceClass == std::wstring::npos || interfaceClass == 0) { + return lowered; + } + const size_t classEnd = lowered.find(L'}', interfaceClass); + if (classEnd == std::wstring::npos) { + return lowered; + } + return lowered.substr(0, interfaceClass) + lowered.substr(classEnd + 1); +} + +int deviceMatchScore( + const std::wstring& candidateName, + const std::wstring& candidateLink, + const std::wstring& requestedName, + const std::wstring& requestedId) { + int score = 0; + const auto normalizedName = normalizeDeviceName(candidateName); + const auto normalizedLink = normalizeDeviceName(candidateLink); + const auto normalizedRequestedName = normalizeDeviceName(requestedName); + const auto normalizedRequestedId = normalizeDeviceName(requestedId); + + if (!normalizedRequestedName.empty()) { + if (normalizedName == normalizedRequestedName) { + score = std::max(score, 1000); + } + if (containsInsensitive(normalizedName, normalizedRequestedName)) { + score = std::max(score, 900); + } + if (containsInsensitive(normalizedLink, normalizedRequestedName)) { + score = std::max(score, 800); + } + } + + if (!normalizedRequestedId.empty()) { + if (containsInsensitive(normalizedLink, normalizedRequestedId)) { + score = std::max(score, 700); + } + if (containsInsensitive(normalizedName, normalizedRequestedId)) { + score = std::max(score, 600); + } + } + + return score; +} + +int selectUnclaimedDevice( + const std::vector& candidates, + const std::wstring& requestedName, + const std::wstring& requestedId, + const DeviceClaims& claims) { + const bool requested = !requestedName.empty() || !requestedId.empty(); + int selected = -1; + int bestScore = 0; + for (size_t index = 0; index < candidates.size(); ++index) { + if (claims.contains(candidates[index].identity)) { + continue; + } + const int score = + deviceMatchScore(candidates[index].name, candidates[index].identity, requestedName, requestedId); + if (requested && score <= 0) { + continue; + } + if (selected < 0 || score > bestScore) { + selected = static_cast(index); + bestScore = score; + } + } + return selected; +} diff --git a/electron/native/wgc-capture/src/device_selection.h b/electron/native/wgc-capture/src/device_selection.h new file mode 100644 index 000000000..bcdc2ad8f --- /dev/null +++ b/electron/native/wgc-capture/src/device_selection.h @@ -0,0 +1,89 @@ +#pragma once + +#include +#include + +/** + * Which camera devices are already recording in this take. + * + * Two webcams of the same model report the same friendly name, and the id the + * browser hands us is a salted hash that never matches a device path, so name + * matching alone sends every such camera to the FIRST physical device. The + * second open then fails as busy and that camera is lost. Each camera that + * opens adds its device identity here, and later cameras of the take skip it. + * + * Identities are compared normalized (see `normalizeDeviceIdentity`), so a + * device opened through Media Foundation is recognized when it turns up again + * on the DirectShow fallback. + */ +class DeviceClaims { +public: + bool contains(const std::wstring& identity) const; + /** Ignores an empty identity: a device we cannot name cannot be claimed. */ + void add(const std::wstring& identity); + +private: + std::vector identities_; +}; + +/** One device a capture backend enumerated. */ +struct DeviceCandidate { + std::wstring name; + /** The Media Foundation symbolic link or the DirectShow DevicePath. */ + std::wstring identity; +}; + +/** + * The part of a device interface path that names the physical device. + * + * Media Foundation's symbolic link and DirectShow's DevicePath are the same + * `\\?\usb#vid_…##{interface class}\` string, except that + * each registers the camera under its own interface class GUID (measured on a + * Snapdragon front camera: KSCATEGORY_VIDEO_CAMERA against KSCATEGORY_VIDEO). + * Dropping the `#{…}` class and lowercasing leaves the device instance plus the + * reference string, which both share. The reference string is kept because it + * tells apart two cameras of one device (a colour and an IR sensor). A value + * without that shape (a moniker display name) is only lowercased. + */ +std::wstring normalizeDeviceIdentity(const std::wstring& identity); + +/** + * How well a candidate answers a requested name, or 0 for "not this one". + * + * Only decisive matches count: the names being equal once normalized, or one + * containing the other -- which is the ordinary case, since Chromium appends USB + * ids to what the driver reports. + * + * A further tier used to score shared WORDS, to bridge names differing more than + * that. It bridged names that were not the same device. "Logi Capture" and + * "Logitech StreamCam" share no word, yet "logi" sits inside "logitech" and that + * scored high enough to win -- so asking for a camera Media Foundation cannot + * enumerate opened a DIFFERENT camera, instead of returning nothing and letting + * the DirectShow fallback find the real one (getopenscreen/openscreen#405). + * + * Returning 0 is what makes that fallback reachable, so it is a real answer + * rather than a weak match. Keep this in step with + * `electron/recording/deviceNameMatching.ts`, which states the same rules for + * the Electron side and carries their unit tests. + */ +int deviceMatchScore( + const std::wstring& candidateName, + const std::wstring& candidateLink, + const std::wstring& requestedName, + const std::wstring& requestedId); + +/** + * The candidate a camera should open, or -1 for none. + * + * Claimed candidates are skipped; among the rest the best `deviceMatchScore` + * wins, the earlier one on a tie. With nothing requested that is the first + * unclaimed device. With a name or id requested, a candidate scoring 0 is never + * picked -- the same rule as before claims existed, and what lets the caller + * fall back to DirectShow. With an empty claim set this is exactly the old + * selection. + */ +int selectUnclaimedDevice( + const std::vector& candidates, + const std::wstring& requestedName, + const std::wstring& requestedId, + const DeviceClaims& claims); diff --git a/electron/native/wgc-capture/src/device_selection_test.cpp b/electron/native/wgc-capture/src/device_selection_test.cpp new file mode 100644 index 000000000..aa6329f66 --- /dev/null +++ b/electron/native/wgc-capture/src/device_selection_test.cpp @@ -0,0 +1,100 @@ +#include "device_selection.h" + +#include +#include +#include + +namespace { +int failures = 0; +void expect(bool ok, const char* label) { + if (!ok) { + std::printf("FAIL %s\n", label); + ++failures; + } +} +} // namespace + +int main() { + const std::vector twins = { + {L"USB Camera", L"\\\\?\\usb#vid_0c45&pid_6366&mi_00#7&aaa&0&0000#{e5323777-f976-4f5b-9b55-b94699c46e44}\\global"}, + {L"USB Camera", L"\\\\?\\usb#vid_0c45&pid_6366&mi_00#7&bbb&0&0000#{e5323777-f976-4f5b-9b55-b94699c46e44}\\global"}, + }; + + DeviceClaims none; + expect(selectUnclaimedDevice(twins, L"USB Camera", L"", none) == 0, + "same names, nothing claimed: first"); + + DeviceClaims first; + first.add(twins[0].identity); + expect(selectUnclaimedDevice(twins, L"USB Camera", L"", first) == 1, + "same names, first claimed: second"); + + DeviceClaims both; + both.add(twins[0].identity); + both.add(twins[1].identity); + expect(selectUnclaimedDevice(twins, L"USB Camera", L"", both) == -1, + "same names, both claimed: none"); + expect(selectUnclaimedDevice(twins, L"", L"", both) == -1, + "nothing requested, both claimed: none"); + + DeviceClaims upper; + upper.add(L"\\\\?\\USB#VID_0C45&PID_6366&MI_00#7&AAA&0&0000#{E5323777-F976-4F5B-9B55-B94699C46E44}\\GLOBAL"); + expect(selectUnclaimedDevice(twins, L"USB Camera", L"", upper) == 1, + "claim comparison is case-insensitive"); + + // DirectShow registers the same device under its own interface class GUID. + DeviceClaims viaDirectShow; + viaDirectShow.add( + L"\\\\?\\usb#vid_0c45&pid_6366&mi_00#7&aaa&0&0000#{65e8773d-8f56-11d0-a3b9-00a0c9223196}\\global"); + expect(viaDirectShow.contains(twins[0].identity), "MF link and DirectShow path are one device"); + expect(!viaDirectShow.contains(twins[1].identity), "the twin is a different device"); + + const std::vector mixed = { + {L"Studio Cam", L"\\\\?\\usb#vid_1#a#{e5323777-f976-4f5b-9b55-b94699c46e44}\\global"}, + {L"Studio Cam Pro", L"\\\\?\\usb#vid_2#b#{e5323777-f976-4f5b-9b55-b94699c46e44}\\global"}, + {L"Other Camera", L"\\\\?\\usb#vid_3#c#{e5323777-f976-4f5b-9b55-b94699c46e44}\\global"}, + }; + DeviceClaims exact; + exact.add(mixed[0].identity); + expect(selectUnclaimedDevice(mixed, L"Studio Cam", L"", none) == 0, "exact name wins unclaimed"); + expect(selectUnclaimedDevice(mixed, L"Studio Cam", L"", exact) == 1, + "claimed exact match loses to a weaker unclaimed match"); + DeviceClaims bothStudio = exact; + bothStudio.add(mixed[1].identity); + expect(selectUnclaimedDevice(mixed, L"Studio Cam", L"", bothStudio) == -1, + "a non-matching device is never picked for a request"); + expect(selectUnclaimedDevice(mixed, L"Nonexistent", L"", none) == -1, + "no match for a request: none"); + expect(selectUnclaimedDevice(mixed, L"", L"", none) == 0, "nothing requested: first device"); + expect(selectUnclaimedDevice(mixed, L"", L"", exact) == 1, + "nothing requested: first unclaimed device"); + expect(selectUnclaimedDevice({}, L"", L"", none) == -1, "no candidates: none"); + + // Measured on an ARM64 dev machine: the front camera as Media Foundation and + // as DirectShow list it. + DeviceClaims frontCamera; + frontCamera.add( + L"\\\\?\\display#qcom_avstream_8380#3&2dd9d5f4&0&uid32768#{e5323777-f976-4f5b-9b55-b94699c46e44}" + L"\\{4faeafd4-041b-4e46-85fd-400473891182}"); + expect(frontCamera.contains( + L"\\\\?\\DISPLAY#QCOM_AVSTREAM_8380#3&2DD9D5F4&0&UID32768#{65E8773D-8F56-11D0-A3B9-00A0C9223196}" + L"\\{4FAEAFD4-041B-4E46-85FD-400473891182}"), + "measured MF link and DirectShow path are one device"); + expect(!frontCamera.contains( + L"\\\\?\\display#qcom_avstream_8380#3&2dd9d5f4&0&uid32768#{e5323777-f976-4f5b-9b55-b94699c46e44}" + L"\\{00000000-0000-0000-0000-000000000001}"), + "another reference string on the same device is another camera"); + + DeviceClaims empty; + empty.add(L""); + expect(!empty.contains(L""), "an empty identity is never claimed"); + expect(normalizeDeviceIdentity(L"@device:sw:{ABC}") == L"@device:sw:{abc}", + "identity without an interface class is only lowercased"); + + if (failures == 0) { + std::printf("device_selection_test: all assertions passed\n"); + return 0; + } + std::printf("device_selection_test: %d failure(s)\n", failures); + return 1; +} diff --git a/electron/native/wgc-capture/src/dshow_webcam_capture.cpp b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp index f39e4b065..837f70f68 100644 --- a/electron/native/wgc-capture/src/dshow_webcam_capture.cpp +++ b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp @@ -1,5 +1,9 @@ #include "dshow_webcam_capture.h" +#include "realtime_scheduling.h" +#include "webcam_format.h" +#include "webcam_loss.h" + #include #include #include @@ -29,6 +33,19 @@ ISampleGrabber : public IUnknown { virtual HRESULT STDMETHODCALLTYPE SetCallback(IUnknown* callback, long whichMethodToCallback) = 0; }; +/** + * Is this subtype something the graph must DECODE before this class can unpack it? + * + * Listed positively -- an unknown subtype counts as compressed -- because + * guessing wrong this way only costs resolution, while guessing wrong the other + * way costs the camera track entirely. + */ +bool isCompressedDshowSubtype(const GUID& subtype) { + return !(subtype == MEDIASUBTYPE_YUY2 || subtype == MEDIASUBTYPE_NV12 || + subtype == MEDIASUBTYPE_RGB32 || subtype == MEDIASUBTYPE_RGB24 || + subtype == MEDIASUBTYPE_UYVY || subtype == MEDIASUBTYPE_YV12); +} + bool succeeded(HRESULT hr, const char* label) { if (SUCCEEDED(hr)) { return true; @@ -93,6 +110,63 @@ std::array yuvToBgr(int y, int u, int v) { return {clampToByte(blue), clampToByte(green), clampToByte(red)}; } +std::wstring readPropertyString(IPropertyBag* properties, const wchar_t* name) { + VARIANT value; + VariantInit(&value); + std::wstring result; + if (SUCCEEDED(properties->Read(name, &value, nullptr)) && value.vt == VT_BSTR && value.bstrVal) { + result = value.bstrVal; + } + VariantClear(&value); + return result; +} + +/** + * The video input devices DirectShow lists for the filter `clsid`. + * + * The fallback opens its filter by CLSID, which says nothing about WHICH + * device that is. The moniker does: its DevicePath is the same interface path + * Media Foundation reports as the symbolic link, so a device already opened + * there is recognized here. A moniker without a DevicePath (a plain software + * filter) is named by its display name instead. + */ +std::vector enumerateDevicesForClsid(const CLSID& clsid) { + std::vector candidates; + Microsoft::WRL::ComPtr deviceEnumerator; + if (FAILED(CoCreateInstance(CLSID_SystemDeviceEnum, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&deviceEnumerator)))) { + return candidates; + } + Microsoft::WRL::ComPtr monikers; + // S_FALSE: the category is empty. + if (deviceEnumerator->CreateClassEnumerator(CLSID_VideoInputDeviceCategory, &monikers, 0) != S_OK || !monikers) { + return candidates; + } + Microsoft::WRL::ComPtr moniker; + while (monikers->Next(1, &moniker, nullptr) == S_OK) { + Microsoft::WRL::ComPtr properties; + if (SUCCEEDED(moniker->BindToStorage(nullptr, nullptr, IID_PPV_ARGS(&properties)))) { + CLSID monikerClsid{}; + const std::wstring clsidText = readPropertyString(properties.Get(), L"CLSID"); + if (!clsidText.empty() && SUCCEEDED(CLSIDFromString(clsidText.c_str(), &monikerClsid)) && + IsEqualCLSID(monikerClsid, clsid)) { + DeviceCandidate candidate{ + readPropertyString(properties.Get(), L"FriendlyName"), + readPropertyString(properties.Get(), L"DevicePath")}; + if (candidate.identity.empty()) { + LPOLESTR displayName = nullptr; + if (SUCCEEDED(moniker->GetDisplayName(nullptr, nullptr, &displayName)) && displayName) { + candidate.identity = displayName; + CoTaskMemFree(displayName); + } + } + candidates.push_back(std::move(candidate)); + } + } + moniker.Reset(); + } + return candidates; +} + } // namespace struct DirectShowWebcamCapture::Impl { @@ -103,6 +177,8 @@ struct DirectShowWebcamCapture::Impl { Microsoft::WRL::ComPtr sampleGrabber; Microsoft::WRL::ComPtr nullRenderer; Microsoft::WRL::ComPtr mediaControl; + /** Where the graph says the device left; optional, see captureLoop. */ + Microsoft::WRL::ComPtr mediaEvent; bool comInitialized = false; bool running = false; }; @@ -112,10 +188,97 @@ DirectShowWebcamCapture::~DirectShowWebcamCapture() { delete impl_; } -bool DirectShowWebcamCapture::buildGraph(const CLSID& sourceClsid, const GUID* preferredSubtype) { +void DirectShowWebcamCapture::applyPreferredFormat(int requestedWidth, int requestedHeight, int requestedFps) { + Microsoft::WRL::ComPtr streamConfig; + if (FAILED(impl_->captureGraph->FindInterface( + &PIN_CATEGORY_CAPTURE, + &MEDIATYPE_Video, + impl_->captureFilter.Get(), + IID_PPV_ARGS(&streamConfig)))) { + // Virtual cameras commonly expose a single fixed format and no + // IAMStreamConfig at all. Nothing to choose from; let the graph connect. + return; + } + + int capCount = 0; + int capSize = 0; + if (FAILED(streamConfig->GetNumberOfCapabilities(&capCount, &capSize)) || + capSize != sizeof(VIDEO_STREAM_CONFIG_CAPS)) { + return; + } + + std::vector formats; + formats.reserve(static_cast(std::max(0, capCount))); + for (int index = 0; index < capCount; ++index) { + VIDEO_STREAM_CONFIG_CAPS caps{}; + AM_MEDIA_TYPE* mediaType = nullptr; + if (FAILED(streamConfig->GetStreamCaps(index, &mediaType, reinterpret_cast(&caps))) || !mediaType) { + continue; + } + if (mediaType->formattype == FORMAT_VideoInfo && mediaType->pbFormat) { + const auto* videoInfo = reinterpret_cast(mediaType->pbFormat); + // AvgTimePerFrame is in 100ns units; 333333 is 30 fps. + const int fps = videoInfo->AvgTimePerFrame > 0 + ? static_cast((10'000'000LL + videoInfo->AvgTimePerFrame / 2) / videoInfo->AvgTimePerFrame) + : 0; + formats.push_back(WebcamFormat{ + std::abs(videoInfo->bmiHeader.biWidth), + std::abs(videoInfo->bmiHeader.biHeight), + fps, + isCompressedDshowSubtype(mediaType->subtype)}); + } + freeMediaType(*mediaType); + CoTaskMemFree(mediaType); + } + + const WebcamFormat chosen = chooseWebcamFormat( + formats, + requestedWidth > 0 ? requestedWidth : 1920, + requestedHeight > 0 ? requestedHeight : 1080, + std::max(1, requestedFps)); + + // Second pass: set the first capability that matches the choice. GetStreamCaps + // hands back the media type we must give to SetFormat, so it has to be + // re-read rather than cached across the loop above. + for (int index = 0; index < capCount; ++index) { + VIDEO_STREAM_CONFIG_CAPS caps{}; + AM_MEDIA_TYPE* mediaType = nullptr; + if (FAILED(streamConfig->GetStreamCaps(index, &mediaType, reinterpret_cast(&caps))) || !mediaType) { + continue; + } + bool matched = false; + if (mediaType->formattype == FORMAT_VideoInfo && mediaType->pbFormat) { + auto* videoInfo = reinterpret_cast(mediaType->pbFormat); + if (std::abs(videoInfo->bmiHeader.biWidth) == chosen.width && + std::abs(videoInfo->bmiHeader.biHeight) == chosen.height && + isCompressedDshowSubtype(mediaType->subtype) == chosen.compressed) { + if (chosen.fps > 0) { + videoInfo->AvgTimePerFrame = 10'000'000LL / chosen.fps; + } + matched = SUCCEEDED(streamConfig->SetFormat(mediaType)); + if (matched) { + std::cerr << "INFO: DirectShow webcam format " << chosen.width << "x" << chosen.height + << "@" << chosen.fps << std::endl; + } + } + } + freeMediaType(*mediaType); + CoTaskMemFree(mediaType); + if (matched) { + return; + } + } +} + +bool DirectShowWebcamCapture::buildGraph( + const CLSID& sourceClsid, + const GUID* preferredSubtype, + int preferredWidth, + int preferredHeight) { // Every attempt starts from empty filters. A RenderStream that fails can // leave pins connected behind it, and retrying on top of that half-built // graph is how you get a second failure that says nothing about the format. + impl_->mediaEvent.Reset(); impl_->mediaControl.Reset(); impl_->nullRenderer.Reset(); impl_->sampleGrabber.Reset(); @@ -175,6 +338,8 @@ bool DirectShowWebcamCapture::buildGraph(const CLSID& sourceClsid, const GUID* p return false; } + applyPreferredFormat(preferredWidth, preferredHeight, fps_); + return succeeded(impl_->captureGraph->RenderStream( &PIN_CATEGORY_CAPTURE, &MEDIATYPE_Video, @@ -190,7 +355,8 @@ bool DirectShowWebcamCapture::initialize( const std::wstring& directShowClsid, int requestedWidth, int requestedHeight, - int requestedFps) { + int requestedFps, + const DeviceClaims& claims) { (void)deviceId; stop(); delete impl_; @@ -217,6 +383,27 @@ bool DirectShowWebcamCapture::initialize( } selectedDeviceName_ = deviceName.empty() ? directShowClsid : deviceName; + // Every device this filter stands for has already been matched by name in + // Electron, so the only thing left to choose on is which one is free. With + // no moniker naming the filter, the CLSID itself is the identity. + const std::vector candidates = enumerateDevicesForClsid(selectedClsid); + if (candidates.empty()) { + deviceIdentity_ = directShowClsid; + if (claims.contains(deviceIdentity_)) { + std::cerr << "ERROR: DirectShow webcam filter is already recording in this take" << std::endl; + return false; + } + } else { + const int selectedIndex = selectUnclaimedDevice(candidates, L"", L"", claims); + if (selectedIndex < 0) { + std::cerr << "ERROR: Every DirectShow webcam for this filter is already recording in this take" + << std::endl; + return false; + } + deviceIdentity_ = candidates[selectedIndex].identity; + } + std::wcerr << L"INFO: DirectShow webcam device " << deviceIdentity_ << std::endl; + // The camera's own format first, a forced RGB32 conversion only if we cannot // read it. // @@ -231,14 +418,14 @@ bool DirectShowWebcamCapture::initialize( // it connects with a subtype this file cannot read // (getopenscreen/openscreen#387). Naming a concrete subtype on the retry is // what makes DirectShow insert a colour converter for it. - if (!buildGraph(selectedClsid, nullptr)) { + if (!buildGraph(selectedClsid, nullptr, requestedWidth, requestedHeight)) { return false; } if (!resolveConnectedFormat(requestedWidth, requestedHeight, false)) { std::cerr << "WARNING: DirectShow webcam speaks a format this build cannot unpack; " "asking for RGB32 so the graph converts it" << std::endl; - if (!buildGraph(selectedClsid, &MEDIASUBTYPE_RGB32)) { + if (!buildGraph(selectedClsid, &MEDIASUBTYPE_RGB32, requestedWidth, requestedHeight)) { return false; } if (!resolveConnectedFormat(requestedWidth, requestedHeight, true)) { @@ -251,6 +438,10 @@ bool DirectShowWebcamCapture::initialize( if (!succeeded(impl_->graph.As(&impl_->mediaControl), "QueryInterface(IMediaControl)")) { return false; } + // Best-effort: without it a lost device goes unnoticed, as it always did. + if (FAILED(impl_->graph.As(&impl_->mediaEvent))) { + impl_->mediaEvent.Reset(); + } return true; } @@ -291,9 +482,28 @@ bool DirectShowWebcamCapture::resolveConnectedFormat( sourceStride_ = ((width_ * bitsPerPixel + 31) / 32) * 4; } sourceTopDown_ = pixelFormat_ != PixelFormat::Bgra || videoInfo->bmiHeader.biHeight < 0; + // The rate the graph settled on, not the one that was asked for. + // + // `chooseWebcamFormat` deliberately settles for less than the target + // when a camera offers nothing faster, and a driver may pick its own + // nearest rate after SetFormat regardless. `fps()` feeds three things + // that all have to agree with the frames actually arriving: the + // `webcam-format` event, the webcam encoder's nominal rate, and the + // constant-rate write interval in main.cpp. Left at the requested + // value, a 24 fps camera asked for 30 is encoded and paced as 30. + if (videoInfo->AvgTimePerFrame > 0) { + const int negotiated = static_cast( + (10'000'000LL + videoInfo->AvgTimePerFrame / 2) / videoInfo->AvgTimePerFrame); + if (negotiated > 0 && negotiated != fps_) { + std::cerr << "INFO: DirectShow webcam negotiated " << negotiated << " fps (asked for " + << fps_ << ")" << std::endl; + fps_ = std::clamp(negotiated, 1, 60); + } + } } std::cerr << "INFO: DirectShow webcam connected subtype " << guidToString(connectedType.subtype) - << " " << width_ << "x" << height_ << " stride=" << sourceStride_ << std::endl; + << " " << width_ << "x" << height_ << "@" << fps_ << " stride=" << sourceStride_ + << std::endl; freeMediaType(connectedType); if (width_ <= 0 || height_ <= 0) { width_ = requestedWidth > 0 ? requestedWidth : 1280; @@ -344,6 +554,7 @@ void DirectShowWebcamCapture::stop() { impl_->mediaControl->Stop(); } impl_->running = false; + impl_->mediaEvent.Reset(); impl_->mediaControl.Reset(); impl_->nullRenderer.Reset(); impl_->sampleGrabber.Reset(); @@ -358,8 +569,14 @@ void DirectShowWebcamCapture::stop() { } void DirectShowWebcamCapture::captureLoop() { + const MmcssThread mmcss(L"Capture"); const HRESULT coinitHr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); while (!stopRequested_ && impl_ && impl_->sampleGrabber) { + // The sample grabber keeps handing back its last buffer after the + // camera is gone, so the frames cannot tell; the graph's events can. + if (deviceLeftGraph()) { + break; + } long bufferSize = 0; HRESULT hr = impl_->sampleGrabber->GetCurrentBuffer(&bufferSize, nullptr); if (SUCCEEDED(hr) && bufferSize > 0) { @@ -376,6 +593,32 @@ void DirectShowWebcamCapture::captureLoop() { } } +bool DirectShowWebcamCapture::deviceLeftGraph() { + if (!impl_->mediaEvent) { + return false; + } + long code = 0; + LONG_PTR param1 = 0; + LONG_PTR param2 = 0; + // A zero timeout drains what is queued without waiting for more. + while (SUCCEEDED(impl_->mediaEvent->GetEvent(&code, ¶m1, ¶m2, 0))) { + impl_->mediaEvent->FreeEventParams(code, param1, param2); + // EC_DEVICE_LOST's second parameter is 0 when the device was removed, + // 1 when it came back; a stream error or abort stops the graph. + const bool lost = (code == EC_DEVICE_LOST && param2 == 0) || code == EC_ERRORABORT || + code == EC_STREAM_ERROR_STOPPED; + if (lost && !lost_.exchange(true)) { + reportWebcamLost(selectedDeviceName_, static_cast(code)); + return true; + } + } + return false; +} + +bool DirectShowWebcamCapture::isLost() const { + return lost_; +} + void DirectShowWebcamCapture::storeFrame(const BYTE* buffer, long length) { const int destinationStride = width_ * 4; const int sourceStride = sourceStride_ > 0 ? sourceStride_ : destinationStride; @@ -452,17 +695,10 @@ void DirectShowWebcamCapture::storeFrame(const BYTE* buffer, long length) { latestFrameSequence_ += 1; } -bool DirectShowWebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination) { +bool DirectShowWebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination, uint64_t lastSeenSequence) { std::scoped_lock lock(frameMutex_); - if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) { - return false; - } - - destination.data = latestFrame_; - destination.width = width_; - destination.height = height_; - destination.sequence = latestFrameSequence_; - return true; + return snapshotWebcamFrame( + latestFrame_, width_, height_, latestFrameSequence_, lastSeenSequence, destination); } int DirectShowWebcamCapture::width() const { @@ -477,6 +713,10 @@ int DirectShowWebcamCapture::fps() const { return fps_; } +const std::wstring& DirectShowWebcamCapture::deviceIdentity() const { + return deviceIdentity_; +} + const std::wstring& DirectShowWebcamCapture::selectedDeviceName() const { return selectedDeviceName_; } diff --git a/electron/native/wgc-capture/src/dshow_webcam_capture.h b/electron/native/wgc-capture/src/dshow_webcam_capture.h index cf510e2df..5e26098d0 100644 --- a/electron/native/wgc-capture/src/dshow_webcam_capture.h +++ b/electron/native/wgc-capture/src/dshow_webcam_capture.h @@ -1,5 +1,7 @@ #pragma once +#include "device_selection.h" + #include #include @@ -16,6 +18,33 @@ struct WebcamFrameSnapshot { uint64_t sequence = 0; }; +/** + * Copies the latest camera frame into `destination`, unless the caller already has it. + * + * The video writer asks on every screen tick, 60 times a second, while a camera + * delivers 30: copying first and comparing sequences after cost a full frame + * copied for nothing on every other tick. Returns false, leaving `destination` + * untouched, when there is no frame yet or its sequence is `lastSeenSequence`. + * Both capture backends route `copyLatestFrame` through this. + */ +inline bool snapshotWebcamFrame( + const std::vector& frame, + int width, + int height, + uint64_t sequence, + uint64_t lastSeenSequence, + WebcamFrameSnapshot& destination) { + if (frame.empty() || width <= 0 || height <= 0 || sequence == lastSeenSequence) { + return false; + } + + destination.data = frame; + destination.width = width; + destination.height = height; + destination.sequence = sequence; + return true; +} + class DirectShowWebcamCapture { public: DirectShowWebcamCapture() = default; @@ -30,15 +59,26 @@ class DirectShowWebcamCapture { const std::wstring& directShowClsid, int requestedWidth, int requestedHeight, - int requestedFps); + int requestedFps, + const DeviceClaims& claims); bool start(); void stop(); - bool copyLatestFrame(WebcamFrameSnapshot& destination); + bool copyLatestFrame(WebcamFrameSnapshot& destination, uint64_t lastSeenSequence); int width() const; int height() const; int fps() const; const std::wstring& selectedDeviceName() const; + /** + * The opened device's DevicePath (the moniker's display name when it has + * none, the filter CLSID when no moniker names it), for the take's claims. + */ + const std::wstring& deviceIdentity() const; + /** + * Did the graph report the device gone (EC_DEVICE_LOST removal, a stream + * error, or an abort)? Latched; safe to ask from any thread. + */ + bool isLost() const; void storeFrame(const BYTE* buffer, long length); private: @@ -50,6 +90,8 @@ class DirectShowWebcamCapture { struct Impl; void captureLoop(); + /** Drains the graph's queued events; true once one of them says the device left. */ + bool deviceLeftGraph(); /** * Builds source -> sample grabber -> null renderer and connects it. * @@ -59,7 +101,21 @@ class DirectShowWebcamCapture { * without leaving the graph half-built, so the caller can retry with a * different constraint. */ - bool buildGraph(const CLSID& sourceClsid, const GUID* preferredSubtype); + bool buildGraph( + const CLSID& sourceClsid, + const GUID* preferredSubtype, + int preferredWidth, + int preferredHeight); + /** + * Pins the capture pin to the best format the device offers, before the + * graph is rendered. + * + * Skipped, RenderStream's intelligent connect takes the pin's default + * format, which on a UVC camera is the first one it enumerates -- 640x480 + * on hardware that can do far better. Best-effort: a device without + * IAMStreamConfig, or one that rejects the format, still gets a graph. + */ + void applyPreferredFormat(int requestedWidth, int requestedHeight, int requestedFps); /** * Reads back what the graph actually negotiated and records how to unpack it. * @@ -72,6 +128,7 @@ class DirectShowWebcamCapture { Impl* impl_ = nullptr; std::thread thread_; std::atomic stopRequested_ = false; + std::atomic lost_ = false; std::mutex frameMutex_; std::vector latestFrame_; uint64_t latestFrameSequence_ = 0; @@ -82,4 +139,5 @@ class DirectShowWebcamCapture { bool sourceTopDown_ = false; PixelFormat pixelFormat_ = PixelFormat::Bgra; std::wstring selectedDeviceName_; + std::wstring deviceIdentity_; }; diff --git a/electron/native/wgc-capture/src/frame_visibility.cpp b/electron/native/wgc-capture/src/frame_visibility.cpp new file mode 100644 index 000000000..d296b5f0e --- /dev/null +++ b/electron/native/wgc-capture/src/frame_visibility.cpp @@ -0,0 +1,78 @@ +#include "frame_visibility.h" + +#include + +namespace { + +// Sampling at most this many pixels keeps the probe cheap on a 4K frame while +// still covering it evenly. +constexpr size_t kMaxSamples = 4096; + +// Luma is judged on a full-range 0..255 scale. A frame counts as visible when +// any sample is clearly above black, or when the picture is dim but not empty. +constexpr int kMaxLumaThreshold = 24; +constexpr int kAverageLumaThreshold = 4; + +// BT.709/BT.601 studio range: black sits at 16, white at 235. +constexpr int kStudioBlack = 16; +constexpr int kStudioSpan = 235 - kStudioBlack; + +bool isVisible(uint64_t lumaTotal, int maxLuma, size_t sampled) { + const uint64_t averageLuma = sampled > 0 ? lumaTotal / sampled : 0; + return maxLuma > kMaxLumaThreshold || averageLuma > kAverageLumaThreshold; +} + +} // namespace + +bool hasVisibleBgraContent(const std::vector& frame) { + if (frame.size() < 4) { + return false; + } + + uint64_t lumaTotal = 0; + int maxLuma = 0; + const size_t pixelCount = frame.size() / 4; + const size_t step = std::max(1, pixelCount / kMaxSamples); + size_t sampled = 0; + for (size_t pixel = 0; pixel < pixelCount; pixel += step) { + const size_t offset = pixel * 4; + const int b = frame[offset + 0]; + const int g = frame[offset + 1]; + const int r = frame[offset + 2]; + const int luma = (r * 54 + g * 183 + b * 19) >> 8; + lumaTotal += static_cast(luma); + maxLuma = std::max(maxLuma, luma); + sampled += 1; + } + + return isVisible(lumaTotal, maxLuma, sampled); +} + +bool hasVisibleNv12Content(const std::vector& frame) { + if (frame.size() < 6) { + return false; + } + + const size_t lumaCount = frame.size() * 2 / 3; + const size_t step = std::max(1, lumaCount / kMaxSamples); + uint64_t lumaTotal = 0; + int maxLuma = 0; + size_t sampled = 0; + for (size_t offset = 0; offset < lumaCount; offset += step) { + // Studio range mapped onto 0..255, so the thresholds mean the same here + // as they do for BGRA. Skip this and an all-black frame averages 16 -- + // past the average threshold on its own, so every black frame would + // count as a picture. + const int studio = static_cast(frame[offset]); + const int luma = std::clamp(((studio - kStudioBlack) * 255) / kStudioSpan, 0, 255); + lumaTotal += static_cast(luma); + maxLuma = std::max(maxLuma, luma); + sampled += 1; + } + + return isVisible(lumaTotal, maxLuma, sampled); +} + +bool hasVisibleWebcamContent(const std::vector& frame, bool isNv12) { + return isNv12 ? hasVisibleNv12Content(frame) : hasVisibleBgraContent(frame); +} diff --git a/electron/native/wgc-capture/src/frame_visibility.h b/electron/native/wgc-capture/src/frame_visibility.h new file mode 100644 index 000000000..9f62c2298 --- /dev/null +++ b/electron/native/wgc-capture/src/frame_visibility.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +/** + * Does this frame contain any picture, or is it the blank one a camera emits + * while it is still warming up? + * + * Used before recording starts, to tell "the camera is running" from "the + * camera is open but has not produced anything yet". + */ +bool hasVisibleBgraContent(const std::vector& frame); + +/** + * The NV12 twin: luma is the Y plane, one byte per pixel, so no colour + * conversion is needed. The Y plane is the first two thirds of the buffer. + * + * A camera's NV12 is studio-range, where black is 16 rather than 0. Those + * values are normalised to full range before the thresholds are applied -- + * without that, the average of an all-black frame is 16, clears the average + * threshold on its own, and every black frame counts as visible. + */ +bool hasVisibleNv12Content(const std::vector& frame); + +/** Dispatches to the probe matching the frame's layout. */ +bool hasVisibleWebcamContent(const std::vector& frame, bool isNv12); diff --git a/electron/native/wgc-capture/src/frame_visibility_test.cpp b/electron/native/wgc-capture/src/frame_visibility_test.cpp new file mode 100644 index 000000000..51e9f7531 --- /dev/null +++ b/electron/native/wgc-capture/src/frame_visibility_test.cpp @@ -0,0 +1,66 @@ +#include "frame_visibility.h" + +#include +#include +#include + +namespace { + +int failures = 0; + +void expectVisible(const std::string& label, bool actual, bool expected) { + if (actual == expected) { + return; + } + std::printf("FAIL %s: expected %s, got %s\n", label.c_str(), expected ? "visible" : "blank", + actual ? "visible" : "blank"); + ++failures; +} + +std::vector nv12(int width, int height, std::uint8_t luma) { + std::vector frame(static_cast(width) * height * 3 / 2, 128); + std::fill(frame.begin(), frame.begin() + static_cast(width) * height, luma); + return frame; +} + +std::vector bgra(int width, int height, std::uint8_t value) { + return std::vector(static_cast(width) * height * 4, value); +} + +} // namespace + +int main() { + // The regression this file exists for. A camera's NV12 is studio-range, so an + // all-black frame is a plane of 16, not 0. Averaged raw that is 16 -- past the + // average threshold on its own -- and every black frame counted as a picture, + // which is exactly what the warm-up probe exists to rule out. + expectVisible("studio-range black is blank", hasVisibleNv12Content(nv12(640, 480, 16)), false); + expectVisible("true zero is blank", hasVisibleNv12Content(nv12(640, 480, 0)), false); + + // Just above black stays blank; a clearly lit frame does not. + expectVisible("near-black is blank", hasVisibleNv12Content(nv12(640, 480, 20)), false); + expectVisible("mid grey is visible", hasVisibleNv12Content(nv12(640, 480, 128)), true); + expectVisible("studio white is visible", hasVisibleNv12Content(nv12(640, 480, 235)), true); + + // BGRA keeps its own behaviour: full-range, black is 0. + expectVisible("bgra black is blank", hasVisibleBgraContent(bgra(640, 480, 0)), false); + expectVisible("bgra grey is visible", hasVisibleBgraContent(bgra(640, 480, 128)), true); + + // Buffers too small to carry a frame must not be read past their end. + expectVisible("empty nv12", hasVisibleNv12Content({}), false); + expectVisible("empty bgra", hasVisibleBgraContent({}), false); + expectVisible("runt nv12", hasVisibleNv12Content({16, 16, 16}), false); + + // The dispatcher has to pick the layout it is told, not guess. The same bytes + // read as BGRA are bright; read as studio-range NV12 they are black. + const std::vector blackNv12 = nv12(320, 240, 16); + expectVisible("dispatch nv12", hasVisibleWebcamContent(blackNv12, true), false); + expectVisible("dispatch bgra", hasVisibleWebcamContent(bgra(320, 240, 200), false), true); + + if (failures == 0) { + std::printf("frame_visibility_test: all assertions passed\n"); + return 0; + } + std::printf("frame_visibility_test: %d assertion(s) failed\n", failures); + return 1; +} diff --git a/electron/native/wgc-capture/src/json_fields.cpp b/electron/native/wgc-capture/src/json_fields.cpp new file mode 100644 index 000000000..e36a43e81 --- /dev/null +++ b/electron/native/wgc-capture/src/json_fields.cpp @@ -0,0 +1,125 @@ +#include "json_fields.h" + +#include +#include + +// Lightweight field lookups over the helper's single JSON argument. They match +// the first occurrence of a key anywhere in the document. + +bool findBool(const std::string& json, const std::string& key, bool fallback) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return fallback; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return fallback; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + if (json.compare(pos, 4, "true") == 0) { + return true; + } + if (json.compare(pos, 5, "false") == 0) { + return false; + } + return fallback; +} + +int64_t findInt64(const std::string& json, const std::string& key, int64_t fallback) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return fallback; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return fallback; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + try { + return std::stoll(json.substr(pos)); + } catch (...) { + return fallback; + } +} + +int findInt(const std::string& json, const std::string& key, int fallback) { + return static_cast(findInt64(json, key, fallback)); +} + +double findDouble(const std::string& json, const std::string& key, double fallback) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return fallback; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return fallback; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + try { + return std::stod(json.substr(pos)); + } catch (...) { + return fallback; + } +} + +std::string findString(const std::string& json, const std::string& key) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return {}; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return {}; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + if (pos >= json.size() || json[pos] != '"') { + return {}; + } + pos += 1; + + std::string result; + while (pos < json.size()) { + const char c = json[pos++]; + if (c == '"') { + break; + } + if (c == '\\' && pos < json.size()) { + const char escaped = json[pos++]; + switch (escaped) { + case '\\': + case '"': + case '/': + result.push_back(escaped); + break; + case 'n': + result.push_back('\n'); + break; + case 'r': + result.push_back('\r'); + break; + case 't': + result.push_back('\t'); + break; + default: + result.push_back(escaped); + break; + } + continue; + } + result.push_back(c); + } + return result; +} diff --git a/electron/native/wgc-capture/src/json_fields.h b/electron/native/wgc-capture/src/json_fields.h new file mode 100644 index 000000000..0921b3381 --- /dev/null +++ b/electron/native/wgc-capture/src/json_fields.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +bool findBool(const std::string& json, const std::string& key, bool fallback); +int64_t findInt64(const std::string& json, const std::string& key, int64_t fallback); +int findInt(const std::string& json, const std::string& key, int fallback); +double findDouble(const std::string& json, const std::string& key, double fallback); +std::string findString(const std::string& json, const std::string& key); diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index c86524d6d..77dc692db 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -1,12 +1,16 @@ #include "audio_sample_utils.h" #include "desktop_icon_cover.h" #include "dpi_awareness.h" +#include "realtime_scheduling.h" #include "mf_encoder.h" #include "monitor_utils.h" #include "wasapi_device_watcher.h" #include "wasapi_loopback_capture.h" #include "wasapi_render_keepalive.h" +#include "frame_visibility.h" +#include "json_fields.h" #include "webcam_capture.h" +#include "webcam_config.h" #include "wgc_session.h" #include @@ -24,6 +28,7 @@ #include #include #include +#include namespace { @@ -35,7 +40,6 @@ struct CaptureConfig { std::string sourceId; std::string windowHandle; std::string outputPath; - std::string webcamOutputPath; int fps = 60; int width = 0; int height = 0; @@ -136,6 +140,49 @@ struct CaptureControl { } }; +// One camera of the take: its capture, its encoder and the writer-loop state that used +// to be single variables. All cameras share the recording's T0. +struct WebcamStream { + WebcamConfig config; + // The camera's position in the config list, kept when an earlier camera is + // dropped, so every event names the camera the caller asked for. + size_t index = 0; + WebcamCapture capture; + MFEncoder encoder; + bool active = false; + // False only for the legacy inline picture-in-picture camera, which has no + // file of its own and is drawn into the screen frame instead. + bool writeSeparate = false; + std::vector latestFrame; + int latestWidth = 0; + int latestHeight = 0; + uint64_t latestSequence = 0; + bool hasVisibleFrame = false; + int64_t lastTimestampHns = -1; + int64_t nextWriteDueHns = 0; + int64_t nominalIntervalHns = 0; + // Captured in the writer's pull block, submitted after it (issue #115). + Microsoft::WRL::ComPtr pendingSample; + // Owned here because the shutdown watchdog reads the current step's name + // through a raw pointer from another thread. + std::string finalizeStepName; + + // Takes the camera's newest frame if it carries a picture. Returns whether it did. + bool pullVisibleFrame() { + WebcamFrameSnapshot candidate; + if (!capture.copyLatestFrame(candidate, latestSequence) || + !hasVisibleWebcamContent(candidate.data, capture.deliversNv12())) { + return false; + } + latestFrame = std::move(candidate.data); + latestWidth = candidate.width; + latestHeight = candidate.height; + latestSequence = candidate.sequence; + hasVisibleFrame = true; + return true; + } +}; + int readEnvInt(const char* name, int fallback) { char raw[32]{}; const DWORD length = GetEnvironmentVariableA(name, raw, static_cast(sizeof(raw))); @@ -393,147 +440,13 @@ void reportCaptureAdapters(ID3D11Device* device, HMONITOR targetMonitor) { } } -bool hasVisibleBgraContent(const std::vector& frame) { - if (frame.size() < 4) { - return false; - } - - uint64_t lumaTotal = 0; - BYTE maxLuma = 0; - const size_t pixelCount = frame.size() / 4; - const size_t step = std::max(1, pixelCount / 4096); - size_t sampledPixels = 0; - for (size_t pixel = 0; pixel < pixelCount; pixel += step) { - const size_t offset = pixel * 4; - const BYTE b = frame[offset + 0]; - const BYTE g = frame[offset + 1]; - const BYTE r = frame[offset + 2]; - const BYTE luma = static_cast((static_cast(r) * 54 + static_cast(g) * 183 + static_cast(b) * 19) >> 8); - lumaTotal += luma; - maxLuma = std::max(maxLuma, luma); - sampledPixels += 1; - } - const uint64_t averageLuma = sampledPixels > 0 ? lumaTotal / sampledPixels : 0; - return maxLuma > 24 || averageLuma > 4; -} -bool findBool(const std::string& json, const std::string& key, bool fallback) { - auto pos = json.find("\"" + key + "\""); - if (pos == std::string::npos) { - return fallback; - } - pos = json.find(':', pos); - if (pos == std::string::npos) { - return fallback; - } - pos += 1; - while (pos < json.size() && std::isspace(static_cast(json[pos]))) { - pos += 1; - } - if (json.compare(pos, 4, "true") == 0) { - return true; - } - if (json.compare(pos, 5, "false") == 0) { - return false; - } - return fallback; -} - -int64_t findInt64(const std::string& json, const std::string& key, int64_t fallback) { - auto pos = json.find("\"" + key + "\""); - if (pos == std::string::npos) { - return fallback; - } - pos = json.find(':', pos); - if (pos == std::string::npos) { - return fallback; - } - pos += 1; - while (pos < json.size() && std::isspace(static_cast(json[pos]))) { - pos += 1; - } - try { - return std::stoll(json.substr(pos)); - } catch (...) { - return fallback; - } -} - -int findInt(const std::string& json, const std::string& key, int fallback) { - return static_cast(findInt64(json, key, fallback)); -} - -double findDouble(const std::string& json, const std::string& key, double fallback) { - auto pos = json.find("\"" + key + "\""); - if (pos == std::string::npos) { - return fallback; - } - pos = json.find(':', pos); - if (pos == std::string::npos) { - return fallback; - } - pos += 1; - while (pos < json.size() && std::isspace(static_cast(json[pos]))) { - pos += 1; - } - try { - return std::stod(json.substr(pos)); - } catch (...) { - return fallback; - } -} - -std::string findString(const std::string& json, const std::string& key) { - auto pos = json.find("\"" + key + "\""); - if (pos == std::string::npos) { - return {}; - } - pos = json.find(':', pos); - if (pos == std::string::npos) { - return {}; - } - pos += 1; - while (pos < json.size() && std::isspace(static_cast(json[pos]))) { - pos += 1; - } - if (pos >= json.size() || json[pos] != '"') { - return {}; - } - pos += 1; - - std::string result; - while (pos < json.size()) { - const char c = json[pos++]; - if (c == '"') { - break; - } - if (c == '\\' && pos < json.size()) { - const char escaped = json[pos++]; - switch (escaped) { - case '\\': - case '"': - case '/': - result.push_back(escaped); - break; - case 'n': - result.push_back('\n'); - break; - case 'r': - result.push_back('\r'); - break; - case 't': - result.push_back('\t'); - break; - default: - result.push_back(escaped); - break; - } - continue; - } - result.push_back(c); - } - return result; +// `code` stays ahead of `index`: older readers match on the `"code":…` substring. +void printWebcamUnavailable(size_t index, const std::string& deviceName, const char* message) { + std::cout << "{\"event\":\"warning\",\"code\":\"webcam-unavailable\",\"index\":" << index + << ",\"deviceName\":\"" << jsonEscape(deviceName) << "\",\"message\":\"" << message << "\"}" + << std::endl; } std::string parseWindowHandleFromSourceId(const std::string& sourceId) { @@ -607,7 +520,6 @@ bool parseConfig(const std::string& json, CaptureConfig& config) { config.webcamDeviceId = findString(json, "webcamDeviceId"); config.webcamDeviceName = findString(json, "webcamDeviceName"); config.webcamDirectShowClsid = findString(json, "webcamDirectShowClsid"); - config.webcamOutputPath = findString(json, "webcamPath"); config.webcamWidth = findInt(json, "webcamWidth", 0); config.webcamHeight = findInt(json, "webcamHeight", 0); config.webcamFps = findInt(json, "webcamFps", 0); @@ -673,6 +585,7 @@ int wmain(int argc, wchar_t* argv[]) { std::cerr << "ERROR: Could not enable per-monitor-v2 DPI awareness" << std::endl; return 1; } + const HighResolutionTiming highResolutionTiming; if (argc < 2) { std::cerr << "ERROR: Missing JSON config argument" << std::endl; @@ -681,8 +594,9 @@ int wmain(int argc, wchar_t* argv[]) { winrt::init_apartment(winrt::apartment_type::multi_threaded); + const std::string configJson = wideToUtf8(argv[1]); CaptureConfig config; - if (!parseConfig(wideToUtf8(argv[1]), config)) { + if (!parseConfig(configJson, config)) { std::cerr << "ERROR: Failed to parse config JSON" << std::endl; return 1; } @@ -753,7 +667,14 @@ int wmain(int argc, wchar_t* argv[]) { // A window is captured by whichever display it currently sits on, which // is the adapter that matters for the same reason a monitor's does. capturedMonitor = MonitorFromWindow(window, MONITOR_DEFAULTTONEAREST); - if (!session.initialize(window, config.fps, config.captureCursor)) { + // A window source would otherwise lose every context menu and dropdown, + // which are windows of their own (getopenscreen/openscreen#894). Windows 11 + // 24H2+ can draw them into this capture; OPENSCREEN_WGC_DISABLE_SECONDARY_ + // WINDOWS=1 restores the window-only capture. Never for a monitor source, + // which already contains everything on that screen. + const bool includeSecondaryWindows = + readEnvInt("OPENSCREEN_WGC_DISABLE_SECONDARY_WINDOWS", 0) != 1; + if (!session.initialize(window, config.fps, config.captureCursor, includeSecondaryWindows)) { std::cerr << "ERROR: Failed to initialize WGC window session" << std::endl; return 1; } @@ -775,37 +696,109 @@ int wmain(int argc, wchar_t* argv[]) { const int pixels = width * height; const int bitrate = pixels >= 3840 * 2160 ? 45'000'000 : pixels >= 2560 * 1440 ? 28'000'000 : 18'000'000; - WebcamCapture webcamCapture; - bool webcamActive = false; - bool writeSeparateWebcam = false; - if (config.webcamEnabled) { - if (!webcamCapture.initialize( - utf8ToWide(config.webcamDeviceId), - utf8ToWide(config.webcamDeviceName), - utf8ToWide(config.webcamDirectShowClsid), - config.webcamWidth, - config.webcamHeight, - config.webcamFps > 0 ? config.webcamFps : config.fps)) { + // Every camera of the take, in config order. The `webcams` list (or the + // legacy fields with a webcamPath) gives cameras that each write their own + // file. The legacy fields without a webcamPath are the one camera drawn + // into the screen frame as an inline picture-in-picture. + std::vector webcamConfigs = parseWebcamConfigs(configJson); + if (webcamConfigs.empty() && config.webcamEnabled) { + WebcamConfig inlineCamera; + inlineCamera.deviceId = config.webcamDeviceId; + inlineCamera.deviceName = config.webcamDeviceName; + inlineCamera.directShowClsid = config.webcamDirectShowClsid; + inlineCamera.width = config.webcamWidth; + inlineCamera.height = config.webcamHeight; + inlineCamera.fps = config.webcamFps; + webcamConfigs.push_back(std::move(inlineCamera)); + } + + std::vector> webcams; // unique_ptr: WebcamCapture/MFEncoder are not movable + // The devices this take's cameras opened, so two cameras of the same model + // (same name, and a browser id that matches no device) open two devices + // rather than the first one twice. A camera dropped later keeps its claim. + DeviceClaims deviceClaims; + for (size_t index = 0; index < webcamConfigs.size(); ++index) { + auto stream = std::make_unique(); + stream->config = webcamConfigs[index]; + stream->index = index; + stream->finalizeStepName = "webcam-encoder-finalize-" + std::to_string(index); + // Decided before initialize(), not after: it selects the capture pixel + // format, and only a camera going to its own file can use NV12 -- an inline + // picture-in-picture composite needs the frame as BGRA. + stream->writeSeparate = !stream->config.outputPath.empty(); + if (!stream->capture.initialize( + utf8ToWide(stream->config.deviceId), + utf8ToWide(stream->config.deviceName), + utf8ToWide(stream->config.directShowClsid), + stream->config.width, + stream->config.height, + stream->config.fps > 0 ? stream->config.fps : config.fps, + stream->writeSeparate, + deviceClaims)) { // Non-fatal: a screen+audio recording the user can still use is far // better than losing the whole recording because one camera device // didn't match. Report it so the renderer can inform the user (and, // historically, fall back to a browser-recorded webcam sidecar), but - // let capture continue without a native webcam track. - std::cerr << "WARNING: Failed to initialize native webcam capture; continuing without webcam" - << std::endl; - std::cout << "{\"event\":\"warning\",\"code\":\"webcam-unavailable\",\"message\":" - "\"Failed to initialize native webcam capture\"}" - << std::endl; - config.webcamEnabled = false; - } else { - std::cout << "{\"event\":\"webcam-format\",\"schemaVersion\":2,\"width\":" << webcamCapture.width() - << ",\"height\":" << webcamCapture.height() - << ",\"fps\":" << webcamCapture.fps() - << ",\"deviceName\":\"" << jsonEscape(wideToUtf8(webcamCapture.selectedDeviceName())) - << "\"}" << std::endl; - writeSeparateWebcam = !config.webcamOutputPath.empty(); + // let capture continue without this camera's track. + std::cerr << "WARNING: Failed to initialize native webcam capture for camera " << index + << "; continuing without it" << std::endl; + printWebcamUnavailable( + index, stream->config.deviceName, "Failed to initialize native webcam capture"); + continue; } - } + std::cout << "{\"event\":\"webcam-format\",\"schemaVersion\":2,\"index\":" << index + << ",\"width\":" << stream->capture.width() + << ",\"height\":" << stream->capture.height() + << ",\"fps\":" << stream->capture.fps() + << ",\"deviceName\":\"" << jsonEscape(wideToUtf8(stream->capture.selectedDeviceName())) + << "\"}" << std::endl; + stream->nominalIntervalHns = + static_cast(10'000'000ULL / std::max(1, stream->capture.fps())); + webcams.push_back(std::move(stream)); + } + // Only the legacy single camera without a file of its own is composited + // into the screen frame; every listed camera writes separately. + WebcamStream* inlineWebcam = + webcams.size() == 1 && !webcams[0]->writeSeparate ? webcams[0].get() : nullptr; + // A camera that opened but whose encoder or capture then would not start + // costs that camera, not the take -- with the screen and up to four cameras, + // running out of hardware encoder sessions is the likeliest way to get here. + // Only before the video writer runs: the vector is not touched after that. + const auto dropWebcam = [&](WebcamStream& stream, const char* message) { + std::cerr << "WARNING: " << message << " for camera " << stream.index << "; continuing without it" + << std::endl; + printWebcamUnavailable(stream.index, stream.config.deviceName, message); + stream.capture.stop(); + if (stream.writeSeparate) { + // Balances whatever initialize() got through, then removes the + // file it may have created: nothing was ever written to it. + stream.encoder.finalize(); + // A file that never got created is fine; one that could not be + // removed is left as a 0-byte stub, which the app cleans up. + if (!DeleteFileW(utf8ToWide(stream.config.outputPath).c_str())) { + const DWORD deleteError = GetLastError(); + if (deleteError != ERROR_FILE_NOT_FOUND) { + std::cerr << "WARNING: Could not remove the dropped camera file " + << stream.config.outputPath << " (GetLastError=" << deleteError << ")" + << std::endl; + } + } + } + stream.active = false; + if (inlineWebcam == &stream) { + inlineWebcam = nullptr; + } + }; + const auto eraseDroppedWebcams = [&](const std::vector& dropped) { + webcams.erase( + std::remove_if( + webcams.begin(), + webcams.end(), + [&](const std::unique_ptr& stream) { + return std::find(dropped.begin(), dropped.end(), stream.get()) != dropped.end(); + }), + webcams.end()); + }; WasapiLoopbackCapture loopbackCapture; WasapiLoopbackCapture microphoneCapture; @@ -930,13 +923,13 @@ int wmain(int argc, wchar_t* argv[]) { // // The other two conditions are unchanged and still required: software // encoding and inline webcam PiP both need the frame in system memory, - // which the DXGI path does not produce. config.webcamEnabled, not - // webcamActive -- the latter is only set once webcam capture has started, - // well after this. + // which the DXGI path does not produce. Decided from the initialized + // cameras, not from `active` -- that is only set once webcam capture has + // started, well after this. encoderOptions.useDxgiInput = readEnvInt("OPENSCREEN_WGC_ENABLE_DXGI_INPUT", 0) == 1 && !config.preferSoftwareEncoder && - (!config.webcamEnabled || writeSeparateWebcam); + inlineWebcam == nullptr; MFEncoder encoder; if (!encoder.initialize( @@ -975,27 +968,41 @@ int wmain(int argc, wchar_t* argv[]) { // on stop. << ",\"videoEncoderRuntime\":\"" << encoder.videoEncoderRuntime() << "\"}" << std::endl; - MFEncoder webcamEncoder; - if (writeSeparateWebcam) { + std::vector droppedWebcams; + for (const auto& stream : webcams) { + if (!stream->writeSeparate) { + continue; + } MFEncoderOptions webcamEncoderOptions = encoderOptions; webcamEncoderOptions.injectDefaultSinkWriterFailureOnce = false; webcamEncoderOptions.useDxgiInput = false; - const int webcamPixels = std::max(1, webcamCapture.width()) * std::max(1, webcamCapture.height()); - const int webcamBitrate = webcamPixels >= 1280 * 720 ? 8'000'000 : 4'000'000; - if (!webcamEncoder.initialize( - utf8ToWide(config.webcamOutputPath), - webcamCapture.width(), - webcamCapture.height(), - webcamCapture.fps(), + // The two-step ladder this replaces topped out at 8 Mbit/s for anything + // 720p or larger. That was sized for a camera nobody had configured + // above 640x480; now that the capture runs at the camera's real + // resolution, 8 Mbit/s starves a 1440p or 2160p frame badly enough to + // undo the extra pixels. The tiers mirror the screen ladder above. + const int webcamPixels = + std::max(1, stream->capture.width()) * std::max(1, stream->capture.height()); + const int webcamBitrate = webcamPixels >= 3840 * 2160 ? 40'000'000 + : webcamPixels >= 2560 * 1440 ? 24'000'000 + : webcamPixels >= 1920 * 1080 ? 16'000'000 + : webcamPixels >= 1280 * 720 ? 8'000'000 + : 4'000'000; + if (!stream->encoder.initialize( + utf8ToWide(stream->config.outputPath), + stream->capture.width(), + stream->capture.height(), + stream->capture.fps(), webcamBitrate, session.device(), session.context(), nullptr, webcamEncoderOptions)) { - std::cerr << "ERROR: Failed to initialize native webcam encoder" << std::endl; - return 1; + dropWebcam(*stream, "Failed to initialize native webcam encoder"); + droppedWebcams.push_back(stream.get()); } } + eraseDroppedWebcams(droppedWebcams); // By default, no mutex guards frame handoff: writeVideoFrames is the // only thread that ever touches WGC or latestFrameTexture. It pulls each @@ -1019,11 +1026,6 @@ int wmain(int argc, wchar_t* argv[]) { // them is the next bug report, and neither is worth a log line each. std::atomic contendedFrames = 0; Microsoft::WRL::ComPtr latestFrameTexture; - std::vector latestWebcamFrame; - int latestWebcamWidth = 0; - int latestWebcamHeight = 0; - uint64_t latestWebcamSequence = 0; - bool hasVisibleWebcamFrame = false; // Legacy-path-only state. frameMutex guards latestFrameTexture/ // legacyLatestFrameTimestampHns between WGC's callback thread (writer) @@ -1073,11 +1075,11 @@ int wmain(int argc, wchar_t* argv[]) { } auto writeVideoFrames = [&]() { + const MmcssThread mmcss(L"Capture"); const auto frameDuration = std::chrono::duration_cast( std::chrono::duration(1.0 / config.fps)); uint64_t frameIndex = 0; int64_t lastEncodedVideoTimestampHns = -1; - int64_t lastWebcamTimestampHns = -1; // Media Foundation's H.264 encoder MFT does not honor irregular input // sample times for a VFR source: it numbers output samples // sequentially at its configured nominal frame rate regardless of the @@ -1088,18 +1090,21 @@ int wmain(int argc, wchar_t* argv[]) { // webcam encoder on a real-time-paced cadence (duplicating the // latest available camera frame when the camera hasn't produced a // newer one yet), so "sample N is at N/fps" is actually correct. - int64_t nextWebcamWriteDueHns = 0; - const int64_t nominalWebcamIntervalHns = - static_cast(10'000'000ULL / std::max(1, webcamCapture.fps())); + // The cadence state is per camera: see WebcamStream. auto nextFrameDue = std::chrono::steady_clock::now(); int64_t firstFrameTimestampHns = -1; int64_t latestFrameTimestampHns = 0; while (!control.stopRequested && !encodeFailed) { Microsoft::WRL::ComPtr videoSample; - Microsoft::WRL::ComPtr webcamSample; bool hasVideoSample = false; - bool hasWebcamSample = false; + for (const auto& stream : webcams) { + stream->pendingSample.Reset(); + } + // Whether the picture this tick encodes differs from the last one: + // a new WGC frame, or a new camera frame drawn into it. The legacy + // callback path cannot tell, so it always reads back. + bool pictureChanged = legacyFrameCallback; std::unique_lock legacyLock; { @@ -1138,6 +1143,7 @@ int wmain(int argc, wchar_t* argv[]) { ID3D11Texture2D* wgcTexture = nullptr; int64_t wgcTimestampHns = 0; const bool gotFrame = session.tryGetNextFrame(&wgcTexture, &wgcTimestampHns); + pictureChanged = gotFrame; if (gotFrame) { if (!latestFrameTexture) { D3D11_TEXTURE2D_DESC desc{}; @@ -1171,22 +1177,35 @@ int wmain(int argc, wchar_t* argv[]) { continue; } } - if (webcamActive) { - WebcamFrameSnapshot candidateWebcamFrame; - if (webcamCapture.copyLatestFrame(candidateWebcamFrame) && - candidateWebcamFrame.sequence != latestWebcamSequence && - hasVisibleBgraContent(candidateWebcamFrame.data)) { - latestWebcamFrame = std::move(candidateWebcamFrame.data); - latestWebcamWidth = candidateWebcamFrame.width; - latestWebcamHeight = candidateWebcamFrame.height; - latestWebcamSequence = candidateWebcamFrame.sequence; - hasVisibleWebcamFrame = true; + // Screen first, then every camera, as before there was more than one. + for (const auto& stream : webcams) { + if (stream->active && stream->capture.isLost()) { + // Unplugged mid-take: its file ends here instead of + // repeating the last picture to the end, it is left out + // of `webcamPaths` and still finalized at stop. The + // screen and the other cameras go on. + std::cerr << "ERROR: Camera " << stream->index + << " was lost during the take; disabling it" << std::endl; + stream->active = false; + if (inlineWebcam == stream.get()) { + // Nor is its frozen picture drawn into the screen. + inlineWebcam = nullptr; + pictureChanged = true; + } + continue; + } + if (stream->active && stream->pullVisibleFrame()) { + pictureChanged = pictureChanged || !stream->writeSeparate; } } - const BgraFrameView webcamFrame{ - hasVisibleWebcamFrame && !latestWebcamFrame.empty() ? latestWebcamFrame.data() : nullptr, - latestWebcamWidth, - latestWebcamHeight, + // The frame composited into the screen picture, for the legacy + // inline camera only. + const BgraFrameView inlineWebcamFrame{ + inlineWebcam && inlineWebcam->hasVisibleFrame && !inlineWebcam->latestFrame.empty() + ? inlineWebcam->latestFrame.data() + : nullptr, + inlineWebcam ? inlineWebcam->latestWidth : 0, + inlineWebcam ? inlineWebcam->latestHeight : 0, }; const int64_t syntheticTimestampHns = static_cast((frameIndex * 10'000'000ULL) / config.fps); @@ -1204,16 +1223,24 @@ int wmain(int argc, wchar_t* argv[]) { frameTimestampHns = lastEncodedVideoTimestampHns + static_cast(10'000'000ULL / config.fps); } - if (writeSeparateWebcam && webcamFrame.data) { - // Anchor to the same recording-start origin as screen video/audio, - // using real elapsed host-clock time (not a synthetic frame-index - // clock) so a long recording can't accumulate clock-origin drift. - const auto elapsedSinceStart = std::chrono::steady_clock::now() - control.recordingStartedAt; - const int64_t elapsedHns = std::chrono::duration_cast< - std::chrono::duration>>(elapsedSinceStart) - .count(); - const int64_t targetElapsedHns = - std::max(0, elapsedHns - control.pausedDurationHns()); + // Anchor to the same recording-start origin as screen video/audio, + // using real elapsed host-clock time (not a synthetic frame-index + // clock) so a long recording can't accumulate clock-origin drift. + // Read once per tick: every camera of this tick shares it. + int64_t targetElapsedHns = -1; + for (const auto& stream : webcams) { + if (!stream->active || !stream->writeSeparate || !stream->hasVisibleFrame || + stream->latestFrame.empty()) { + continue; + } + if (targetElapsedHns < 0) { + const auto elapsedSinceStart = + std::chrono::steady_clock::now() - control.recordingStartedAt; + const int64_t elapsedHns = std::chrono::duration_cast< + std::chrono::duration>>(elapsedSinceStart) + .count(); + targetElapsedHns = std::max(0, elapsedHns - control.pausedDurationHns()); + } // The H.264 encoder MFT does not honor irregular per-sample // timestamps for a VFR source -- it numbers output samples // sequentially at its configured nominal rate regardless of the @@ -1222,27 +1249,40 @@ int wmain(int argc, wchar_t* argv[]) { // to feed the encoder *at* that nominal cadence, duplicating // the latest available camera frame when the camera hasn't // produced a newer one yet (VFR capture -> CFR encode resampling). - if (targetElapsedHns >= nextWebcamWriteDueHns) { - int64_t webcamTimestampHns = targetElapsedHns; - if (lastWebcamTimestampHns >= 0 && webcamTimestampHns <= lastWebcamTimestampHns) { - webcamTimestampHns = lastWebcamTimestampHns + nominalWebcamIntervalHns; - } - // Capture the sample here, but submit it to the sink - // writer OUTSIDE this block below (issue #115) so a - // slow WriteSample can't hold up the next frame pull. - hasWebcamSample = webcamEncoder.captureBgraSample(webcamFrame, webcamTimestampHns, webcamSample); - if (!hasWebcamSample) { - encodeFailed = true; - control.requestStop(); - break; - } - lastWebcamTimestampHns = webcamTimestampHns; - nextWebcamWriteDueHns += nominalWebcamIntervalHns; - if (nextWebcamWriteDueHns <= targetElapsedHns) { - // Fell behind (e.g. coming out of a pause, or a stall) -- - // resync to now instead of trying to catch up frame-by-frame. - nextWebcamWriteDueHns = targetElapsedHns + nominalWebcamIntervalHns; - } + if (targetElapsedHns < stream->nextWriteDueHns) { + continue; + } + int64_t webcamTimestampHns = targetElapsedHns; + if (stream->lastTimestampHns >= 0 && webcamTimestampHns <= stream->lastTimestampHns) { + webcamTimestampHns = stream->lastTimestampHns + stream->nominalIntervalHns; + } + const BgraFrameView webcamFrame{ + stream->latestFrame.data(), stream->latestWidth, stream->latestHeight}; + // Capture the sample here, but submit it to the sink + // writer OUTSIDE this block below (issue #115) so a + // slow WriteSample can't hold up the next frame pull. + const bool captured = + stream->capture.deliversNv12() + ? stream->encoder.captureNv12Sample( + Nv12FrameView{webcamFrame.data, webcamFrame.width, webcamFrame.height}, + webcamTimestampHns, + stream->pendingSample) + : stream->encoder.captureBgraSample( + webcamFrame, webcamTimestampHns, stream->pendingSample); + if (!captured) { + // One camera failing costs that camera, not the take. + std::cerr << "ERROR: Failed to capture a sample for camera " << stream->index + << "; disabling it" << std::endl; + stream->pendingSample.Reset(); + stream->active = false; + continue; + } + stream->lastTimestampHns = webcamTimestampHns; + stream->nextWriteDueHns += stream->nominalIntervalHns; + if (stream->nextWriteDueHns <= targetElapsedHns) { + // Fell behind (e.g. coming out of a pause, or a stall) -- + // resync to now instead of trying to catch up frame-by-frame. + stream->nextWriteDueHns = targetElapsedHns + stream->nominalIntervalHns; } } if (testStallReadbackMs > 0) { @@ -1266,7 +1306,12 @@ int wmain(int argc, wchar_t* argv[]) { // this struct's request: it falls back to the CPU path on // its own when the GPU path does not fit the machine. bool captured = false; - if (usesDxgiInput) { + if (!usesDxgiInput && !pictureChanged && + encoder.repeatLastVideoSample(frameTimestampHns, videoSample)) { + // Nothing new to read back (#925): the static screen, + // most of a demo, costs no GPU copy and no conversion. + captured = true; + } else if (usesDxgiInput) { captured = encoder.captureDxgiSample( latestFrameTexture.Get(), frameTimestampHns, @@ -1275,7 +1320,7 @@ int wmain(int argc, wchar_t* argv[]) { captured = encoder.captureVideoSample( latestFrameTexture.Get(), frameTimestampHns, - !writeSeparateWebcam && webcamFrame.data ? &webcamFrame : nullptr, + inlineWebcamFrame.data ? &inlineWebcamFrame : nullptr, videoSample); } if (!captured) { @@ -1315,10 +1360,15 @@ int wmain(int argc, wchar_t* argv[]) { // Stop detection has nothing to do with this ordering -- that is // CaptureControl::stopMutex/stopCv, checked by the loop condition // above, unrelated to sample submission (issue #252). - if (hasWebcamSample && !webcamEncoder.submitVideoSample(webcamSample.Get())) { - encodeFailed = true; - control.requestStop(); - break; + for (const auto& stream : webcams) { + if (stream->pendingSample && !stream->encoder.submitVideoSample(stream->pendingSample.Get())) { + // Disables this camera only; the screen and the other + // cameras keep recording. + std::cerr << "ERROR: Failed to submit a sample for camera " << stream->index + << "; disabling it" << std::endl; + stream->active = false; + } + stream->pendingSample.Reset(); } if (hasVideoSample && !encoder.submitVideoSample(videoSample.Get())) { encodeFailed = true; @@ -1444,41 +1494,52 @@ int wmain(int argc, wchar_t* argv[]) { stopRenderKeepAliveIfActive(); return 1; } - if (config.webcamEnabled) { - if (!webcamCapture.start()) { - microphoneCapture.stop(); - loopbackCapture.stop(); - stopDeviceWatchIfActive(); - stopRenderKeepAliveIfActive(); - if (audioMixer) { - audioMixer->stop(); - } - std::cerr << "ERROR: Failed to start native webcam capture" << std::endl; - return 1; + const auto stopWebcamCaptures = [&]() { + for (const auto& stream : webcams) { + stream->capture.stop(); } - webcamActive = true; + }; + droppedWebcams.clear(); + for (const auto& stream : webcams) { + if (!stream->capture.start()) { + dropWebcam(*stream, "Failed to start native webcam capture"); + droppedWebcams.push_back(stream.get()); + continue; + } + stream->active = true; + } + eraseDroppedWebcams(droppedWebcams); + if (!webcams.empty()) { + // One 3 s budget for all cameras together, not 3 s each: they warm up + // in parallel, and the screen recording should not start later per camera. const auto webcamDeadline = std::chrono::steady_clock::now() + std::chrono::seconds(3); - while (std::chrono::steady_clock::now() < webcamDeadline && !hasVisibleWebcamFrame) { - WebcamFrameSnapshot candidateWebcamFrame; - if (webcamCapture.copyLatestFrame(candidateWebcamFrame) && - hasVisibleBgraContent(candidateWebcamFrame.data)) { - latestWebcamFrame = std::move(candidateWebcamFrame.data); - latestWebcamWidth = candidateWebcamFrame.width; - latestWebcamHeight = candidateWebcamFrame.height; - latestWebcamSequence = candidateWebcamFrame.sequence; - hasVisibleWebcamFrame = true; + const auto allVisible = [&]() { + return std::all_of(webcams.begin(), webcams.end(), [](const auto& stream) { + return stream->hasVisibleFrame; + }); + }; + while (std::chrono::steady_clock::now() < webcamDeadline && !allVisible()) { + for (const auto& stream : webcams) { + if (!stream->hasVisibleFrame) { + stream->pullVisibleFrame(); + } + } + if (allVisible()) { break; } std::this_thread::sleep_for(std::chrono::milliseconds(20)); } - if (!hasVisibleWebcamFrame) { - std::cerr << "WARNING: Native webcam started but no visible frame was available before screen capture" - << std::endl; + for (const auto& stream : webcams) { + if (!stream->hasVisibleFrame) { + std::cerr << "WARNING: Native webcam " << stream->index + << " started but no visible frame was available before screen capture" + << std::endl; + } } } if (!session.start()) { - webcamCapture.stop(); + stopWebcamCaptures(); microphoneCapture.stop(); loopbackCapture.stop(); stopDeviceWatchIfActive(); @@ -1528,7 +1589,7 @@ int wmain(int argc, wchar_t* argv[]) { loopbackCapture.stop(); stopDeviceWatchIfActive(); stopRenderKeepAliveIfActive(); - webcamCapture.stop(); + stopWebcamCaptures(); if (audioMixer) { audioMixer->stop(); } @@ -1542,6 +1603,17 @@ int wmain(int argc, wchar_t* argv[]) { control.waitForStop(); + // The take ends here, for the sound as for the picture: the video writer + // stops on this same request. The mixer is clock-driven, so left running + // it would go on writing chunks through the half second the steps below + // take, and every take ended with ~0.5 s of audio past its video + // (getopenscreen/openscreen#942, measured on three takes). A pause writes + // what the cushion holds up to this instant and then nothing; one already + // in place is left where it is. + if (audioMixer) { + audioMixer->setPaused(true); + } + const auto stopStart = std::chrono::steady_clock::now(); auto stopElapsedMs = [&] { return std::chrono::duration_cast( @@ -1674,7 +1746,7 @@ int wmain(int argc, wchar_t* argv[]) { logStopStep("render-keepalive"); } beginStopStep("webcam", stepBudgetMs); - webcamCapture.stop(); + stopWebcamCaptures(); logStopStep("webcam"); beginStopStep("audio-mixer", stepBudgetMs); if (audioMixer) { @@ -1770,20 +1842,52 @@ int wmain(int argc, wchar_t* argv[]) { if (!encodeFailed && screenFinalized) { std::cout << "{\"event\":\"recording-stopped\",\"schemaVersion\":2,\"screenPath\":\"" << jsonEscape(config.outputPath) << "\""; - if (writeSeparateWebcam) { - std::cout << ",\"webcamPath\":\"" << jsonEscape(config.webcamOutputPath) << "\""; + // `webcamPath` stays for camera 0, as before; `webcamPaths` lists every + // camera still recording at stop, in index order. Both are printed + // before the camera files are finalized, for the reason above -- so they + // name the files that were being written, and a camera whose finalize + // fails below is reported on stderr, not removed from this list. + // `webcamPaths` is printed, possibly empty, whenever a camera wrote a + // file of its own: its presence is how the app knows a camera missing + // from it stopped early, rather than an older helper that never sent it. + std::vector recordedWebcams; + bool anySeparateWebcam = false; + for (const auto& stream : webcams) { + anySeparateWebcam = anySeparateWebcam || stream->writeSeparate; + if (stream->writeSeparate && stream->active) { + recordedWebcams.push_back(stream.get()); + } + } + if (!recordedWebcams.empty() && recordedWebcams.front()->index == 0) { + std::cout << ",\"webcamPath\":\"" << jsonEscape(recordedWebcams.front()->config.outputPath) + << "\""; + } + if (anySeparateWebcam) { + std::cout << ",\"webcamPaths\":["; + for (size_t i = 0; i < recordedWebcams.size(); ++i) { + std::cout << (i == 0 ? "\"" : ",\"") << jsonEscape(recordedWebcams[i]->config.outputPath) + << "\""; + } + std::cout << "]"; } std::cout << "}" << std::endl; std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl; } + // Every camera that wrote a file is finalized, including one disabled + // mid-take: what it wrote before failing is still worth a playable index. bool webcamFinalized = true; - if (writeSeparateWebcam) { - beginStopStep("webcam-encoder-finalize", shutdownBudgetMs); - webcamFinalized = webcamEncoder.finalize(); - logStopStep("webcam-encoder-finalize"); - if (!webcamFinalized) { - std::cerr << "ERROR: Failed to finalize the webcam recording" << std::endl; + for (const auto& stream : webcams) { + if (!stream->writeSeparate) { + continue; + } + beginStopStep(stream->finalizeStepName.c_str(), shutdownBudgetMs); + const bool finalized = stream->encoder.finalize(); + logStopStep(stream->finalizeStepName.c_str()); + if (!finalized) { + std::cerr << "ERROR: Failed to finalize the webcam recording for camera " << stream->index + << std::endl; + webcamFinalized = false; } } diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp index b4dabcc54..23bc69331 100644 --- a/electron/native/wgc-capture/src/mf_encoder.cpp +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -10,6 +10,8 @@ #include #include +#include + #include #include #include @@ -592,6 +594,94 @@ struct StageGuard { } // namespace +// BGRA to NV12, BT.709 studio range, which is what the compositor decodes every +// recording as (getopenscreen/openscreen#923). Media Foundation's own colour +// converter, which the sink writer used to insert in front of the encoder for +// RGB32 input, produced BT.601 whatever the media types said. Chroma is the +// average of each 2x2 block, and the width must be even, as H.264 needs it anyway +// (main.cpp rounds the capture down); an odd last row repeats its neighbour. +// Fixed point: the coefficients are BT.709's scaled by 2^15 (luma) and 2^16 +// (chroma), so a solid primary lands within one code value of the table. +void convertBgraToNv12Bt709(const BYTE* bgra, int stride, int width, int height, BYTE* nv12) { + BYTE* luma = nv12; + BYTE* chroma = nv12 + static_cast(width) * height; + // Luma four pixels at a time in SSE2, which every x64 CPU has: this runs on + // the video writer's thread for every frame, and the scalar loop alone cost + // 2.5 ms a 1080p frame. 15-bit coefficients so they fit madd's int16 lanes. + const __m128i zero = _mm_setzero_si128(); + const __m128i lumaCoefficients = _mm_setr_epi16(2032, 20127, 5983, 0, 2032, 20127, 5983, 0); + const __m128i lumaRounding = _mm_set1_epi32(16384); + const __m128i lumaOffset = _mm_set1_epi32(16); + const auto lumaOfTwo = [&](__m128i pixels) { + // [b0*cb + g0*cg, r0*cr, b1*cb + g1*cg, r1*cr] -> each pixel's sum in lanes 0 and 2. + const __m128i products = _mm_madd_epi16(pixels, lumaCoefficients); + return _mm_add_epi32(products, _mm_shuffle_epi32(products, _MM_SHUFFLE(2, 3, 0, 1))); + }; + for (int y = 0; y < height; y += 1) { + const BYTE* row = bgra + static_cast(y) * stride; + BYTE* out = luma + static_cast(y) * width; + int x = 0; + for (; x + 4 <= width; x += 4) { + const __m128i pixels = _mm_loadu_si128(reinterpret_cast(row + x * 4)); + const __m128i first = lumaOfTwo(_mm_unpacklo_epi8(pixels, zero)); + const __m128i second = lumaOfTwo(_mm_unpackhi_epi8(pixels, zero)); + __m128i sums = _mm_castps_si128(_mm_shuffle_ps( + _mm_castsi128_ps(first), _mm_castsi128_ps(second), _MM_SHUFFLE(2, 0, 2, 0))); + sums = _mm_add_epi32(_mm_srai_epi32(_mm_add_epi32(sums, lumaRounding), 15), lumaOffset); + const __m128i bytes = _mm_packus_epi16(_mm_packs_epi32(sums, zero), zero); + const int packed = _mm_cvtsi128_si32(bytes); + std::memcpy(out + x, &packed, 4); + } + for (; x < width; x += 1) { + const int b = row[x * 4]; + const int g = row[x * 4 + 1]; + const int r = row[x * 4 + 2]; + out[x] = static_cast(((5983 * r + 20127 * g + 2032 * b + 16384) >> 15) + 16); + } + } + // Chroma the same way, two 2x2 blocks at a time: the four pixels of each + // block summed in int16 lanes (at most 1020), then one madd per channel. + const __m128i cbCoefficients = _mm_setr_epi16(28784, -22189, -6596, 0, 28784, -22189, -6596, 0); + const __m128i crCoefficients = _mm_setr_epi16(-2642, -26142, 28784, 0, -2642, -26142, 28784, 0); + const __m128i chromaRounding = _mm_set1_epi32(131072); + const __m128i chromaOffset = _mm_set1_epi32(128); + const auto chromaOfTwo = [&](__m128i blocks, __m128i coefficients) { + const __m128i products = _mm_madd_epi16(blocks, coefficients); + const __m128i sums = _mm_add_epi32(products, _mm_shuffle_epi32(products, _MM_SHUFFLE(2, 3, 0, 1))); + return _mm_shuffle_epi32(sums, _MM_SHUFFLE(2, 0, 2, 0)); + }; + for (int y = 0; y < height; y += 2) { + const BYTE* top = bgra + static_cast(y) * stride; + const BYTE* bottom = y + 1 < height ? top + stride : top; + BYTE* out = chroma + static_cast(y / 2) * width; + int x = 0; + for (; x + 4 <= width; x += 4) { + const __m128i upper = _mm_loadu_si128(reinterpret_cast(top + x * 4)); + const __m128i lower = _mm_loadu_si128(reinterpret_cast(bottom + x * 4)); + // [pixel 0 + pixel 1 | pixel 2 + pixel 3] of both rows, as int16 B, G, R, A. + const __m128i left = _mm_add_epi16(_mm_unpacklo_epi8(upper, zero), _mm_unpacklo_epi8(lower, zero)); + const __m128i right = _mm_add_epi16(_mm_unpackhi_epi8(upper, zero), _mm_unpackhi_epi8(lower, zero)); + const __m128i blocks = _mm_unpacklo_epi64( + _mm_add_epi16(left, _mm_srli_si128(left, 8)), _mm_add_epi16(right, _mm_srli_si128(right, 8))); + __m128i values = _mm_unpacklo_epi32( + chromaOfTwo(blocks, cbCoefficients), chromaOfTwo(blocks, crCoefficients)); + values = _mm_add_epi32(_mm_srai_epi32(_mm_add_epi32(values, chromaRounding), 18), chromaOffset); + const int packed = _mm_cvtsi128_si32(_mm_packus_epi16(_mm_packs_epi32(values, zero), zero)); + std::memcpy(out + x, &packed, 4); + } + for (; x < width; x += 2) { + const int right = (x + 1 < width ? x + 1 : x) * 4; + const int left = x * 4; + const int b = top[left] + top[right] + bottom[left] + bottom[right]; + const int g = top[left + 1] + top[right + 1] + bottom[left + 1] + bottom[right + 1]; + const int r = top[left + 2] + top[right + 2] + bottom[left + 2] + bottom[right + 2]; + // Sums of four, so the shift is two bits wider. + out[x] = static_cast(((-6596 * r - 22189 * g + 28784 * b + 131072) >> 18) + 128); + out[x + 1] = static_cast(((28784 * r - 26142 * g - 2642 * b + 131072) >> 18) + 128); + } + } +} + MFEncoder::~MFEncoder() { finalize(); } @@ -696,6 +786,7 @@ bool MFEncoder::initialize( if (!succeeded(MFStartup(MF_VERSION), "MFStartup")) { return false; } + mfStarted_ = true; if (useDxgiInput_ && !initializeDxgiPipeline()) { std::cerr << "WARNING: The GPU DXGI encode path is unavailable on this machine; " @@ -715,6 +806,21 @@ bool MFEncoder::initialize( setFrameSize(outputType.Get(), static_cast(width_), static_cast(height_)); setFrameRate(outputType.Get(), static_cast(fps_)); setPixelAspectRatio(outputType.Get()); + // Left unset, every encoder on the machine chose Constrained Baseline: one + // reference frame, CAVLC. Re-encoding real takes through the same NVENC MFT + // at an equal bitrate, High gained 1.7 dB PSNR on the screen and 1.9 VMAF on + // the webcam (getopenscreen/openscreen#922). Every hardware H.264 encoder and + // the Microsoft software one support it. No B-frames: the encoders add none + // unless asked, and they broke the fragmented writer on macOS. + outputType->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_High); + // Carried on the H.264 type as well so the MP4 sink writes the matching + // colour tags instead of leaving players to guess from the frame size. + // Primaries and transfer too: without them the webcam track read + // `reserved`, and the screen track, untagged, read `unknown` throughout. + outputType->SetUINT32(MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_16_235); + outputType->SetUINT32(MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709); + outputType->SetUINT32(MF_MT_VIDEO_PRIMARIES, MFVideoPrimaries_BT709); + outputType->SetUINT32(MF_MT_TRANSFER_FUNCTION, MFVideoTransFunc_709); Microsoft::WRL::ComPtr inputType; if (!succeeded(MFCreateMediaType(&inputType), "MFCreateMediaType(input)")) { @@ -725,52 +831,50 @@ bool MFEncoder::initialize( // type the RGB32 path would have produced from scratch. Every attribute // one mode sets is deleted by the other; nothing carries over. auto configureVideoInputType = [&](bool dxgi) { + // NV12 on every path: GPU NV12, and system-memory NV12 -- from the + // camera as it came (the webcam), or converted here from BGRA + // (`convertBgraToNv12Bt709`). BGRA used to go in as RGB32, and the + // colour converter the sink writer put in front of the encoder turned it + // into BT.601, whatever the types said (#923). inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); - inputType->SetGUID(MF_MT_SUBTYPE, dxgi ? MFVideoFormat_NV12 : MFVideoFormat_RGB32); + inputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12); inputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); - if (dxgi) { - inputType->DeleteItem(MF_MT_DEFAULT_STRIDE); - // The video processor below converts full-range BGRA into - // studio-range BT.709, so say so. Left untagged, the encoder and - // the player each pick their own default (BT.601 is the common - // one) and the recording comes back with shifted colours the CPU - // path does not have. + if (!dxgi) { + // NV12's declared stride is the Y plane's, which is one byte per + // pixel -- not the four an RGB32 row needs. + inputType->SetUINT32(MF_MT_DEFAULT_STRIDE, static_cast(width_)); + // A camera's NV12 is studio-range, and at these sizes BT.709. + // Untagged, the encoder and the player each fall back to their own + // default and the recording comes back with shifted colours. inputType->SetUINT32(MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_16_235); inputType->SetUINT32(MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709); - } else { - inputType->SetUINT32(MF_MT_DEFAULT_STRIDE, static_cast(width_ * 4)); - inputType->DeleteItem(MF_MT_VIDEO_NOMINAL_RANGE); - inputType->DeleteItem(MF_MT_YUV_MATRIX); + setFrameSize(inputType.Get(), static_cast(width_), static_cast(height_)); + setFrameRate(inputType.Get(), static_cast(fps_)); + setPixelAspectRatio(inputType.Get()); + return; } + inputType->DeleteItem(MF_MT_DEFAULT_STRIDE); + // The video processor below converts full-range BGRA into + // studio-range BT.709, so say so. Left untagged, the encoder and + // the player each pick their own default (BT.601 is the common + // one) and the recording comes back with shifted colours. + inputType->SetUINT32(MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_16_235); + inputType->SetUINT32(MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709); setFrameSize(inputType.Get(), static_cast(width_), static_cast(height_)); setFrameRate(inputType.Get(), static_cast(fps_)); setPixelAspectRatio(inputType.Get()); }; - // Carried on the H.264 type as well so the MP4 sink writes the matching - // colour tags instead of leaving players to guess from the frame size. - auto configureOutputColorTags = [&](bool dxgi) { - if (dxgi) { - outputType->SetUINT32(MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_16_235); - outputType->SetUINT32(MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709); - } else { - outputType->DeleteItem(MF_MT_VIDEO_NOMINAL_RANGE); - outputType->DeleteItem(MF_MT_YUV_MATRIX); - } - }; - // The allocator is the last thing that can refuse the GPU path, and it can // only be built once the NV12 type exists. Falling back here costs nothing // but the type rewrite, because no sink writer has been created yet. configureVideoInputType(useDxgiInput_); - configureOutputColorTags(useDxgiInput_); if (useDxgiInput_ && !initializeSampleAllocator(inputType.Get())) { std::cerr << "WARNING: The DXGI sample allocator is unavailable on this machine; " << "using the CPU readback path." << std::endl; releaseDxgiPipeline(); useDxgiInput_ = false; configureVideoInputType(false); - configureOutputColorTags(false); } bool injectedDefaultSinkWriterFailure = false; @@ -869,8 +973,18 @@ bool MFEncoder::initialize( // track: an H.264 input type against an AAC stream sink has no encoder // that can bridge it, so a bad mapping fails loudly here instead of // quietly writing video samples into the audio track. - if (!succeeded(sinkWriter_->SetInputMediaType(videoStreamIndex_, inputType.Get(), nullptr), - "SetInputMediaType")) { + // No B-frames. High allows them and the Microsoft software encoder uses + // them unless told otherwise (has_b_frames=1, measured); their negative + // composition offsets are what broke the fragmented writer on macOS. + // Passed as encoding parameters because the encoder only reads this + // before its types are set: ICodecAPI::SetValue afterwards is refused. + Microsoft::WRL::ComPtr encodingParameters; + if (SUCCEEDED(MFCreateAttributes(&encodingParameters, 1))) { + encodingParameters->SetUINT32(CODECAPI_AVEncMPVDefaultBPictureCount, 0); + } + if (!succeeded( + sinkWriter_->SetInputMediaType(videoStreamIndex_, inputType.Get(), encodingParameters.Get()), + "SetInputMediaType")) { return false; } if (!forceSoftwareEncoder) { @@ -928,7 +1042,6 @@ bool MFEncoder::initialize( releaseDxgiPipeline(); useDxgiInput_ = false; configureVideoInputType(false); - configureOutputColorTags(false); if (configureSinkWriterAttempt(false, kVideoEncoderSelectionDefault, false, true)) { return true; } @@ -1078,19 +1191,24 @@ bool MFEncoder::copyFrameToBuffer( } const DWORD rowBytes = static_cast(width_ * 4); - const DWORD requiredBytes = rowBytes * static_cast(height_); - if (destinationSize < requiredBytes) { + if (destinationSize < static_cast(width_ * height_ * 3 / 2)) { context_->Unmap(stagingTexture_.Get(), 0); std::cerr << "ERROR: Media Foundation buffer is too small" << std::endl; return false; } auto* source = static_cast(mapped.pData); - for (int y = 0; y < height_; y += 1) { - std::memcpy(destination + rowBytes * y, source + mapped.RowPitch * y, rowBytes); - } if (webcamFrame) { - compositeWebcam(destination, width_, height_, *webcamFrame); + // The picture-in-picture is drawn in BGRA, so the frame goes through a + // copy it can be drawn on before the conversion. + bgraScratch_.resize(static_cast(rowBytes) * height_); + for (int y = 0; y < height_; y += 1) { + std::memcpy(bgraScratch_.data() + rowBytes * y, source + mapped.RowPitch * y, rowBytes); + } + compositeWebcam(bgraScratch_.data(), width_, height_, *webcamFrame); + convertBgraToNv12Bt709(bgraScratch_.data(), static_cast(rowBytes), width_, height_, destination); + } else { + convertBgraToNv12Bt709(source, static_cast(mapped.RowPitch), width_, height_, destination); } context_->Unmap(stagingTexture_.Get(), 0); @@ -1103,25 +1221,24 @@ bool MFEncoder::copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destinat } const DWORD rowBytes = static_cast(width_ * 4); - const DWORD requiredBytes = rowBytes * static_cast(height_); - if (destinationSize < requiredBytes) { + if (destinationSize < static_cast(width_ * height_ * 3 / 2)) { std::cerr << "ERROR: Media Foundation webcam buffer is too small" << std::endl; return false; } if (frame.width == width_ && frame.height == height_) { - for (DWORD i = 0; i < requiredBytes; i += 4) { - destination[i] = frame.data[i]; - destination[i + 1] = frame.data[i + 1]; - destination[i + 2] = frame.data[i + 2]; - destination[i + 3] = 255; - } + // Straight from the camera's frame, never through a per-byte loop: one + // that forced alpha to 255 measured out at ~12 fps of real camera + // motion at 3840x2160, inside a file whose container claimed 30. The + // conversion ignores alpha anyway. + convertBgraToNv12Bt709(frame.data, static_cast(rowBytes), width_, height_, destination); return true; } + bgraScratch_.resize(static_cast(rowBytes) * height_); for (int y = 0; y < height_; y += 1) { const int sourceY = static_cast((static_cast(y) * frame.height) / height_); - BYTE* destinationRow = destination + rowBytes * y; + BYTE* destinationRow = bgraScratch_.data() + rowBytes * y; for (int x = 0; x < width_; x += 1) { const int sourceX = static_cast((static_cast(x) * frame.width) / width_); const BYTE* source = frame.data + (sourceY * frame.width + sourceX) * 4; @@ -1132,7 +1249,7 @@ bool MFEncoder::copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destinat target[3] = 255; } } - + convertBgraToNv12Bt709(bgraScratch_.data(), static_cast(rowBytes), width_, height_, destination); return true; } @@ -1618,7 +1735,7 @@ bool MFEncoder::captureVideoSample( // which holds the shared frame-state mutex across this call but not // across submitVideoSample). Microsoft::WRL::ComPtr buffer; - const DWORD frameBytes = static_cast(width_ * height_ * 4); + const DWORD frameBytes = static_cast(width_ * height_ * 3 / 2); if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer")) { return false; } @@ -1645,6 +1762,76 @@ bool MFEncoder::captureVideoSample( sample->SetSampleTime(sampleTime); sample->SetSampleDuration(sampleDuration); + lastVideoBuffer_ = buffer; + outSample = sample; + return true; +} + +bool MFEncoder::repeatLastVideoSample(int64_t timestampHns, Microsoft::WRL::ComPtr& outSample) { + outSample.Reset(); + if (!lastVideoBuffer_) { + return false; + } + const int64_t sampleDuration = 10'000'000LL / fps_; + Microsoft::WRL::ComPtr sample; + if (!succeeded(MFCreateSample(&sample), "MFCreateSample(repeat)")) { + return false; + } + // Shared, not copied: a buffer is never written again once its sample is + // built, so two samples can hand the encoder the same bytes. + sample->AddBuffer(lastVideoBuffer_.Get()); + sample->SetSampleTime(nextSampleTime(timestampHns, sampleDuration)); + sample->SetSampleDuration(sampleDuration); + outSample = sample; + return true; +} + +bool MFEncoder::captureNv12Sample( + const Nv12FrameView& frame, + int64_t timestampHns, + Microsoft::WRL::ComPtr& outSample) { + outSample.Reset(); + + if (!frame.data || frame.width != width_ || frame.height != height_) { + // No rescaler here on purpose: the webcam encoder is created with the + // capture's own dimensions, so a mismatch means a bug upstream rather + // than a frame worth stretching. + std::cerr << "ERROR: NV12 webcam frame does not match the encoder's size" << std::endl; + return false; + } + + const int64_t sampleDuration = 10'000'000LL / fps_; + const int64_t sampleTime = nextSampleTime(timestampHns, sampleDuration); + const DWORD frameBytes = static_cast(width_ * height_ * 3 / 2); + + Microsoft::WRL::ComPtr buffer; + if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer(webcam NV12)")) { + return false; + } + + BYTE* data = nullptr; + DWORD maxLength = 0; + DWORD currentLength = 0; + if (!succeeded(buffer->Lock(&data, &maxLength, ¤tLength), "IMFMediaBuffer::Lock(webcam NV12)")) { + return false; + } + if (maxLength < frameBytes) { + buffer->Unlock(); + std::cerr << "ERROR: Media Foundation webcam NV12 buffer is too small" << std::endl; + return false; + } + std::memcpy(data, frame.data, frameBytes); + buffer->Unlock(); + buffer->SetCurrentLength(frameBytes); + + Microsoft::WRL::ComPtr sample; + if (!succeeded(MFCreateSample(&sample), "MFCreateSample(webcam NV12)")) { + return false; + } + sample->AddBuffer(buffer.Get()); + sample->SetSampleTime(sampleTime); + sample->SetSampleDuration(sampleDuration); + outSample = sample; return true; } @@ -1659,7 +1846,7 @@ bool MFEncoder::captureBgraSample( const int64_t sampleTime = nextSampleTime(timestampHns, sampleDuration); Microsoft::WRL::ComPtr buffer; - const DWORD frameBytes = static_cast(width_ * height_ * 4); + const DWORD frameBytes = static_cast(width_ * height_ * 3 / 2); if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer(webcam)")) { return false; } @@ -1798,6 +1985,9 @@ bool MFEncoder::finalize() { captureDevice_.Reset(); context_.Reset(); device_.Reset(); - MFShutdown(); + if (mfStarted_) { + MFShutdown(); + mfStarted_ = false; + } return ok; } diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h index 8d1d6ae6e..4c2034f87 100644 --- a/electron/native/wgc-capture/src/mf_encoder.h +++ b/electron/native/wgc-capture/src/mf_encoder.h @@ -11,6 +11,7 @@ #include #include #include +#include struct BgraFrameView { const BYTE* data = nullptr; @@ -18,6 +19,25 @@ struct BgraFrameView { int height = 0; }; +/** + * A planar NV12 frame: a width*height Y plane followed by an interleaved + * width/2 * height/2 UV plane, so width*height*3/2 bytes in total. + * + * Used for the webcam, whose camera can hand Media Foundation NV12 directly. + * Asking that same camera for RGB32 made the source reader decode and convert + * every frame, which measured at 92ms per 4K frame -- a hard ceiling near 11 + * fps -- against 32ms for NV12. + */ +struct Nv12FrameView { + const BYTE* data = nullptr; + int width = 0; + int height = 0; +}; + +// BGRA to NV12, BT.709 studio range, for every frame the encoder gets from +// system memory. `width` must be even. Exposed for mf_encoder_color_test. +void convertBgraToNv12Bt709(const BYTE* bgra, int stride, int width, int height, BYTE* nv12); + struct AudioInputFormat { GUID subtype = MFAudioFormat_PCM; UINT32 sampleRate = 0; @@ -118,10 +138,19 @@ class MFEncoder { int64_t timestampHns, const BgraFrameView* webcamFrame, Microsoft::WRL::ComPtr& outSample); + // The last frame captureVideoSample read back, again, at a new time: for a + // tick on which WGC delivered nothing new. No GPU copy, no Map, no + // conversion -- the readback a static screen used to pay on every tick + // (getopenscreen/openscreen#925). False before any frame was captured. + bool repeatLastVideoSample(int64_t timestampHns, Microsoft::WRL::ComPtr& outSample); bool captureDxgiSample( ID3D11Texture2D* texture, int64_t timestampHns, Microsoft::WRL::ComPtr& outSample); + bool captureNv12Sample( + const Nv12FrameView& frame, + int64_t timestampHns, + Microsoft::WRL::ComPtr& outSample); bool captureBgraSample( const BgraFrameView& frame, int64_t timestampHns, @@ -230,12 +259,22 @@ class MFEncoder { DWORD videoStreamIndex_ = 0; DWORD audioStreamIndex_ = 0; bool hasAudioStream_ = false; + // The BGRA frame when it has to be drawn on or rescaled before the NV12 + // conversion; reused so a frame does not allocate one. + std::vector bgraScratch_; + Microsoft::WRL::ComPtr lastVideoBuffer_; int width_ = 0; int height_ = 0; int fps_ = 60; int64_t firstTimestampHns_ = -1; int64_t lastTimestampHns_ = -1; bool finalized_ = false; + // Whether initialize() got as far as a successful MFStartup(). finalize() + // may only balance a startup this encoder made: MF's startup count is + // process-wide, and an encoder that was never initialized (a dropped + // camera's) would otherwise shut Media Foundation down under every other + // encoder and camera still running. + bool mfStarted_ = false; bool useDxgiInput_ = false; const char* videoEncoderSelection_ = kVideoEncoderSelectionDefault; const char* videoEncoderRuntime_ = kVideoEncoderRuntimeUnknown; diff --git a/electron/native/wgc-capture/src/mf_encoder_color_test.cpp b/electron/native/wgc-capture/src/mf_encoder_color_test.cpp new file mode 100644 index 000000000..ac33a5e1b --- /dev/null +++ b/electron/native/wgc-capture/src/mf_encoder_color_test.cpp @@ -0,0 +1,475 @@ +// What the H.264 track actually says about itself (getopenscreen/openscreen#922, +// #923). Drives the real MFEncoder with synthetic BGRA frames -- three solid +// bands, red, green and blue -- into a temporary MP4, then reads the file back +// with ffprobe and ffmpeg: the profile, the colour tags, and the YUV values the +// encoder really produced for each band. The compositor decodes every +// recording as BT.709 limited range, so that is what the file has to be and +// has to say. +// +// Needs ffprobe and ffmpeg on PATH; without them the checks are skipped, not +// failed. The hardware encoder is covered when the host has one. + +#include "mf_encoder.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int g_ran = 0; +int g_failed = 0; + +void expect(const std::string& name, bool ok, const std::string& detail) { + g_ran += 1; + std::cout << (ok ? "PASS " : "FAIL ") << name << (ok ? "" : " " + detail) << "\n"; + if (!ok) { + g_failed += 1; + } +} + +void skip(const std::string& name, const std::string& reason) { + std::cout << "SKIP " << name << " " << reason << "\n"; +} + +std::string run(const std::string& command) { + std::string output; + FILE* pipe = _popen(command.c_str(), "rb"); + if (!pipe) { + return output; + } + char buffer[65536]; + size_t read = 0; + while ((read = fread(buffer, 1, sizeof(buffer), pipe)) > 0) { + output.append(buffer, read); + } + _pclose(pipe); + return output; +} + +// GetTempPathA answers in the ANSI code page, which _popen also speaks; the +// encoder takes a wide path, so widen it through that code page, not per byte. +std::wstring widen(const std::string& ansi) { + const int size = MultiByteToWideChar(CP_ACP, 0, ansi.data(), static_cast(ansi.size()), nullptr, 0); + std::wstring result(size, L'\0'); + MultiByteToWideChar(CP_ACP, 0, ansi.data(), static_cast(ansi.size()), result.data(), size); + return result; +} + +bool toolsAvailable() { + return run("ffprobe -version 2>NUL").find("ffprobe") != std::string::npos && + run("ffmpeg -version 2>NUL").find("ffmpeg") != std::string::npos; +} + +std::string field(const std::string& probe, const std::string& key) { + const auto at = probe.find(key + "="); + if (at == std::string::npos) { + return ""; + } + const auto start = at + key.size() + 1; + const auto end = probe.find_first_of("\r\n", start); + return probe.substr(start, end - start); +} + +constexpr int kWidth = 480; +constexpr int kHeight = 272; +constexpr int kFrames = 30; + +struct Expected { + const char* name; + int y, cb, cr; +}; +// BT.709, studio range. BT.601 would put red at Y 81, Cb 90. +constexpr Expected kBands[] = {{"red", 63, 102, 240}, {"green", 173, 42, 26}, {"blue", 32, 240, 118}}; + +void checkEncoder(ID3D11Device* device, ID3D11DeviceContext* context, bool software) { + const std::string label = software ? "software" : "default"; + char tempDir[MAX_PATH]{}; + GetTempPathA(MAX_PATH, tempDir); + const std::string path = std::string(tempDir) + "openscreen-mf-encoder-color-" + label + ".mp4"; + const std::wstring widePath = widen(path); + DeleteFileA(path.c_str()); + + std::vector bgra(static_cast(kWidth) * kHeight * 4); + for (int y = 0; y < kHeight; y += 1) { + for (int x = 0; x < kWidth; x += 1) { + BYTE* pixel = &bgra[(static_cast(y) * kWidth + x) * 4]; + const int band = x * 3 / kWidth; + pixel[0] = band == 2 ? 255 : 0; // B + pixel[1] = band == 1 ? 255 : 0; // G + pixel[2] = band == 0 ? 255 : 0; // R + pixel[3] = 255; + } + } + + { + MFEncoder encoder; + MFEncoderOptions options; + options.preferSoftwareEncoder = software; + if (!encoder.initialize(widePath, kWidth, kHeight, 30, 2'000'000, device, context, nullptr, options)) { + expect("encoder-initialize-" + label, false, "initialize failed"); + return; + } + const std::string runtime = encoder.videoEncoderRuntime(); + std::cout << "COLOR_RAW " << label << " runtime=" << runtime << "\n"; + if (!software && runtime != kVideoEncoderRuntimeHardware) { + skip("encoder-" + label, "no hardware H.264 encoder on this host (" + runtime + ")"); + encoder.finalize(); + return; + } + const BgraFrameView frame{bgra.data(), kWidth, kHeight}; + bool wrote = true; + for (int i = 0; i < kFrames && wrote; i += 1) { + Microsoft::WRL::ComPtr sample; + wrote = encoder.captureBgraSample(frame, static_cast(i) * 333'333, sample) && + encoder.submitVideoSample(sample.Get()); + } + expect("encoder-writes-" + label, wrote && encoder.finalize(), "write or finalize failed"); + } + + const std::string probe = run( + "ffprobe -v error -select_streams v:0 -show_entries " + "stream=profile,has_b_frames,color_range,color_space,color_primaries,color_transfer -of default=nw=1 \"" + + path + "\""); + std::cout << "COLOR_RAW " << label << " profile=" << field(probe, "profile") + << " range=" << field(probe, "color_range") << " space=" << field(probe, "color_space") + << " primaries=" << field(probe, "color_primaries") + << " transfer=" << field(probe, "color_transfer") << "\n"; + expect("profile-high-" + label, field(probe, "profile") == "High", probe); + // B-frames broke the fragmented writer on macOS; the profile must not bring them. + expect("no-b-frames-" + label, field(probe, "has_b_frames") == "0", probe); + expect( + "colour-tags-bt709-limited-" + label, + field(probe, "color_range") == "tv" && field(probe, "color_space") == "bt709" && + field(probe, "color_primaries") == "bt709" && field(probe, "color_transfer") == "bt709", + probe); + + // The last frame, as the encoder wrote its samples: yuv420p out of a yuv420p + // stream is a straight copy, no range or matrix conversion on the way. + const std::string yuv = run( + "ffmpeg -v error -sseof -0.2 -i \"" + path + "\" -frames:v 1 -f rawvideo -pix_fmt yuv420p -"); + const size_t lumaSize = static_cast(kWidth) * kHeight; + if (yuv.size() < lumaSize * 3 / 2) { + expect("yuv-readback-" + label, false, "ffmpeg returned " + std::to_string(yuv.size()) + " bytes"); + return; + } + for (int band = 0; band < 3; band += 1) { + const int x = kWidth * (2 * band + 1) / 6; + const int y = kHeight / 2; + const int luma = static_cast(yuv[static_cast(y) * kWidth + x]); + const size_t chroma = static_cast(y / 2) * (kWidth / 2) + x / 2; + const int cb = static_cast(yuv[lumaSize + chroma]); + const int cr = static_cast(yuv[lumaSize + lumaSize / 4 + chroma]); + const Expected& want = kBands[band]; + char detail[128]{}; + sprintf_s( + detail, "%s Y=%d Cb=%d Cr=%d, BT.709 limited wants %d/%d/%d", want.name, luma, cb, cr, want.y, + want.cb, want.cr); + std::cout << "COLOR_RAW " << label << " " << detail << "\n"; + expect( + std::string("yuv-bt709-") + want.name + "-" + label, + std::abs(luma - want.y) <= 3 && std::abs(cb - want.cb) <= 3 && std::abs(cr - want.cr) <= 3, + detail); + } + DeleteFileA(path.c_str()); +} + +// The converter against BT.709 in double precision, on noise: solid bands +// cannot tell a 2x2 average from a wrong pairing of pixels, and 38 wide with a +// padded stride runs the SIMD body, its scalar tail and the row pitch. +void checkConverterAgainstReference() { + constexpr int width = 38; + constexpr int height = 22; + constexpr int stride = width * 4 + 24; + std::vector bgra(static_cast(stride) * height); + uint32_t seed = 12345; + for (BYTE& value : bgra) { + seed = seed * 1664525u + 1013904223u; + value = static_cast(seed >> 24); + } + std::vector nv12(static_cast(width) * height * 3 / 2); + convertBgraToNv12Bt709(bgra.data(), stride, width, height, nv12.data()); + + const double kr = 0.2126; + const double kb = 0.0722; + const auto channel = [&](int x, int y, int c) { return bgra[static_cast(y) * stride + x * 4 + c] / 255.0; }; + int worst = 0; + for (int y = 0; y < height; y += 1) { + for (int x = 0; x < width; x += 1) { + const double luma = kr * channel(x, y, 2) + (1 - kr - kb) * channel(x, y, 1) + kb * channel(x, y, 0); + const int want = static_cast(std::lround(16 + 219 * luma)); + worst = std::max(worst, std::abs(want - nv12[static_cast(y) * width + x])); + } + } + for (int y = 0; y < height; y += 2) { + for (int x = 0; x < width; x += 2) { + double r = 0, g = 0, b = 0; + for (int dy = 0; dy < 2; dy += 1) { + for (int dx = 0; dx < 2; dx += 1) { + r += channel(x + dx, y + dy, 2) / 4; + g += channel(x + dx, y + dy, 1) / 4; + b += channel(x + dx, y + dy, 0) / 4; + } + } + const double luma = kr * r + (1 - kr - kb) * g + kb * b; + const int cb = static_cast(std::lround(128 + 224 * (b - luma) / (2 * (1 - kb)))); + const int cr = static_cast(std::lround(128 + 224 * (r - luma) / (2 * (1 - kr)))); + const size_t at = static_cast(width) * height + static_cast(y / 2) * width + x; + worst = std::max({worst, std::abs(cb - nv12[at]), std::abs(cr - nv12[at + 1])}); + } + } + std::cout << "COLOR_RAW converter worst deviation from BT.709 = " << worst << " code values" << std::endl; + expect("converter-matches-bt709-reference", worst <= 1, "worst=" + std::to_string(worst)); +} + +// Not a pass/fail: what one 1080p frame costs to hand over and encode, for a +// before/after comparison of the conversion's cost. +void timeFullHd(ID3D11Device* device, ID3D11DeviceContext* context) { + constexpr int width = 1920; + constexpr int height = 1080; + constexpr int frames = 120; + char tempDir[MAX_PATH]{}; + GetTempPathA(MAX_PATH, tempDir); + const std::string path = std::string(tempDir) + "openscreen-mf-encoder-timing.mp4"; + const std::wstring widePath = widen(path); + std::vector bgra(static_cast(width) * height * 4); + for (size_t i = 0; i < bgra.size(); i += 1) { + bgra[i] = static_cast((i * 2654435761u) >> 24); + } + MFEncoder encoder; + if (!encoder.initialize(widePath, width, height, 60, 18'000'000, device, context, nullptr, {})) { + return; + } + const BgraFrameView frame{bgra.data(), width, height}; + double captureMs = 0.0; + double submitMs = 0.0; + for (int i = 0; i < frames; i += 1) { + Microsoft::WRL::ComPtr sample; + const auto start = std::chrono::steady_clock::now(); + encoder.captureBgraSample(frame, static_cast(i) * 166'667, sample); + const auto captured = std::chrono::steady_clock::now(); + encoder.submitVideoSample(sample.Get()); + const auto submitted = std::chrono::steady_clock::now(); + captureMs += std::chrono::duration(captured - start).count(); + submitMs += std::chrono::duration(submitted - captured).count(); + } + encoder.finalize(); + DeleteFileA(path.c_str()); + std::cout << "COLOR_RAW timing 1080p " << encoder.videoEncoderRuntime() << ": capture " + << captureMs / frames << " ms, submit " << submitMs / frames << " ms per frame" << std::endl; +} + +// A tick with no new WGC frame repeats the last buffer instead of reading the +// texture back again (getopenscreen/openscreen#925). The file must still hold +// one frame per tick, each the captured picture; and the repeat must cost next +// to nothing next to a readback, or it is not worth having. +void checkRepeatedFrames(ID3D11Device* device, ID3D11DeviceContext* context) { + const auto makeTexture = [&](int width, int height, Microsoft::WRL::ComPtr& texture) { + std::vector bgra(static_cast(width) * height * 4); + for (int y = 0; y < height; y += 1) { + for (int x = 0; x < width; x += 1) { + BYTE* pixel = &bgra[(static_cast(y) * width + x) * 4]; + pixel[2] = 255; // solid red + pixel[3] = 255; + } + } + D3D11_TEXTURE2D_DESC desc{}; + desc.Width = width; + desc.Height = height; + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + D3D11_SUBRESOURCE_DATA initial{bgra.data(), static_cast(width * 4), 0}; + return SUCCEEDED(device->CreateTexture2D(&desc, &initial, &texture)); + }; + char tempDir[MAX_PATH]{}; + GetTempPathA(MAX_PATH, tempDir); + + { + const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat.mp4"; + const std::wstring widePath = widen(path); + Microsoft::WRL::ComPtr texture; + MFEncoder encoder; + if (!makeTexture(kWidth, kHeight, texture) || + !encoder.initialize(widePath, kWidth, kHeight, 30, 2'000'000, device, context, nullptr, {})) { + expect("repeat-setup", false, "texture or encoder"); + return; + } + Microsoft::WRL::ComPtr sample; + bool wrote = !encoder.repeatLastVideoSample(0, sample) && // nothing to repeat yet + encoder.captureVideoSample(texture.Get(), 0, nullptr, sample) && encoder.submitVideoSample(sample.Get()); + for (int i = 1; i < kFrames && wrote; i += 1) { + wrote = encoder.repeatLastVideoSample(static_cast(i) * 333'333, sample) && + encoder.submitVideoSample(sample.Get()); + } + expect("repeat-writes", wrote && encoder.finalize(), "write or finalize failed"); + const std::string frames = run( + "ffprobe -v error -count_frames -select_streams v:0 -show_entries stream=nb_read_frames " + "-of default=nw=1:nk=1 \"" + path + "\""); + const std::string yuv = run( + "ffmpeg -v error -sseof -0.2 -i \"" + path + "\" -frames:v 1 -f rawvideo -pix_fmt yuv420p -"); + const int luma = yuv.empty() ? -1 : static_cast(yuv[static_cast(kHeight / 2) * kWidth + kWidth / 2]); + std::cout << "REPEAT_RAW frames=" << std::atoi(frames.c_str()) << " last-frame Y=" << luma << std::endl; + expect("repeat-keeps-one-frame-per-tick", std::atoi(frames.c_str()) == kFrames, frames); + expect("repeat-keeps-the-picture", std::abs(luma - 63) <= 3, "Y=" + std::to_string(luma)); + DeleteFileA(path.c_str()); + } + + { + const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat-timing.mp4"; + const std::wstring widePath = widen(path); + Microsoft::WRL::ComPtr texture; + MFEncoder encoder; + if (!makeTexture(1920, 1080, texture) || + !encoder.initialize(widePath, 1920, 1080, 60, 18'000'000, device, context, nullptr, {})) { + return; + } + double readbackMs = 0.0; + double repeatMs = 0.0; + constexpr int rounds = 60; + for (int i = 0; i < rounds; i += 1) { + Microsoft::WRL::ComPtr sample; + const auto start = std::chrono::steady_clock::now(); + encoder.captureVideoSample(texture.Get(), static_cast(2 * i) * 166'667, nullptr, sample); + const auto read = std::chrono::steady_clock::now(); + encoder.submitVideoSample(sample.Get()); + const auto repeatStart = std::chrono::steady_clock::now(); + encoder.repeatLastVideoSample(static_cast(2 * i + 1) * 166'667, sample); + const auto repeated = std::chrono::steady_clock::now(); + encoder.submitVideoSample(sample.Get()); + readbackMs += std::chrono::duration(read - start).count(); + repeatMs += std::chrono::duration(repeated - repeatStart).count(); + } + encoder.finalize(); + DeleteFileA(path.c_str()); + std::cout << "REPEAT_RAW 1080p readback " << readbackMs / rounds << " ms, repeat " << repeatMs / rounds + << " ms per frame" << std::endl; + } +} + +// An encoder that was never initialized -- a camera the helper dropped before +// its encoder was set up -- must not touch Media Foundation when it is +// finalized or destroyed. Its finalize() used to call an unmatched MFShutdown(), +// which in the helper left every other encoder writing nothing (Finalize: +// MF_E_SINK_NO_SAMPLES_PROCESSED). With an initialized encoder alive, as here, +// that takes MF's count to zero under a live sink writer, and the live encoder +// then blocks instead of failing -- so the whole sequence runs on its own thread +// with a deadline, and a regression fails this test instead of hanging the +// build. Needs no ffmpeg. +void checkUninitializedEncoderLeavesOthersRunning() { + Microsoft::WRL::ComPtr device; + Microsoft::WRL::ComPtr context; + if (FAILED(D3D11CreateDevice( + nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, D3D11_CREATE_DEVICE_BGRA_SUPPORT, nullptr, 0, + D3D11_SDK_VERSION, &device, nullptr, &context)) && + FAILED(D3D11CreateDevice( + nullptr, D3D_DRIVER_TYPE_WARP, nullptr, D3D11_CREATE_DEVICE_BGRA_SUPPORT, nullptr, 0, + D3D11_SDK_VERSION, &device, nullptr, &context))) { + skip("uninitialized-encoder-leaves-mf-running", "no D3D11 device"); + return; + } + char tempDir[MAX_PATH]{}; + GetTempPathA(MAX_PATH, tempDir); + const std::string path = std::string(tempDir) + "openscreen-mf-encoder-uninitialized-peer.mp4"; + DeleteFileA(path.c_str()); + + // 0 running, 1 passed, 2 failed, 3 skipped. + std::atomic outcome = 0; + std::thread worker([&] { + CoInitializeEx(nullptr, COINIT_MULTITHREADED); + { + MFEncoder live; + if (!live.initialize( + widen(path), kWidth, kHeight, 30, 2'000'000, device.Get(), context.Get(), nullptr, {})) { + outcome = 3; + } else { + { + MFEncoder neverInitialized; + neverInitialized.finalize(); + } // and destroyed, which finalizes again + Microsoft::WRL::ComPtr probe; + std::vector bgra(static_cast(kWidth) * kHeight * 4, 0x80); + const BgraFrameView frame{bgra.data(), kWidth, kHeight}; + bool wrote = SUCCEEDED(MFCreateSample(&probe)); + for (int i = 0; i < kFrames && wrote; i += 1) { + Microsoft::WRL::ComPtr sample; + wrote = live.captureBgraSample(frame, static_cast(i) * 333'333, sample) && + live.submitVideoSample(sample.Get()); + } + outcome = wrote && live.finalize() ? 1 : 2; + } + } + CoUninitialize(); + }); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (outcome == 0 && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + if (outcome == 0) { + expect("uninitialized-encoder-leaves-mf-running", false, "the live encoder blocked in Media Foundation"); + std::cout << "ran " << g_ran << " tests\n" << g_failed << " failed" << std::endl; + // The blocked thread can be neither joined nor safely abandoned. + TerminateProcess(GetCurrentProcess(), 1); + } + worker.join(); + DeleteFileA(path.c_str()); + if (outcome == 3) { + skip("uninitialized-encoder-leaves-mf-running", "encoder initialize failed on this host"); + return; + } + expect("uninitialized-encoder-leaves-mf-running", outcome == 1, "write or finalize failed"); +} + +} // namespace + +int main() { + checkConverterAgainstReference(); + if (SUCCEEDED(CoInitializeEx(nullptr, COINIT_MULTITHREADED))) { + checkUninitializedEncoderLeavesOthersRunning(); + CoUninitialize(); + } + if (!toolsAvailable()) { + skip("mf-encoder-color", "ffprobe/ffmpeg not on PATH"); + return g_failed == 0 ? 0 : 1; + } + if (FAILED(CoInitializeEx(nullptr, COINIT_MULTITHREADED))) { + skip("mf-encoder-color", "CoInitializeEx failed"); + return 0; + } + Microsoft::WRL::ComPtr device; + Microsoft::WRL::ComPtr context; + const UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT; + if (FAILED(D3D11CreateDevice( + nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, flags, nullptr, 0, D3D11_SDK_VERSION, &device, + nullptr, &context)) && + FAILED(D3D11CreateDevice( + nullptr, D3D_DRIVER_TYPE_WARP, nullptr, D3D11_CREATE_DEVICE_BGRA_SUPPORT, nullptr, 0, + D3D11_SDK_VERSION, &device, nullptr, &context))) { + skip("mf-encoder-color", "no D3D11 device"); + return 0; + } + checkEncoder(device.Get(), context.Get(), true); + checkEncoder(device.Get(), context.Get(), false); + timeFullHd(device.Get(), context.Get()); + checkRepeatedFrames(device.Get(), context.Get()); + + std::cout << "ran " << g_ran << " tests\n"; + if (g_failed != 0) { + std::cout << g_failed << " failed\n"; + return 1; + } + return 0; +} diff --git a/electron/native/wgc-capture/src/realtime_scheduling.h b/electron/native/wgc-capture/src/realtime_scheduling.h new file mode 100644 index 000000000..dc8ca5abc --- /dev/null +++ b/electron/native/wgc-capture/src/realtime_scheduling.h @@ -0,0 +1,77 @@ +#pragma once + +#include +#include +#include + +#include + +// Every wait in the capture helpers -- the video writer's deadline, the WASAPI +// poll, the audio mixer's chunk clock, the cursor sampler's interval -- lands on +// the system timer tick, and a process that asks for nothing gets the default +// 15.625 ms one. Measured on a real take (getopenscreen/openscreen#920, #921): +// webcam frames spaced in whole ticks, 25.9 unique frames/s for a 30 fps camera; +// the cursor sampled at 22.7 Hz for 30 asked; WASAPI packets handed over late +// enough to leave holes in the voice (#911). +// +// Held for the life of the process. Windows 11 ignores the timer request of a +// process it throttles (a minimized or occluded app, which a recording helper +// always is), so the process is also opted out of power throttling for both +// execution speed and timer resolution. Either call failing is logged and +// survived: the recording still works at the coarser tick, as it always has. +class HighResolutionTiming { +public: + HighResolutionTiming() { + PROCESS_POWER_THROTTLING_STATE throttling{}; + throttling.Version = PROCESS_POWER_THROTTLING_CURRENT_VERSION; + throttling.ControlMask = + PROCESS_POWER_THROTTLING_EXECUTION_SPEED | PROCESS_POWER_THROTTLING_IGNORE_TIMER_RESOLUTION; + throttling.StateMask = 0; + if (!SetProcessInformation( + GetCurrentProcess(), ProcessPowerThrottling, &throttling, sizeof(throttling))) { + std::cerr << "WARNING: Could not opt out of power throttling (error " << GetLastError() + << ")" << std::endl; + } + periodSet_ = timeBeginPeriod(1) == TIMERR_NOERROR; + if (!periodSet_) { + std::cerr << "WARNING: Could not raise the timer resolution to 1 ms" << std::endl; + } + } + ~HighResolutionTiming() { + if (periodSet_) { + timeEndPeriod(1); + } + } + HighResolutionTiming(const HighResolutionTiming&) = delete; + HighResolutionTiming& operator=(const HighResolutionTiming&) = delete; + +private: + bool periodSet_ = false; +}; + +// Registers the calling thread with MMCSS for as long as it lives, so the +// scheduler favours it over ordinary work while the machine is busy -- the +// stalls that outlast the timer tick. `task` is a task name from the registry's +// MMCSS profile: "Pro Audio" for the audio threads, "Capture" for video. A thread +// that cannot register runs at its normal priority, as before. +class MmcssThread { +public: + explicit MmcssThread(const wchar_t* task) { + DWORD taskIndex = 0; + handle_ = AvSetMmThreadCharacteristicsW(task, &taskIndex); + if (!handle_) { + std::wcerr << L"WARNING: Could not register thread with MMCSS task \"" << task + << L"\" (error " << GetLastError() << L")" << std::endl; + } + } + ~MmcssThread() { + if (handle_) { + AvRevertMmThreadCharacteristics(handle_); + } + } + MmcssThread(const MmcssThread&) = delete; + MmcssThread& operator=(const MmcssThread&) = delete; + +private: + HANDLE handle_ = nullptr; +}; diff --git a/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp index c19c44160..875dda566 100644 --- a/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp +++ b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp @@ -1,4 +1,5 @@ #include "wasapi_loopback_capture.h" +#include "realtime_scheduling.h" #include #include @@ -352,6 +353,7 @@ bool WasapiLoopbackCapture::start(AudioCallback callback) { } thread_ = std::thread([this] { + const MmcssThread mmcss(L"Pro Audio"); captureLoop(); }); return true; diff --git a/electron/native/wgc-capture/src/webcam_capture.cpp b/electron/native/wgc-capture/src/webcam_capture.cpp index 6377a799b..45d2ebdd7 100644 --- a/electron/native/wgc-capture/src/webcam_capture.cpp +++ b/electron/native/wgc-capture/src/webcam_capture.cpp @@ -1,13 +1,17 @@ #include "webcam_capture.h" +#include "realtime_scheduling.h" +#include "webcam_format.h" +#include "webcam_loss.h" + #include #include #include #include #include -#include #include +#include namespace { @@ -33,114 +37,6 @@ std::wstring readAllocatedString(IMFActivate* activate, REFGUID key) { return result; } -/** - * Does one of these appear inside the other as WHOLE WORDS? - * - * Plain containment answered for devices that merely share a spelling: a - * requested "Logi" is inside "Logitech", and "Micro" inside "Microphone", - * neither of them as a word. Matching on that resolved a camera nobody asked - * for -- and resolving one is exactly what stops the request reaching the - * DirectShow fallback, where the cameras Media Foundation cannot enumerate live. - * - * Both sides arrive normalized, so a boundary is the start of the string, its - * end, or a space. - */ -bool containsAsWords(const std::wstring& haystack, const std::wstring& needle) { - if (haystack.empty() || needle.empty()) { - return false; - } - size_t pos = haystack.find(needle); - while (pos != std::wstring::npos) { - const bool startsOnBoundary = pos == 0 || haystack[pos - 1] == L' '; - const size_t after = pos + needle.size(); - const bool endsOnBoundary = after == haystack.size() || haystack[after] == L' '; - if (startsOnBoundary && endsOnBoundary) { - return true; - } - pos = haystack.find(needle, pos + 1); - } - return false; -} - -bool containsInsensitive(const std::wstring& haystack, const std::wstring& needle) { - return containsAsWords(haystack, needle) || containsAsWords(needle, haystack); -} - -std::wstring normalizeDeviceName(const std::wstring& value) { - std::wstring normalized; - normalized.reserve(value.size()); - bool lastWasSpace = true; - for (const wchar_t ch : value) { - if (std::iswalnum(ch)) { - normalized.push_back(static_cast(std::towlower(ch))); - lastWasSpace = false; - continue; - } - if (!lastWasSpace) { - normalized.push_back(L' '); - lastWasSpace = true; - } - } - while (!normalized.empty() && normalized.back() == L' ') { - normalized.pop_back(); - } - return normalized; -} - -/** - * How well a candidate answers a requested name, or 0 for "not this one". - * - * Only decisive matches count: the names being equal once normalized, or one - * containing the other -- which is the ordinary case, since Chromium appends USB - * ids to what the driver reports. - * - * A further tier used to score shared WORDS, to bridge names differing more than - * that. It bridged names that were not the same device. "Logi Capture" and - * "Logitech StreamCam" share no word, yet "logi" sits inside "logitech" and that - * scored high enough to win -- so asking for a camera Media Foundation cannot - * enumerate opened a DIFFERENT camera, instead of returning nothing and letting - * the DirectShow fallback find the real one (getopenscreen/openscreen#405). - * - * Returning 0 is what makes that fallback reachable, so it is a real answer - * rather than a weak match. Keep this in step with - * `electron/recording/deviceNameMatching.ts`, which states the same rules for - * the Electron side and carries their unit tests. - */ -int deviceMatchScore( - const std::wstring& candidateName, - const std::wstring& candidateLink, - const std::wstring& requestedName, - const std::wstring& requestedId) { - int score = 0; - const auto normalizedName = normalizeDeviceName(candidateName); - const auto normalizedLink = normalizeDeviceName(candidateLink); - const auto normalizedRequestedName = normalizeDeviceName(requestedName); - const auto normalizedRequestedId = normalizeDeviceName(requestedId); - - if (!normalizedRequestedName.empty()) { - if (normalizedName == normalizedRequestedName) { - score = std::max(score, 1000); - } - if (containsInsensitive(normalizedName, normalizedRequestedName)) { - score = std::max(score, 900); - } - if (containsInsensitive(normalizedLink, normalizedRequestedName)) { - score = std::max(score, 800); - } - } - - if (!normalizedRequestedId.empty()) { - if (containsInsensitive(normalizedLink, normalizedRequestedId)) { - score = std::max(score, 700); - } - if (containsInsensitive(normalizedName, normalizedRequestedId)) { - score = std::max(score, 600); - } - } - - return score; -} - } // namespace WebcamCapture::~WebcamCapture() { @@ -153,53 +49,49 @@ bool WebcamCapture::initialize( const std::wstring& directShowClsid, int requestedWidth, int requestedHeight, - int requestedFps) { + int requestedFps, + bool preferNv12, + DeviceClaims& claims) { fps_ = std::clamp(requestedFps > 0 ? requestedFps : 30, 1, 60); usingDirectShow_ = false; - selectedMatchScore_ = 0; - if (!succeeded(MFStartup(MF_VERSION), "MFStartup(webcam)")) { - if (directShowCapture_.initialize(deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_)) { - usingDirectShow_ = true; - return true; + selectedIdentity_.clear(); + const auto initializeDirectShow = [&]() { + if (!directShowCapture_.initialize( + deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_, claims)) { + if (!deviceId.empty() || !deviceName.empty()) { + std::cerr << "ERROR: Requested webcam device was not found by native Windows webcam providers" + << std::endl; + } + return false; } - return false; + usingDirectShow_ = true; + claims.add(directShowCapture_.deviceIdentity()); + return true; + }; + + if (!succeeded(MFStartup(MF_VERSION), "MFStartup(webcam)")) { + return initializeDirectShow(); } mfStarted_ = true; - if (!selectDevice(deviceId, deviceName)) { + if (!selectDevice(deviceId, deviceName, claims)) { if (mfStarted_) { MFShutdown(); mfStarted_ = false; } - if (directShowCapture_.initialize(deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_)) { - usingDirectShow_ = true; - return true; - } - return false; + return initializeDirectShow(); } - if ((!deviceId.empty() || !deviceName.empty()) && selectedMatchScore_ <= 0) { - if (mediaSource_) { - mediaSource_->Shutdown(); - } - sourceReader_.Reset(); - mediaSource_.Reset(); - if (mfStarted_) { - MFShutdown(); - mfStarted_ = false; - } - if (directShowCapture_.initialize(deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_)) { - usingDirectShow_ = true; - return true; - } - std::cerr << "ERROR: Requested webcam device was not found by native Windows webcam providers" - << std::endl; + if (!configureReader(requestedWidth, requestedHeight, fps_, preferNv12)) { return false; } - - return configureReader(requestedWidth, requestedHeight, fps_); + claims.add(selectedIdentity_); + return true; } -bool WebcamCapture::selectDevice(const std::wstring& deviceId, const std::wstring& deviceName) { +bool WebcamCapture::selectDevice( + const std::wstring& deviceId, + const std::wstring& deviceName, + const DeviceClaims& claims) { Microsoft::WRL::ComPtr attributes; if (!succeeded(MFCreateAttributes(&attributes, 1), "MFCreateAttributes(webcam enumeration)")) { return false; @@ -222,37 +114,169 @@ bool WebcamCapture::selectDevice(const std::wstring& deviceId, const std::wstrin return false; } - UINT32 selectedIndex = 0; - int bestScore = 0; + std::vector candidates; + candidates.reserve(deviceCount); + bool anyClaimed = false; for (UINT32 index = 0; index < deviceCount; index += 1) { - const std::wstring name = readAllocatedString(devices[index], MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME); - const std::wstring symbolicLink = readAllocatedString(devices[index], MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK); - const int score = deviceMatchScore(name, symbolicLink, deviceName, deviceId); - std::wcerr << L"INFO: Native webcam candidate [" << index << L"] name=\"" << name << L"\" score=" << score << std::endl; - if (score > bestScore) { - selectedIndex = index; - bestScore = score; - } - } - - if ((!deviceId.empty() || !deviceName.empty()) && bestScore <= 0) { + DeviceCandidate candidate{ + readAllocatedString(devices[index], MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME), + readAllocatedString(devices[index], MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK)}; + const bool claimed = claims.contains(candidate.identity); + anyClaimed = anyClaimed || claimed; + std::wcerr << L"INFO: Native webcam candidate [" << index << L"] name=\"" << candidate.name + << L"\" score=" << deviceMatchScore(candidate.name, candidate.identity, deviceName, deviceId) + << (claimed ? L" (already recording in this take)" : L"") << std::endl; + candidates.push_back(std::move(candidate)); + } + + const int selectedIndex = selectUnclaimedDevice(candidates, deviceName, deviceId, claims); + if (selectedIndex >= 0) { + selectedDeviceName_ = candidates[selectedIndex].name; + selectedIdentity_ = candidates[selectedIndex].identity; + hr = devices[selectedIndex]->ActivateObject(IID_PPV_ARGS(&mediaSource_)); + } else if (anyClaimed) { + std::cerr << "WARNING: Every matching webcam is already recording in this take; trying DirectShow" + << std::endl; + } else { std::cerr << "WARNING: Requested webcam device was not found by Media Foundation; trying DirectShow" << std::endl; } - selectedMatchScore_ = bestScore; - selectedDeviceName_ = readAllocatedString(devices[selectedIndex], MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME); - hr = devices[selectedIndex]->ActivateObject(IID_PPV_ARGS(&mediaSource_)); - for (UINT32 index = 0; index < deviceCount; index += 1) { devices[index]->Release(); } CoTaskMemFree(devices); - return succeeded(hr, "ActivateObject(webcam)"); + return selectedIndex >= 0 && succeeded(hr, "ActivateObject(webcam)"); +} + +namespace { + +/** + * Is this subtype something the source reader must DECODE before it can convert? + * + * Listed positively -- an unknown subtype counts as compressed -- because the + * cost of guessing wrong the other way is a camera that silently records + * nothing, while guessing wrong this way only costs resolution. + */ +bool isCompressedVideoSubtype(const GUID& subtype) { + static const GUID* const kUncompressed[] = { + &MFVideoFormat_NV12, + &MFVideoFormat_YUY2, + &MFVideoFormat_RGB32, + &MFVideoFormat_RGB24, + &MFVideoFormat_ARGB32, + &MFVideoFormat_UYVY, + &MFVideoFormat_YV12, + &MFVideoFormat_I420, + &MFVideoFormat_IYUV, + }; + for (const GUID* candidate : kUncompressed) { + if (subtype == *candidate) { + return false; + } + } + return true; +} + +/** + * Every video format the camera advertises, read off the source reader. + * + * MF_MT_FRAME_RATE is a ratio; a 30000/1001 entry is NTSC 29.97 and rounds to + * 30, which is what we want to compare against the requested rate. A type + * missing either attribute is skipped rather than guessed at -- it cannot be + * requested by size anyway. + */ +std::vector readNativeFormats(IMFSourceReader* reader) { + std::vector formats; + for (DWORD index = 0;; ++index) { + Microsoft::WRL::ComPtr nativeType; + const HRESULT hr = + reader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, index, &nativeType); + if (hr == MF_E_NO_MORE_TYPES || FAILED(hr)) { + break; + } + + UINT32 width = 0; + UINT32 height = 0; + if (FAILED(MFGetAttributeSize(nativeType.Get(), MF_MT_FRAME_SIZE, &width, &height))) { + continue; + } + + UINT32 numerator = 0; + UINT32 denominator = 0; + int fps = 0; + if (SUCCEEDED(MFGetAttributeRatio(nativeType.Get(), MF_MT_FRAME_RATE, &numerator, &denominator)) && + denominator > 0) { + fps = static_cast((numerator + denominator / 2) / denominator); + } + + GUID subtype{}; + const bool compressed = SUCCEEDED(nativeType->GetGUID(MF_MT_SUBTYPE, &subtype)) + ? isCompressedVideoSubtype(subtype) + : true; + + formats.push_back( + WebcamFormat{static_cast(width), static_cast(height), fps, compressed}); + } + return formats; } -bool WebcamCapture::configureReader(int requestedWidth, int requestedHeight, int requestedFps) { + +/** + * Puts the camera itself on `wanted` by selecting the matching native type. + * + * Best-effort: a driver that refuses leaves the reader on whatever it had, and + * the RGB32 request that follows still produces a usable -- if smaller -- frame. + */ +void selectNativeFormat(IMFSourceReader* reader, const WebcamFormat& wanted) { + for (DWORD index = 0;; ++index) { + Microsoft::WRL::ComPtr nativeType; + const HRESULT hr = + reader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, index, &nativeType); + if (hr == MF_E_NO_MORE_TYPES || FAILED(hr)) { + return; + } + + UINT32 width = 0; + UINT32 height = 0; + if (FAILED(MFGetAttributeSize(nativeType.Get(), MF_MT_FRAME_SIZE, &width, &height))) { + continue; + } + if (static_cast(width) != wanted.width || static_cast(height) != wanted.height) { + continue; + } + + GUID subtype{}; + if (FAILED(nativeType->GetGUID(MF_MT_SUBTYPE, &subtype)) || + isCompressedVideoSubtype(subtype) != wanted.compressed) { + continue; + } + + UINT32 numerator = 0; + UINT32 denominator = 0; + if (wanted.fps > 0 && + SUCCEEDED(MFGetAttributeRatio(nativeType.Get(), MF_MT_FRAME_RATE, &numerator, &denominator)) && + denominator > 0) { + const int fps = static_cast((numerator + denominator / 2) / denominator); + if (fps != wanted.fps) { + continue; + } + } + + if (SUCCEEDED(reader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, nativeType.Get()))) { + return; + } + } +} + +} // namespace + +bool WebcamCapture::configureReader( + int requestedWidth, + int requestedHeight, + int requestedFps, + bool preferNv12) { Microsoft::WRL::ComPtr attributes; if (!succeeded(MFCreateAttributes(&attributes, 3), "MFCreateAttributes(webcam reader)")) { return false; @@ -274,15 +298,57 @@ bool WebcamCapture::configureReader(int requestedWidth, int requestedHeight, int return false; } mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); - mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); - if (requestedWidth > 0 && requestedHeight > 0) { - MFSetAttributeSize(mediaType.Get(), MF_MT_FRAME_SIZE, static_cast(requestedWidth), static_cast(requestedHeight)); - } + // NV12 when the frame only has to reach the webcam encoder, which wants + // NV12 anyway. Asking for RGB32 instead makes the source reader decode AND + // convert every frame: measured at 92ms per 3840x2160 frame against 32ms + // for NV12, which is the difference between 11 fps and the camera's full + // 30. BGRA is still used when the frame is composited into the screen + // recording inline, which needs it in that layout. + deliversNv12_ = preferNv12; + mediaType->SetGUID(MF_MT_SUBTYPE, preferNv12 ? MFVideoFormat_NV12 : MFVideoFormat_RGB32); + + // Pick the capture format, then SELECT it on the source before asking for + // RGB32 output. + // + // Both halves matter. Handing Media Foundation an output type with no + // MF_MT_FRAME_SIZE leaves the device on its default format -- 640x480 on a + // BRIO that offers 3840x2160 -- and that default is what the overlay was + // upscaling. But naming the size on the *output* type alone is not enough + // either: with MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING the reader answers + // S_OK by inserting a converter in front of whichever native type is + // already selected, so the camera keeps running at 640x480 and the size we + // asked for is quietly dropped. Selecting the native type first is what + // actually reconfigures the camera. + const WebcamFormat chosen = chooseWebcamFormat( + readNativeFormats(sourceReader_.Get()), + requestedWidth > 0 ? requestedWidth : 1920, + requestedHeight > 0 ? requestedHeight : 1080, + std::max(1, requestedFps)); + std::cerr << "INFO: Native webcam format " << chosen.width << "x" << chosen.height << "@" + << chosen.fps << (chosen.compressed ? " (compressed)" : " (uncompressed)") << std::endl; + selectNativeFormat(sourceReader_.Get(), chosen); + + MFSetAttributeSize( + mediaType.Get(), MF_MT_FRAME_SIZE, static_cast(chosen.width), static_cast(chosen.height)); MFSetAttributeRatio(mediaType.Get(), MF_MT_FRAME_RATE, static_cast(std::max(1, requestedFps)), 1); - if (!succeeded(sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, mediaType.Get()), - "SetCurrentMediaType(webcam RGB32)")) { - return false; + if (FAILED(sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, mediaType.Get()))) { + // Some drivers refuse a size they nonetheless enumerate. A slightly soft + // camera beats no camera, so drop the size and let the driver choose -- + // exactly what this code did before it asked for anything. + std::cerr << "WARNING: Webcam rejected " << chosen.width << "x" << chosen.height + << "; falling back to the device default format" << std::endl; + Microsoft::WRL::ComPtr fallbackType; + if (!succeeded(MFCreateMediaType(&fallbackType), "MFCreateMediaType(webcam fallback)")) { + return false; + } + fallbackType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + fallbackType->SetGUID(MF_MT_SUBTYPE, preferNv12 ? MFVideoFormat_NV12 : MFVideoFormat_RGB32); + if (!succeeded( + sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, fallbackType.Get()), + "SetCurrentMediaType(webcam RGB32)")) { + return false; + } } sourceReader_->SetStreamSelection(MF_SOURCE_READER_ALL_STREAMS, FALSE); sourceReader_->SetStreamSelection(MF_SOURCE_READER_FIRST_VIDEO_STREAM, TRUE); @@ -335,13 +401,19 @@ void WebcamCapture::stop() { } void WebcamCapture::captureLoop() { + const MmcssThread mmcss(L"Capture"); CoInitializeEx(nullptr, COINIT_MULTITHREADED); + const auto loopStartedAt = std::chrono::steady_clock::now(); + + // Start of the current run of failed reads; empty while reads succeed. + std::optional failureRunStartedAt; while (!stopRequested_) { DWORD streamIndex = 0; DWORD flags = 0; LONGLONG timestamp = 0; Microsoft::WRL::ComPtr sample; + const auto readStartedAt = std::chrono::steady_clock::now(); HRESULT hr = sourceReader_->ReadSample( MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, @@ -349,24 +421,61 @@ void WebcamCapture::captureLoop() { &flags, ×tamp, &sample); + readSampleUs_ += static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - readStartedAt) + .count()); (void)streamIndex; (void)timestamp; if (FAILED(hr)) { - std::cerr << "WARNING: Failed to read webcam sample (hr=0x" << std::hex << hr << std::dec << ")" - << std::endl; + // Counted, not printed: a reader that fails every call would other- + // wise write thousands of identical lines over a long take. + readFailures_ += 1; + lastReadFailure_ = hr; + const auto now = std::chrono::steady_clock::now(); + if (!failureRunStartedAt) { + failureRunStartedAt = now; + } + const int64_t failingForMs = + std::chrono::duration_cast(now - *failureRunStartedAt).count(); + if (isWebcamLossResult(hr, false, failingForMs)) { + markLost(hr); + break; + } std::this_thread::sleep_for(std::chrono::milliseconds(20)); continue; } + failureRunStartedAt.reset(); if ((flags & MF_SOURCE_READERF_ENDOFSTREAM) != 0) { + sawEndOfStream_ = true; + // Stop has its own way out of this loop; end of stream before it + // is the camera leaving. + if (!stopRequested_ && isWebcamLossResult(hr, true, 0)) { + markLost(hr); + } break; } if (!sample) { + // Normal in small numbers (MF_SOURCE_READERF_STREAMTICK); a run of + // nothing else is a camera that never started producing. + emptySamples_ += 1; continue; } + // `Lock()` below is enough; row padding cannot reach `latestFrame_`. + // + // A 2-D media buffer may well pad each row out to a stride wider than + // the frame, but that padding is never what `Lock()` hands back: + // "The Lock method returns a buffer that is guaranteed to be + // contiguous. If the underlying buffer is not contiguous, the method + // copies the data into a new buffer" (IMF2DBuffer, Remarks). Reading + // the native pitch through IMF2DBuffer::Lock2D would only be worth it + // as a measured optimisation -- it saves that copy -- and mixing the + // two interfaces on one buffer is advised against in the same page. Microsoft::WRL::ComPtr buffer; if (FAILED(sample->ConvertToContiguousBuffer(&buffer)) || !buffer) { + bufferFailures_ += 1; continue; } @@ -374,36 +483,82 @@ void WebcamCapture::captureLoop() { DWORD maxLength = 0; DWORD currentLength = 0; if (FAILED(buffer->Lock(&data, &maxLength, ¤tLength)) || !data) { + bufferFailures_ += 1; continue; } - const DWORD expectedLength = static_cast(std::max(0, width_) * std::max(0, height_) * 4); + const int pixels = std::max(0, width_) * std::max(0, height_); + const DWORD expectedLength = + static_cast(deliversNv12_ ? pixels * 3 / 2 : pixels * 4); if (currentLength >= expectedLength && expectedLength > 0) { - std::scoped_lock lock(frameMutex_); - latestFrame_.assign(data, data + expectedLength); - latestFrameSequence_ += 1; + if (framesDelivered_ == 0) { + const auto waitedMs = std::chrono::duration_cast( + std::chrono::steady_clock::now() - loopStartedAt) + .count(); + std::cerr << "INFO: First webcam frame after " << waitedMs << "ms" << std::endl; + } + framesDelivered_ += 1; + const auto storeStartedAt = std::chrono::steady_clock::now(); + { + std::scoped_lock lock(frameMutex_); + latestFrame_.assign(data, data + expectedLength); + latestFrameSequence_ += 1; + } + storeUs_ += static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - storeStartedAt) + .count()); + } else if (shortBuffers_++, !reportedShortBuffer_) { + // Every frame arriving short means the reader is handing us something + // other than the RGB32 we asked for -- a compressed frame it never + // converted, most often. Silently dropped, that reads downstream as a + // camera that produced nothing at all, with no clue why. Said once, + // because it is true for every frame that follows. + reportedShortBuffer_ = true; + std::cerr << "WARNING: Webcam frame is " << currentLength << " bytes, expected " + << expectedLength << " for " << width_ << "x" << height_ << " " + << (deliversNv12_ ? "NV12" : "RGB32") << "; dropping frames" << std::endl; } buffer->Unlock(); } + std::cerr << "INFO: Webcam capture loop ended: delivered=" << framesDelivered_ + << " emptySamples=" << emptySamples_ << " shortBuffers=" << shortBuffers_ + << " bufferFailures=" << bufferFailures_ << " readFailures=" << readFailures_ + << " lastReadHr=0x" << std::hex << lastReadFailure_ << std::dec + << " endOfStream=" << (sawEndOfStream_ ? "yes" : "no") + << " readSampleMs=" << (readSampleUs_ / 1000) << " storeMs=" << (storeUs_ / 1000) + << std::endl; + CoUninitialize(); } -bool WebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination) { +void WebcamCapture::markLost(HRESULT hr) { + if (lost_.exchange(true)) { + return; + } + reportWebcamLost(selectedDeviceName_, hr); +} + +bool WebcamCapture::isLost() const { if (usingDirectShow_) { - return directShowCapture_.copyLatestFrame(destination); + return directShowCapture_.isLost(); } - std::scoped_lock lock(frameMutex_); - if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) { - return false; + return lost_; +} + +bool WebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination, uint64_t lastSeenSequence) { + if (usingDirectShow_) { + return directShowCapture_.copyLatestFrame(destination, lastSeenSequence); } + std::scoped_lock lock(frameMutex_); + return snapshotWebcamFrame( + latestFrame_, width_, height_, latestFrameSequence_, lastSeenSequence, destination); +} - destination.data = latestFrame_; - destination.width = width_; - destination.height = height_; - destination.sequence = latestFrameSequence_; - return true; +bool WebcamCapture::deliversNv12() const { + return !usingDirectShow_ && deliversNv12_; } int WebcamCapture::width() const { diff --git a/electron/native/wgc-capture/src/webcam_capture.h b/electron/native/wgc-capture/src/webcam_capture.h index 5b61aa6b9..5888e5ed9 100644 --- a/electron/native/wgc-capture/src/webcam_capture.h +++ b/electron/native/wgc-capture/src/webcam_capture.h @@ -1,5 +1,6 @@ #pragma once +#include "device_selection.h" #include "dshow_webcam_capture.h" #include @@ -22,32 +23,63 @@ class WebcamCapture { WebcamCapture(const WebcamCapture&) = delete; WebcamCapture& operator=(const WebcamCapture&) = delete; + /** + * Opens the requested camera, skipping devices in `claims`. + * + * `claims` belongs to the take: every camera of it is initialized against + * the same set, in config order, and one that opens adds its device. That + * is what sends a second camera of the same model to the second device + * instead of the busy first one. An empty set selects exactly as before. + */ bool initialize( const std::wstring& deviceId, const std::wstring& deviceName, const std::wstring& directShowClsid, int requestedWidth, int requestedHeight, - int requestedFps); + int requestedFps, + bool preferNv12, + DeviceClaims& claims); bool start(); void stop(); - bool copyLatestFrame(WebcamFrameSnapshot& destination); + bool copyLatestFrame(WebcamFrameSnapshot& destination, uint64_t lastSeenSequence); int width() const; int height() const; int fps() const; + /** + * Do the frames from `copyLatestFrame` carry NV12 rather than BGRA? + * + * Always false on the DirectShow fallback, which unpacks whatever the + * camera speaks into BGRA itself. Callers must ask rather than assume: + * the two layouts differ in size as well as in meaning. + */ + bool deliversNv12() const; const std::wstring& selectedDeviceName() const; + /** + * Has the camera left the take -- unplugged, or its driver given up? + * + * Latched: once true it stays true, and the camera delivers no more + * frames. Safe to ask from any thread. + */ + bool isLost() const; private: - bool selectDevice(const std::wstring& deviceId, const std::wstring& deviceName); - bool configureReader(int requestedWidth, int requestedHeight, int requestedFps); + bool selectDevice( + const std::wstring& deviceId, + const std::wstring& deviceName, + const DeviceClaims& claims); + bool configureReader(int requestedWidth, int requestedHeight, int requestedFps, bool preferNv12); void captureLoop(); + /** Latches `lost_` and says so once; see `isWebcamLossResult`. */ + void markLost(HRESULT hr); Microsoft::WRL::ComPtr mediaSource_; Microsoft::WRL::ComPtr sourceReader_; DirectShowWebcamCapture directShowCapture_; std::thread thread_; std::atomic stopRequested_ = false; + std::atomic lost_ = false; std::mutex frameMutex_; std::vector latestFrame_; uint64_t latestFrameSequence_ = 0; @@ -56,6 +88,28 @@ class WebcamCapture { int fps_ = 30; bool mfStarted_ = false; bool usingDirectShow_ = false; - int selectedMatchScore_ = 0; + bool deliversNv12_ = false; + /** Latches the short-buffer warning so it is said once, not 30x a second. */ + bool reportedShortBuffer_ = false; + /** + * Why the capture loop did or did not produce frames. + * + * A camera that delivers nothing used to surface only as + * MF_E_SINK_NO_SAMPLES_PROCESSED from the encoder's Finalize -- the one + * place that cannot say which of the loop's five silent `continue` paths + * swallowed the frames. These are reported once when the loop ends. + */ + uint64_t framesDelivered_ = 0; + uint64_t emptySamples_ = 0; + uint64_t readFailures_ = 0; + uint64_t shortBuffers_ = 0; + uint64_t bufferFailures_ = 0; + bool sawEndOfStream_ = false; + HRESULT lastReadFailure_ = S_OK; + /** Where the loop's wall time goes, in microseconds. */ + uint64_t readSampleUs_ = 0; + uint64_t storeUs_ = 0; std::wstring selectedDeviceName_; + /** The symbolic link of the device this capture opened, for the take's claims. */ + std::wstring selectedIdentity_; }; diff --git a/electron/native/wgc-capture/src/webcam_config.cpp b/electron/native/wgc-capture/src/webcam_config.cpp new file mode 100644 index 000000000..d437b9d26 --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_config.cpp @@ -0,0 +1,92 @@ +#include "webcam_config.h" + +#include "json_fields.h" + +#include + +namespace { + +// The find* helpers search the whole document, so each array entry is sliced +// out first to keep its fields from being confused with the top-level ones. +std::vector splitTopLevelObjects(const std::string& json, const std::string& arrayKey) { + std::vector objects; + size_t pos = json.find("\"" + arrayKey + "\""); + if (pos == std::string::npos) { + return objects; + } + pos = json.find('[', pos); + if (pos == std::string::npos) { + return objects; + } + ++pos; + + int depth = 0; + bool inString = false; + size_t objectStart = 0; + for (; pos < json.size(); ++pos) { + const char c = json[pos]; + if (inString) { + if (c == '\\') { + ++pos; + } else if (c == '"') { + inString = false; + } + continue; + } + if (c == '"') { + inString = true; + } else if (c == '{') { + if (depth == 0) { + objectStart = pos; + } + ++depth; + } else if (c == '}') { + if (depth > 0 && --depth == 0) { + objects.push_back(json.substr(objectStart, pos - objectStart + 1)); + } + } else if (c == ']' && depth == 0) { + break; + } + } + return objects; +} + +} // namespace + +std::vector parseWebcamConfigs(const std::string& json) { + std::vector configs; + for (const std::string& entry : splitTopLevelObjects(json, "webcams")) { + if (configs.size() >= kMaxWebcams) { + break; + } + WebcamConfig config; + config.outputPath = findString(entry, "camPath"); + if (config.outputPath.empty()) { + continue; + } + config.deviceId = findString(entry, "camDeviceId"); + config.deviceName = findString(entry, "camDeviceName"); + config.directShowClsid = findString(entry, "camClsid"); + config.width = findInt(entry, "camWidth", 0); + config.height = findInt(entry, "camHeight", 0); + config.fps = findInt(entry, "camFps", 0); + configs.push_back(std::move(config)); + } + if (!configs.empty() || !findBool(json, "webcamEnabled", false)) { + return configs; + } + + WebcamConfig legacy; + legacy.outputPath = findString(json, "webcamPath"); + if (legacy.outputPath.empty()) { + return configs; + } + legacy.deviceId = findString(json, "webcamDeviceId"); + legacy.deviceName = findString(json, "webcamDeviceName"); + legacy.directShowClsid = findString(json, "webcamDirectShowClsid"); + legacy.width = findInt(json, "webcamWidth", 0); + legacy.height = findInt(json, "webcamHeight", 0); + legacy.fps = findInt(json, "webcamFps", 0); + configs.push_back(std::move(legacy)); + return configs; +} diff --git a/electron/native/wgc-capture/src/webcam_config.h b/electron/native/wgc-capture/src/webcam_config.h new file mode 100644 index 000000000..b0c0998ea --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_config.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include + +struct WebcamConfig { + std::string deviceId, deviceName, directShowClsid, outputPath; + int width = 0, height = 0, fps = 0; +}; + +constexpr size_t kMaxWebcams = 4; + +// The cameras to record, in order. Reads the `webcams` list when present and +// non-empty; otherwise the legacy single-camera fields (webcamEnabled, webcamDeviceId, +// webcamDeviceName, webcamDirectShowClsid, webcamWidth/Height/Fps, webcamPath). +std::vector parseWebcamConfigs(const std::string& json); diff --git a/electron/native/wgc-capture/src/webcam_config_test.cpp b/electron/native/wgc-capture/src/webcam_config_test.cpp new file mode 100644 index 000000000..1696698bf --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_config_test.cpp @@ -0,0 +1,63 @@ +#include "webcam_config.h" + +#include +#include + +namespace { +int failures = 0; +void expect(bool ok, const char* label) { + if (!ok) { + std::printf("FAIL %s\n", label); + ++failures; + } +} +} // namespace + +int main() { + const std::string legacy = + R"({"fps":60,"width":2560,"height":1440,"webcamEnabled":true,"webcamDeviceId":"id1",)" + R"("webcamDeviceName":"Cam One","webcamDirectShowClsid":"{A}","webcamWidth":1920,)" + R"("webcamHeight":1080,"webcamFps":30,"outputs":{"screenPath":"s.mp4","webcamPath":"w.mp4"}})"; + auto l = parseWebcamConfigs(legacy); + expect(l.size() == 1, "legacy: one camera"); + expect(l.size() == 1 && l[0].deviceName == "Cam One" && l[0].outputPath == "w.mp4" && + l[0].width == 1920 && l[0].fps == 30, + "legacy: fields"); + + expect(parseWebcamConfigs(R"({"webcamEnabled":false,"webcamPath":"w.mp4"})").empty(), + "legacy disabled: none"); + + const std::string list = + R"({"fps":60,"width":2560,"webcamEnabled":true,"webcamDeviceName":"Cam One","webcamPath":"w.mp4",)" + R"("webcams":[{"camDeviceId":"id1","camDeviceName":"Cam One","camClsid":"{A}","camWidth":1920,)" + R"("camHeight":1080,"camFps":30,"camPath":"w.mp4"},)" + R"({"camDeviceId":"id2","camDeviceName":"Desk \"Cam\"","camClsid":"","camWidth":1280,)" + R"("camHeight":720,"camFps":30,"camPath":"w-2.mp4"}]})"; + auto m = parseWebcamConfigs(list); + expect(m.size() == 2, "list: two cameras"); + expect(m.size() == 2 && m[1].deviceName == "Desk \"Cam\"" && m[1].outputPath == "w-2.mp4" && + m[1].width == 1280 && m[1].height == 720, + "list: second camera fields, escaped quote"); + expect(m.size() == 2 && m[0].fps == 30, "list: entry fps not the top-level fps"); + + expect(parseWebcamConfigs(R"({"webcamEnabled":true,"webcamPath":"w.mp4","webcams":[]})").size() == 1, + "empty list falls back to legacy"); + + std::string five = R"({"webcams":[)"; + for (int i = 0; i < 5; ++i) { + five += (i ? "," : ""); + five += R"({"camDeviceName":"C","camPath":"p)" + std::to_string(i) + R"(.mp4"})"; + } + five += "]}"; + expect(parseWebcamConfigs(five).size() == 4, "capped at four"); + + expect(parseWebcamConfigs(R"({"webcams":[{"camDeviceName":"No path"}]})").empty(), + "entry without camPath is skipped"); + + if (failures == 0) { + std::printf("webcam_config_test: all assertions passed\n"); + return 0; + } + std::printf("webcam_config_test: %d assertion(s) failed\n", failures); + return 1; +} diff --git a/electron/native/wgc-capture/src/webcam_format.cpp b/electron/native/wgc-capture/src/webcam_format.cpp new file mode 100644 index 000000000..32e2020ed --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_format.cpp @@ -0,0 +1,82 @@ +#include "webcam_format.h" + +#include + +namespace { + +bool isUsable(const WebcamFormat& format) { + return format.width > 0 && format.height > 0; +} + +bool fitsTarget(const WebcamFormat& format, int targetWidth, int targetHeight) { + return format.width <= targetWidth && format.height <= targetHeight; +} + +long long pixels(const WebcamFormat& format) { + return static_cast(format.width) * static_cast(format.height); +} + +// True when `candidate` is a better frame rate than `best` for a target of +// `targetFps`: the lowest rate that still reaches the target, or -- when no +// mode reaches it -- the highest one on offer. +bool betterFps(int candidate, int best, int targetFps) { + const bool candidateReaches = candidate >= targetFps; + const bool bestReaches = best >= targetFps; + if (candidateReaches != bestReaches) { + return candidateReaches; + } + return candidateReaches ? candidate < best : candidate > best; +} + +} // namespace + +WebcamFormat chooseWebcamFormat( + const std::vector& available, + int targetWidth, + int targetHeight, + int targetFps) { + const WebcamFormat target{targetWidth, targetHeight, targetFps, false}; + + // Compressed modes are a last resort, so they are only considered when the + // camera offers nothing else at all. + const bool anyUncompressed = std::any_of(available.begin(), available.end(), [](const WebcamFormat& format) { + return isUsable(format) && !format.compressed; + }); + + const auto eligible = [&](const WebcamFormat& format) { + return isUsable(format) && !(anyUncompressed && format.compressed); + }; + + const bool anyFits = std::any_of(available.begin(), available.end(), [&](const WebcamFormat& format) { + return eligible(format) && fitsTarget(format, targetWidth, targetHeight); + }); + + const WebcamFormat* best = nullptr; + for (const WebcamFormat& format : available) { + if (!eligible(format)) { + continue; + } + // Once anything fits, oversized modes are out of the running entirely. + // With nothing fitting we rank the oversized modes instead, smallest + // first, so the scaler has the least work to do. + if (anyFits != fitsTarget(format, targetWidth, targetHeight)) { + continue; + } + if (best == nullptr) { + best = &format; + continue; + } + if (pixels(format) != pixels(*best)) { + const bool wins = anyFits ? pixels(format) > pixels(*best) : pixels(format) < pixels(*best); + if (wins) { + best = &format; + } + continue; + } + if (betterFps(format.fps, best->fps, targetFps)) { + best = &format; + } + } + + return best ? *best : target; +} diff --git a/electron/native/wgc-capture/src/webcam_format.h b/electron/native/wgc-capture/src/webcam_format.h new file mode 100644 index 000000000..7cb329af7 --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_format.h @@ -0,0 +1,44 @@ +#pragma once + +#include + +// One capture format a camera advertises. +struct WebcamFormat { + int width = 0; + int height = 0; + int fps = 0; + /** + * True for MJPEG/H264 modes, which reach higher resolutions than a camera's + * uncompressed ones but need a decoder MFT in front of the RGB32 conversion. + */ + bool compressed = false; +}; + +// Picks the format to drive the camera at, given everything it advertises. +// +// Both capture backends used to leave this to the driver: Media Foundation was +// handed a media type with no MF_MT_FRAME_SIZE, and the DirectShow graph was +// rendered without touching IAMStreamConfig. In both cases the device answers +// with its *default* type, and for a UVC camera that is the first format it +// enumerates -- 640x480 even on a BRIO that offers 3840x2160. The recording was +// then upscaled into the overlay, which is the pixelation users reported. +// +// Rules, in order: +// * uncompressed modes beat compressed ones outright, whatever their size. +// Pinning a camera's MJPEG mode and asking the source reader for RGB32 +// leaves it delivering no samples at all on some hosts -- a recording with +// no camera in it, which is far worse than a smaller one that works; +// * only formats that fit inside the target in BOTH dimensions are eligible, +// so an ultrawide mode cannot sneak in on pixel count alone; +// * among those, the most pixels wins; +// * ties go to the lowest frame rate that still reaches the target, else the +// highest available -- a 1080p60 mode is no help when we encode at 30; +// * if nothing fits (a camera that only does 4K), the smallest mode wins, so +// we scale down rather than push 1 GB/s of RGB32 through the encoder; +// * an empty list returns the target unchanged, letting the caller fall back +// to whatever the driver would have picked on its own. +WebcamFormat chooseWebcamFormat( + const std::vector& available, + int targetWidth, + int targetHeight, + int targetFps); diff --git a/electron/native/wgc-capture/src/webcam_format_test.cpp b/electron/native/wgc-capture/src/webcam_format_test.cpp new file mode 100644 index 000000000..9e40f5cbf --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_format_test.cpp @@ -0,0 +1,131 @@ +#include "webcam_format.h" + +#include +#include + +namespace { + +int failures = 0; + +void expectFormat(const std::string& label, WebcamFormat actual, int width, int height, int fps) { + if (actual.width == width && actual.height == height && actual.fps == fps) { + return; + } + std::printf( + "FAIL %s: expected %dx%d@%d, got %dx%d@%d\n", + label.c_str(), + width, + height, + fps, + actual.width, + actual.height, + actual.fps); + ++failures; +} + +// Every mode a Logitech BRIO advertises that this helper could drive, as read +// off the device with `ffmpeg -f dshow -list_options true`. +const std::vector kBrio = { + {640, 480, 30, false}, {640, 360, 30, false}, {1280, 720, 30, false}, + {1920, 1080, 30, false}, {1280, 720, 90, true}, {1920, 1080, 60, true}, + {2560, 1440, 30, true}, {3840, 2160, 30, true}, {160, 120, 30, false}, + {340, 340, 30, false}, {1600, 896, 30, false}, {1024, 576, 30, false}, +}; + +} // namespace + +int main() { + // The regression this file exists for: a BRIO asked for 1080p must not come + // back with the 640x480 default it enumerates first. + expectFormat("brio targets 1080p", chooseWebcamFormat(kBrio, 1920, 1080, 30), 1920, 1080, 30); + + // Asked for more than the camera can do uncompressed, it must stop at the + // best uncompressed mode rather than pin an MJPEG one. Pinning MJPEG and + // asking the source reader for RGB32 produced a reader that delivered no + // samples at all, so the take had no camera in it. + expectFormat( + "4K request settles for the best uncompressed mode", + chooseWebcamFormat(kBrio, 3840, 2160, 30), + 1920, + 1080, + 30); + if (chooseWebcamFormat(kBrio, 3840, 2160, 30).compressed) { + std::printf("FAIL 4K request must not pin a compressed mode\n"); + ++failures; + } + + // A camera with nothing but compressed modes still has to be usable. + expectFormat( + "compressed-only camera is still driven", + chooseWebcamFormat({{1920, 1080, 30, true}, {1280, 720, 30, true}}, 1920, 1080, 30), + 1920, + 1080, + 30); + + // Uncompressed wins even when it is much smaller than a compressed option. + expectFormat( + "uncompressed beats a larger compressed mode", + chooseWebcamFormat({{3840, 2160, 30, true}, {640, 480, 30, false}}, 3840, 2160, 30), + 640, + 480, + 30); + + // 1080p60 exists too; at a 30 fps encode the extra frames only cost bandwidth. + expectFormat( + "prefers the frame rate we encode at", + chooseWebcamFormat({{1920, 1080, 60}, {1920, 1080, 30}}, 1920, 1080, 30), + 1920, + 1080, + 30); + + // Nothing reaches 30, so take the best the camera can do rather than fail. + expectFormat( + "falls back to the highest rate offered", + chooseWebcamFormat({{1920, 1080, 15}, {1920, 1080, 24}}, 1920, 1080, 30), + 1920, + 1080, + 24); + + // A 720p-only camera keeps working, just smaller. + expectFormat( + "720p camera stays 720p", + chooseWebcamFormat({{1280, 720, 30}, {640, 480, 30}}, 1920, 1080, 30), + 1280, + 720, + 30); + + // 2560x800 has fewer pixels than 1920x1080 but is far wider; picking it + // would letterbox the overlay and cost a needless downscale. + expectFormat( + "rejects an oversized dimension even when pixel count fits", + chooseWebcamFormat({{2560, 800, 30}, {1280, 720, 30}}, 1920, 1080, 30), + 1280, + 720, + 30); + + // Only modes above the target: scale down from the cheapest of them. + expectFormat( + "oversized-only camera takes the smallest mode", + chooseWebcamFormat({{3840, 2160, 30}, {2560, 1440, 30}}, 1920, 1080, 30), + 2560, + 1440, + 30); + + // Nothing to choose from: hand the target back and let the driver decide. + expectFormat("empty list returns the target", chooseWebcamFormat({}, 1920, 1080, 30), 1920, 1080, 30); + + // Drivers do report junk; it must not win. + expectFormat( + "ignores degenerate entries", + chooseWebcamFormat({{0, 0, 30}, {-1920, 1080, 30}, {1280, 720, 30}}, 1920, 1080, 30), + 1280, + 720, + 30); + + if (failures == 0) { + std::printf("webcam_format_test: all assertions passed\n"); + return 0; + } + std::printf("webcam_format_test: %d assertion(s) failed\n", failures); + return 1; +} diff --git a/electron/native/wgc-capture/src/webcam_loss.cpp b/electron/native/wgc-capture/src/webcam_loss.cpp new file mode 100644 index 000000000..7548f84c2 --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_loss.cpp @@ -0,0 +1,28 @@ +#include "webcam_loss.h" + +#include + +#include + +bool isWebcamLossResult(HRESULT hr, bool endOfStream, int64_t consecutiveFailureMs) { + if (SUCCEEDED(hr)) { + return endOfStream; + } + if (hr == MF_E_VIDEO_RECORDING_DEVICE_INVALIDATED || hr == MF_E_HW_MFT_FAILED_START_STREAMING) { + return true; + } + return consecutiveFailureMs >= kWebcamLossFailureRunMs; +} + +void reportWebcamLost(const std::wstring& deviceName, HRESULT hr) { + std::string name; + const int size = WideCharToMultiByte( + CP_UTF8, 0, deviceName.data(), static_cast(deviceName.size()), nullptr, 0, nullptr, nullptr); + if (size > 0) { + name.resize(static_cast(size)); + WideCharToMultiByte( + CP_UTF8, 0, deviceName.data(), static_cast(deviceName.size()), name.data(), size, nullptr, nullptr); + } + std::cerr << "WARNING: Webcam lost during the take: " << name << " hr=0x" << std::hex + << static_cast(hr) << std::dec << std::endl; +} diff --git a/electron/native/wgc-capture/src/webcam_loss.h b/electron/native/wgc-capture/src/webcam_loss.h new file mode 100644 index 000000000..64af1f987 --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_loss.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +#include +#include + +/** How long reads may keep failing, back to back, before the camera counts as gone. */ +constexpr int64_t kWebcamLossFailureRunMs = 1000; + +/** + * Does this read result mean the camera has left the take? + * + * Media Foundation reports an unplugged camera as + * MF_E_VIDEO_RECORDING_DEVICE_INVALIDATED on every following ReadSample, and a + * camera whose driver gave up as MF_E_HW_MFT_FAILED_START_STREAMING; both are + * final. End of stream while the take is still running is the same thing said + * differently. Any other failure is forgiven once -- a camera can drop a read -- + * but not for `kWebcamLossFailureRunMs` in a row: a reader that fails that long + * delivers nothing, and treating it as alive is what froze a file on its last + * picture for the rest of the take. + * + * `consecutiveFailureMs` is the time since the first failure of the current + * run; a successful read ends the run. Ignored when `hr` succeeded. + */ +bool isWebcamLossResult(HRESULT hr, bool endOfStream, int64_t consecutiveFailureMs); + +/** + * The one line both capture backends print when a camera leaves the take. + * + * `hr` is the read result that decided it, or the DirectShow event code when + * the graph reported the loss. + */ +void reportWebcamLost(const std::wstring& deviceName, HRESULT hr); diff --git a/electron/native/wgc-capture/src/webcam_loss_test.cpp b/electron/native/wgc-capture/src/webcam_loss_test.cpp new file mode 100644 index 000000000..2a8462f8d --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_loss_test.cpp @@ -0,0 +1,41 @@ +#include "webcam_loss.h" + +#include + +#include + +namespace { +int failures = 0; +void expect(bool ok, const char* label) { + if (!ok) { + std::printf("FAIL %s\n", label); + ++failures; + } +} +} // namespace + +int main() { + // The code the on-site test logged after the C920 was unplugged. + expect(isWebcamLossResult(MF_E_VIDEO_RECORDING_DEVICE_INVALIDATED, false, 0), + "device invalidated: lost on the first read"); + expect(isWebcamLossResult(MF_E_HW_MFT_FAILED_START_STREAMING, false, 0), + "hardware failed to start streaming: lost"); + expect(isWebcamLossResult(S_OK, true, 0), "end of stream during the take: lost"); + + expect(!isWebcamLossResult(E_FAIL, false, 0), "one other failure: not lost"); + expect(!isWebcamLossResult(E_FAIL, false, kWebcamLossFailureRunMs - 1), + "other failures for just under a second: not lost"); + expect(isWebcamLossResult(E_FAIL, false, kWebcamLossFailureRunMs), + "other failures for a second: lost"); + expect(isWebcamLossResult(MF_E_NOTACCEPTING, false, 5000), "any failure code, long run: lost"); + + expect(!isWebcamLossResult(S_OK, false, 0), "S_OK: not lost"); + expect(!isWebcamLossResult(S_OK, false, 5000), "S_OK ends a run of failures: not lost"); + + if (failures == 0) { + std::printf("webcam_loss_test: all assertions passed\n"); + return 0; + } + std::printf("webcam_loss_test: %d failure(s)\n", failures); + return 1; +} diff --git a/electron/native/wgc-capture/src/webcam_snapshot_test.cpp b/electron/native/wgc-capture/src/webcam_snapshot_test.cpp new file mode 100644 index 000000000..0dd555cb4 --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_snapshot_test.cpp @@ -0,0 +1,46 @@ +#include "dshow_webcam_capture.h" + +#include + +namespace { + +int failures = 0; + +void expect(const char* label, bool condition) { + if (condition) { + return; + } + std::printf("FAIL %s\n", label); + ++failures; +} + +} // namespace + +int main() { + const std::vector frame(4 * 2 * 2, 0x80); + + // A new sequence is copied, with the size and sequence it was stored with. + WebcamFrameSnapshot fresh; + expect("new sequence is copied", snapshotWebcamFrame(frame, 2, 2, 7, 6, fresh)); + expect("new sequence carries its data", fresh.data == frame); + expect("new sequence carries its size", fresh.width == 2 && fresh.height == 2); + expect("new sequence carries its number", fresh.sequence == 7); + + // The regression this file exists for: the writer polls at the screen's + // rate, twice the camera's, and a frame it already holds must cost nothing. + WebcamFrameSnapshot unchanged; + expect("unchanged sequence reports nothing new", !snapshotWebcamFrame(frame, 2, 2, 7, 7, unchanged)); + expect("unchanged sequence copies nothing", unchanged.data.empty() && unchanged.sequence == 0); + + // No frame stored yet: nothing to hand out, whatever the caller last saw. + WebcamFrameSnapshot empty; + expect("empty frame is not copied", !snapshotWebcamFrame({}, 2, 2, 1, 0, empty)); + expect("unknown size is not copied", !snapshotWebcamFrame(frame, 0, 2, 1, 0, empty)); + + if (failures > 0) { + std::printf("%d webcam snapshot check(s) failed\n", failures); + return 1; + } + std::printf("webcam snapshot checks passed\n"); + return 0; +} diff --git a/electron/native/wgc-capture/src/wgc_session.cpp b/electron/native/wgc-capture/src/wgc_session.cpp index b75d0819e..726d7effd 100644 --- a/electron/native/wgc-capture/src/wgc_session.cpp +++ b/electron/native/wgc-capture/src/wgc_session.cpp @@ -214,6 +214,27 @@ bool WgcSession::applySessionOptions(bool captureCursor) { // IsBorderRequired is Windows 11-only. Ignore it on older builds. } + // Best effort, never a reason to fail the recording: a menu missing from the + // video is today's behavior, a recording that does not start is not. + if (includeSecondaryWindows_ && !secondaryWindowsReported_) { + bool applied = false; +#if defined(NTDDI_WIN11_GE) + // IGraphicsCaptureSession6 (Windows 11 24H2, SDK 26100). try_as returns + // null on an older runtime, so it degrades to the window alone. + try { + if (auto session6 = session_.try_as()) { + session6.IncludeSecondaryWindows(true); + applied = session6.IncludeSecondaryWindows(); + } + } catch (...) { + // Left at applied=false; the event below says so. + } +#endif + secondaryWindowsReported_ = true; + std::cout << "{\"event\":\"secondary-windows\",\"applied\":" << (applied ? "true" : "false") + << "}" << std::endl; + } + return true; } @@ -240,8 +261,9 @@ bool WgcSession::initialize(HMONITOR monitor, int fps, bool captureCursor) { return true; } -bool WgcSession::initialize(HWND window, int fps, bool captureCursor) { +bool WgcSession::initialize(HWND window, int fps, bool captureCursor, bool includeSecondaryWindows) { fps_ = fps > 0 ? fps : 60; + includeSecondaryWindows_ = includeSecondaryWindows; if (!createD3DDevice()) { return false; } diff --git a/electron/native/wgc-capture/src/wgc_session.h b/electron/native/wgc-capture/src/wgc_session.h index 35eb25a54..4dbb33e43 100644 --- a/electron/native/wgc-capture/src/wgc_session.h +++ b/electron/native/wgc-capture/src/wgc_session.h @@ -53,7 +53,11 @@ class WgcSession { WgcSession& operator=(const WgcSession&) = delete; bool initialize(HMONITOR monitor, int fps, bool captureCursor); - bool initialize(HWND window, int fps, bool captureCursor); + // includeSecondaryWindows asks DWM to draw the popups and tool windows that + // overlap the captured window (context menus, dropdowns) into the same + // texture. Windows 11 24H2+ only; older runtimes silently keep capturing the + // window alone, and a `secondary-windows` event reports which one happened. + bool initialize(HWND window, int fps, bool captureCursor, bool includeSecondaryWindows = false); bool start(); // Returns the most recently arrived frame's texture and timestamp, or // false if none is available since the last call. The returned pointer @@ -113,5 +117,7 @@ class WgcSession { int height_ = 0; int fps_ = 60; bool captureCursor_ = false; + bool includeSecondaryWindows_ = false; + bool secondaryWindowsReported_ = false; bool started_ = false; }; diff --git a/electron/native/whisper-stt/CMakeLists.txt b/electron/native/whisper-stt/CMakeLists.txt index b95baf7d1..a22d02eed 100644 --- a/electron/native/whisper-stt/CMakeLists.txt +++ b/electron/native/whisper-stt/CMakeLists.txt @@ -144,9 +144,45 @@ endif() FetchContent_MakeAvailable(whisper httplib json) -add_executable(whisper-stt-server src/main.cpp) +# Character-level DTW (issue #948): whisper.cpp's word-timing pass, +# whisper_exp_compute_token_level_timestamps_dtw, and the median filter only it +# used, are replaced by whisper-patches/char-dtw.cpp. The whisper target +# compiles that copy of whisper.cpp from the build tree, so the fetched source +# is never modified: a shared FETCHCONTENT_SOURCE_DIR_WHISPER and the Nix build +# get the same result. Bumping WHISPER_REF fails here, loudly, if the code moved. +set(OSC_DTW_FRAGMENT "${CMAKE_CURRENT_SOURCE_DIR}/whisper-patches/char-dtw.cpp") +set(OSC_WHISPER_CPP "${whisper_SOURCE_DIR}/src/whisper.cpp") +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${OSC_DTW_FRAGMENT}" "${OSC_WHISPER_CPP}") +file(READ "${OSC_WHISPER_CPP}" osc_src) +string(REPLACE "\r\n" "\n" osc_src "${osc_src}") +string(FIND "${osc_src}" "\nstruct median_filter_user_data {" osc_begin) +string(FIND "${osc_src}" "\nvoid whisper_log_set(" osc_end) +if(osc_begin GREATER -1 AND osc_end GREATER osc_begin) + math(EXPR osc_len "${osc_end} - ${osc_begin}") + string(SUBSTRING "${osc_src}" ${osc_begin} ${osc_len} osc_old) + string(FIND "${osc_old}" "static void whisper_exp_compute_token_level_timestamps_dtw(" osc_fn) +endif() +if(osc_begin EQUAL -1 OR NOT osc_end GREATER osc_begin OR osc_fn EQUAL -1) + message(FATAL_ERROR "char-dtw: the DTW pass was not found where expected in ${OSC_WHISPER_CPP}") +endif() +string(SUBSTRING "${osc_src}" 0 ${osc_begin} osc_head) +string(SUBSTRING "${osc_src}" ${osc_end} -1 osc_tail) +file(READ "${OSC_DTW_FRAGMENT}" osc_fragment) +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/whisper-patched/whisper.cpp.tmp" + "${osc_head}\n${osc_fragment}${osc_tail}") +configure_file("${CMAKE_CURRENT_BINARY_DIR}/whisper-patched/whisper.cpp.tmp" + "${CMAKE_CURRENT_BINARY_DIR}/whisper-patched/whisper.cpp" COPYONLY) +get_target_property(osc_whisper_srcs whisper SOURCES) +list(TRANSFORM osc_whisper_srcs REPLACE "^whisper\\.cpp$" "${CMAKE_CURRENT_BINARY_DIR}/whisper-patched/whisper.cpp") +if(NOT "${CMAKE_CURRENT_BINARY_DIR}/whisper-patched/whisper.cpp" IN_LIST osc_whisper_srcs) + message(FATAL_ERROR "char-dtw: the whisper target does not list whisper.cpp as expected (${osc_whisper_srcs})") +endif() +set_property(TARGET whisper PROPERTY SOURCES ${osc_whisper_srcs}) + +add_executable(whisper-stt-server src/main.cpp src/ctc_aligner.cpp) target_link_libraries(whisper-stt-server PRIVATE whisper + ggml httplib::httplib nlohmann_json::nlohmann_json ) diff --git a/electron/native/whisper-stt/src/ctc_aligner.cpp b/electron/native/whisper-stt/src/ctc_aligner.cpp new file mode 100644 index 000000000..18d514596 --- /dev/null +++ b/electron/native/whisper-stt/src/ctc_aligner.cpp @@ -0,0 +1,433 @@ +// wav2vec2-for-CTC forward pass on ggml (HuggingFace Wav2Vec2ForCTC, both the +// "group norm, post-LN" base layout and the "layer norm, pre-LN" large one). +// Weights come from scripts/convert-wav2vec2-gguf.mjs. +// +// Layout: activations are [channels, time] (ne0 = channels) for the linear +// layers, and transposed to [time, channels] only where a convolution or a +// per-channel norm needs time contiguous. + +#include "ctc_aligner.h" + +#include +#include +#include +#include + +#include +#include +#include + +namespace { + +constexpr int kGraphSize = 8192; +// Longest input one graph sees; longer ones run in windows. Attention is +// quadratic in time and the first convolutions are wide, so this bounds memory +// (about 265 MB of compute buffers at 20 s, for either model: the first +// convolutions dominate). +constexpr int kWindowFrames = 1000; // 20 s +constexpr int kContextFrames = 50; // 1 s of context kept on each side of a window + +struct Layer { + ggml_tensor *q_w, *q_b, *k_w, *k_b, *v_w, *v_b, *o_w, *o_b; + ggml_tensor *ln1_w, *ln1_b, *ff1_w, *ff1_b, *ff2_w, *ff2_b, *ln2_w, *ln2_b; +}; + +} // namespace + +struct CtcModel { + CtcModelInfo info; + int hidden = 0, n_heads = 0, pos_k = 0, pos_groups = 0; + float eps = 1e-5f; + bool group_norm = false, stable = false; + std::vector conv_k, conv_s; + + std::vector conv_w, conv_b, conv_ln_w, conv_ln_b; + ggml_tensor *fp_ln_w = nullptr, *fp_ln_b = nullptr, *fp_w = nullptr, *fp_b = nullptr; + ggml_tensor *pos_w = nullptr, *pos_b = nullptr, *enc_ln_w = nullptr, *enc_ln_b = nullptr; + ggml_tensor *head_w = nullptr, *head_b = nullptr; + std::vector layers; + + ggml_context* wctx = nullptr; + gguf_context* gctx = nullptr; + ggml_backend_buffer_t wbuf = nullptr; + ggml_backend_t gpu = nullptr, cpu = nullptr; + ggml_backend_sched_t sched = nullptr; + std::string device = "CPU"; + + ~CtcModel() { + if (sched) ggml_backend_sched_free(sched); + if (wbuf) ggml_backend_buffer_free(wbuf); + if (gpu) ggml_backend_free(gpu); + if (cpu) ggml_backend_free(cpu); + if (wctx) ggml_free(wctx); + if (gctx) gguf_free(gctx); + } +}; + +void CtcModelDeleter::operator()(CtcModel* m) const { delete m; } + +const CtcModelInfo& ctc_info(const CtcModel& model) { return model.info; } +std::string ctc_device(const CtcModel& model) { return model.device; } + +namespace { + +int64_t key(gguf_context* g, const char* k, std::string& err) { + const int64_t id = gguf_find_key(g, k); + if (id < 0 && err.empty()) err = std::string("missing metadata ") + k; + return id; +} + +std::vector str_array(gguf_context* g, int64_t id) { + std::vector out; + if (id < 0) return out; + const size_t n = gguf_get_arr_n(g, id); + for (size_t i = 0; i < n; ++i) out.emplace_back(gguf_get_arr_str(g, id, i)); + return out; +} + +std::vector u32_array(gguf_context* g, int64_t id) { + std::vector out; + if (id < 0) return out; + const auto* p = static_cast(gguf_get_arr_data(g, id)); + out.assign(p, p + gguf_get_arr_n(g, id)); + return out; +} + +ggml_tensor* layer_norm(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, ggml_tensor* b, float eps) { + return ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, eps), w), b); +} + +ggml_tensor* linear(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, ggml_tensor* b) { + return ggml_add(ctx, ggml_mul_mat(ctx, w, x), b); +} + +// 1-D convolution of `x` [time, in] with `w` [k, in, out] -> [out, time']. The +// columns are F16, as in whisper.cpp's encoder: half the memory traffic of F32, +// which made the CPU path 1.7x slower, and `w` must be F16 to match. +ggml_tensor* conv1d(ggml_context* ctx, ggml_tensor* w, ggml_tensor* x, int stride, int pad) { + ggml_tensor* cols = ggml_im2col(ctx, w, x, stride, 0, pad, 0, 1, 0, false, GGML_TYPE_F16); + // cols: [k * in, time'] + cols = ggml_reshape_2d(ctx, cols, cols->ne[0], cols->ne[1]); + ggml_tensor* w2 = ggml_reshape_2d(ctx, w, w->ne[0] * w->ne[1], w->ne[2]); + return ggml_mul_mat(ctx, w2, cols); +} + +ggml_tensor* build(CtcModel& m, ggml_context* ctx, ggml_tensor* wave) { + // ---- feature encoder ---- + ggml_tensor* x = wave; // [time, 1] + ggml_tensor* y = nullptr; + for (size_t i = 0; i < m.conv_w.size(); ++i) { + y = conv1d(ctx, m.conv_w[i], x, m.conv_s[i], 0); // [512, t] + if (m.conv_b[i]) y = ggml_add(ctx, y, m.conv_b[i]); + if (m.group_norm) { + // Time-major for the next convolution. Only the first layer is normed: + // GroupNorm with one group per channel, i.e. each channel over time. + ggml_tensor* t = ggml_cont(ctx, ggml_transpose(ctx, y)); // [t, 512] + if (i == 0) { + t = ggml_norm(ctx, t, m.eps); + t = ggml_add(ctx, ggml_mul(ctx, t, ggml_reshape_2d(ctx, m.conv_ln_w[0], 1, t->ne[1])), + ggml_reshape_2d(ctx, m.conv_ln_b[0], 1, t->ne[1])); + } + x = ggml_gelu_erf(ctx, t); + } else { + y = layer_norm(ctx, y, m.conv_ln_w[i], m.conv_ln_b[i], m.eps); + x = ggml_cont(ctx, ggml_transpose(ctx, ggml_gelu_erf(ctx, y))); // [t, 512] + } + } + // x: [t, 512] -> [512, t] + ggml_tensor* h = ggml_cont(ctx, ggml_transpose(ctx, x)); + const int64_t T = h->ne[1]; + + // ---- feature projection ---- + h = layer_norm(ctx, h, m.fp_ln_w, m.fp_ln_b, m.eps); + h = linear(ctx, h, m.fp_w, m.fp_b); // [H, T] + + // ---- positional convolution (grouped, same padding, drop the last frame) ---- + { + ggml_tensor* ht = ggml_cont(ctx, ggml_transpose(ctx, h)); // [T, H] + const int64_t cg = m.hidden / m.pos_groups; + ggml_tensor* pos = nullptr; + for (int g = 0; g < m.pos_groups; ++g) { + ggml_tensor* xin = ggml_view_2d(ctx, ht, T, cg, ht->nb[1], g * cg * ht->nb[1]); + ggml_tensor* wg = ggml_view_3d(ctx, m.pos_w, m.pos_k, cg, cg, m.pos_w->nb[1], + m.pos_w->nb[2], g * cg * m.pos_w->nb[2]); + ggml_tensor* part = conv1d(ctx, wg, ggml_cont(ctx, xin), 1, m.pos_k / 2); // [cg, T+1] + pos = pos ? ggml_concat(ctx, pos, part, 0) : part; + } + pos = ggml_view_2d(ctx, pos, m.hidden, T, pos->nb[1], 0); + pos = ggml_gelu_erf(ctx, ggml_add(ctx, pos, m.pos_b)); + h = ggml_add(ctx, h, pos); + } + if (!m.stable) h = layer_norm(ctx, h, m.enc_ln_w, m.enc_ln_b, m.eps); + + // ---- transformer ---- + const int64_t dh = m.hidden / m.n_heads; + const float scale = 1.0f / std::sqrt(static_cast(dh)); + for (const Layer& L : m.layers) { + ggml_tensor* res = h; + ggml_tensor* a = m.stable ? layer_norm(ctx, h, L.ln1_w, L.ln1_b, m.eps) : h; + ggml_tensor* q = ggml_reshape_3d(ctx, linear(ctx, a, L.q_w, L.q_b), dh, m.n_heads, T); + ggml_tensor* k = ggml_reshape_3d(ctx, linear(ctx, a, L.k_w, L.k_b), dh, m.n_heads, T); + ggml_tensor* v = ggml_reshape_3d(ctx, linear(ctx, a, L.v_w, L.v_b), dh, m.n_heads, T); + q = ggml_permute(ctx, q, 0, 2, 1, 3); // [dh, T, nh] + k = ggml_permute(ctx, k, 0, 2, 1, 3); // [dh, T, nh] + v = ggml_cont(ctx, ggml_permute(ctx, v, 1, 2, 0, 3)); // [T, dh, nh] + ggml_tensor* kq = ggml_mul_mat(ctx, k, q); // [Tk, Tq, nh] + kq = ggml_soft_max_ext(ctx, kq, nullptr, scale, 0.0f); + ggml_tensor* o = ggml_mul_mat(ctx, v, kq); // [dh, Tq, nh] + o = ggml_cont(ctx, ggml_permute(ctx, o, 0, 2, 1, 3)); // [dh, nh, T] + o = linear(ctx, ggml_reshape_2d(ctx, o, m.hidden, T), L.o_w, L.o_b); + h = ggml_add(ctx, res, o); + if (m.stable) { + ggml_tensor* f = layer_norm(ctx, h, L.ln2_w, L.ln2_b, m.eps); + f = linear(ctx, ggml_gelu_erf(ctx, linear(ctx, f, L.ff1_w, L.ff1_b)), L.ff2_w, L.ff2_b); + h = ggml_add(ctx, h, f); + } else { + h = layer_norm(ctx, h, L.ln1_w, L.ln1_b, m.eps); + ggml_tensor* f = linear(ctx, ggml_gelu_erf(ctx, linear(ctx, h, L.ff1_w, L.ff1_b)), L.ff2_w, L.ff2_b); + h = layer_norm(ctx, ggml_add(ctx, h, f), L.ln2_w, L.ln2_b, m.eps); + } + } + if (m.stable) h = layer_norm(ctx, h, m.enc_ln_w, m.enc_ln_b, m.eps); + return linear(ctx, h, m.head_w, m.head_b); // [V, T] +} + +int frames_for(const CtcModel& m, int64_t n) { + for (size_t i = 0; i < m.conv_k.size(); ++i) { + if (n < m.conv_k[i]) return 0; + n = (n - m.conv_k[i]) / m.conv_s[i] + 1; + } + return static_cast(n); +} + +// Logits for one window of already-normalized samples. +bool run_window(CtcModel& m, const float* pcm, int64_t n, std::vector& logits, int& frames, + std::string& err) { + frames = frames_for(m, n); + if (frames <= 0) return true; + ggml_init_params p = {ggml_tensor_overhead() * kGraphSize + ggml_graph_overhead_custom(kGraphSize, false), + nullptr, true}; + ggml_context* ctx = ggml_init(p); + ggml_tensor* wave = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n, 1); + ggml_set_input(wave); + ggml_tensor* out = build(m, ctx, wave); + ggml_set_output(out); + ggml_cgraph* gf = ggml_new_graph_custom(ctx, kGraphSize, false); + ggml_build_forward_expand(gf, out); + ggml_backend_sched_reset(m.sched); + bool ok = ggml_backend_sched_alloc_graph(m.sched, gf); + if (ok) { + ggml_backend_tensor_set(wave, pcm, 0, n * sizeof(float)); + ok = ggml_backend_sched_graph_compute(m.sched, gf) == GGML_STATUS_SUCCESS; + } + if (ok) { + if (out->ne[1] != frames) { + err = "unexpected frame count"; + ok = false; + } else { + logits.resize(static_cast(out->ne[0]) * frames); + ggml_backend_tensor_get(out, logits.data(), 0, logits.size() * sizeof(float)); + } + } else if (err.empty()) { + err = "ggml graph allocation or compute failed"; + } + ggml_free(ctx); + return ok; +} + +} // namespace + +CtcModelPtr ctc_load(const std::string& path, bool use_gpu, int threads, std::string& err) { + CtcModelPtr m(new CtcModel()); + gguf_init_params gp = {true, &m->wctx}; + m->gctx = gguf_init_from_file(path.c_str(), gp); + if (!m->gctx) { + err = "cannot read " + path; + return nullptr; + } + gguf_context* g = m->gctx; + CtcModelInfo& info = m->info; + info.vocab = str_array(g, key(g, "w2v.vocab", err)); + info.languages = str_array(g, key(g, "w2v.languages", err)); + int64_t id; + if ((id = key(g, "w2v.blank_id", err)) >= 0) info.blank = gguf_get_val_u32(g, id); + if ((id = key(g, "w2v.hidden_size", err)) >= 0) m->hidden = gguf_get_val_u32(g, id); + if ((id = key(g, "w2v.num_attention_heads", err)) >= 0) m->n_heads = gguf_get_val_u32(g, id); + if ((id = key(g, "w2v.layer_norm_eps", err)) >= 0) m->eps = gguf_get_val_f32(g, id); + if ((id = key(g, "w2v.num_conv_pos_embeddings", err)) >= 0) m->pos_k = gguf_get_val_u32(g, id); + if ((id = key(g, "w2v.num_conv_pos_embedding_groups", err)) >= 0) m->pos_groups = gguf_get_val_u32(g, id); + if ((id = key(g, "w2v.feat_extract_norm", err)) >= 0) m->group_norm = std::string(gguf_get_val_str(g, id)) == "group"; + if ((id = key(g, "w2v.stable_layer_norm", err)) >= 0) m->stable = gguf_get_val_bool(g, id); + m->conv_k = u32_array(g, key(g, "w2v.conv_kernel", err)); + m->conv_s = u32_array(g, key(g, "w2v.conv_stride", err)); + int n_layers = 0; + if ((id = key(g, "w2v.num_hidden_layers", err)) >= 0) n_layers = gguf_get_val_u32(g, id); + if (!err.empty()) return nullptr; + info.stride = 1; + for (int s : m->conv_s) info.stride *= s; + info.receptive_field = 1; + for (int i = static_cast(m->conv_k.size()) - 1; i >= 0; --i) + info.receptive_field = (info.receptive_field - 1) * m->conv_s[i] + m->conv_k[i]; + + auto T = [&](const std::string& name, bool required = true) -> ggml_tensor* { + ggml_tensor* t = ggml_get_tensor(m->wctx, name.c_str()); + if (!t && required && err.empty()) err = "missing tensor " + name; + return t; + }; + for (size_t i = 0; i < m->conv_k.size(); ++i) { + const std::string p = "feature_extractor.conv_layers." + std::to_string(i) + "."; + m->conv_w.push_back(T(p + "conv.weight")); + m->conv_b.push_back(T(p + "conv.bias", false)); + const bool has_ln = !m->group_norm || i == 0; + m->conv_ln_w.push_back(has_ln ? T(p + "layer_norm.weight") : nullptr); + m->conv_ln_b.push_back(has_ln ? T(p + "layer_norm.bias") : nullptr); + } + m->fp_ln_w = T("feature_projection.layer_norm.weight"); + m->fp_ln_b = T("feature_projection.layer_norm.bias"); + m->fp_w = T("feature_projection.projection.weight"); + m->fp_b = T("feature_projection.projection.bias"); + m->pos_w = T("encoder.pos_conv_embed.conv.weight"); + m->pos_b = T("encoder.pos_conv_embed.conv.bias"); + m->enc_ln_w = T("encoder.layer_norm.weight"); + m->enc_ln_b = T("encoder.layer_norm.bias"); + m->head_w = T("lm_head.weight"); + m->head_b = T("lm_head.bias"); + for (int l = 0; l < n_layers; ++l) { + const std::string p = "encoder.layers." + std::to_string(l) + "."; + m->layers.push_back({ + T(p + "attention.q_proj.weight"), T(p + "attention.q_proj.bias"), + T(p + "attention.k_proj.weight"), T(p + "attention.k_proj.bias"), + T(p + "attention.v_proj.weight"), T(p + "attention.v_proj.bias"), + T(p + "attention.out_proj.weight"), T(p + "attention.out_proj.bias"), + T(p + "layer_norm.weight"), T(p + "layer_norm.bias"), + T(p + "feed_forward.intermediate_dense.weight"), T(p + "feed_forward.intermediate_dense.bias"), + T(p + "feed_forward.output_dense.weight"), T(p + "feed_forward.output_dense.bias"), + T(p + "final_layer_norm.weight"), T(p + "final_layer_norm.bias"), + }); + } + if (!err.empty()) return nullptr; + // conv1d() multiplies them with F16 columns; anything else aborts inside ggml. + for (const ggml_tensor* w : m->conv_w) { + if (w->type != GGML_TYPE_F16) err = "convolution weights must be F16"; + } + if (m->pos_w->type != GGML_TYPE_F16) err = "convolution weights must be F16"; + if (!err.empty()) return nullptr; + if (static_cast(info.vocab.size()) != m->head_w->ne[1]) { + err = "vocabulary does not match lm_head"; + return nullptr; + } + + // ---- backends ---- + if (use_gpu) { + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + const auto type = ggml_backend_dev_type(dev); + if (type != GGML_BACKEND_DEVICE_TYPE_GPU && type != GGML_BACKEND_DEVICE_TYPE_IGPU) continue; + m->gpu = ggml_backend_dev_init(dev, nullptr); + if (m->gpu) { + m->device = ggml_backend_dev_name(dev); + break; + } + } + } + m->cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + if (!m->cpu) { + err = "no CPU backend"; + return nullptr; + } + { + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(m->cpu)); + using set_threads_t = void (*)(ggml_backend_t, int); + auto set_threads = reinterpret_cast( + ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads")); + if (set_threads) set_threads(m->cpu, threads); + } + ggml_backend_t weights_on = m->gpu ? m->gpu : m->cpu; + m->wbuf = ggml_backend_alloc_ctx_tensors(m->wctx, weights_on); + if (!m->wbuf) { + err = "cannot allocate the model's weights"; + return nullptr; + } + { + FILE* f = std::fopen(path.c_str(), "rb"); + if (!f) { + err = "cannot open " + path; + return nullptr; + } + std::vector buf; + const size_t data_off = gguf_get_data_offset(g); + for (int64_t i = 0; i < gguf_get_n_tensors(g); ++i) { + ggml_tensor* t = ggml_get_tensor(m->wctx, gguf_get_tensor_name(g, i)); + const size_t nbytes = ggml_nbytes(t); + buf.resize(nbytes); +#ifdef _WIN32 + _fseeki64(f, static_cast(data_off + gguf_get_tensor_offset(g, i)), SEEK_SET); +#else + fseeko(f, static_cast(data_off + gguf_get_tensor_offset(g, i)), SEEK_SET); +#endif + if (std::fread(buf.data(), 1, nbytes, f) != nbytes) { + std::fclose(f); + err = "truncated model file " + path; + return nullptr; + } + ggml_backend_tensor_set(t, buf.data(), 0, nbytes); + } + std::fclose(f); + } + std::vector backends; + if (m->gpu) backends.push_back(m->gpu); + backends.push_back(m->cpu); + m->sched = ggml_backend_sched_new(backends.data(), nullptr, static_cast(backends.size()), + kGraphSize, false, true); + if (!m->sched) { + err = "cannot create the ggml scheduler"; + return nullptr; + } + return m; +} + +bool ctc_emissions(CtcModel& m, const float* pcm, size_t n, std::vector& out, int& frames, + std::string& err) { + const int V = static_cast(m.info.vocab.size()); + const int64_t stride = m.info.stride; + frames = frames_for(m, static_cast(n)); + out.assign(static_cast(std::max(frames, 0)) * V, 0.0f); + if (frames <= 0) return true; + + // Wav2Vec2FeatureExtractor(do_normalize=True): zero mean, unit variance, per input. + double mean = 0.0, var = 0.0; + for (size_t i = 0; i < n; ++i) mean += pcm[i]; + mean /= static_cast(n); + for (size_t i = 0; i < n; ++i) var += (pcm[i] - mean) * (pcm[i] - mean); + var /= static_cast(n); + const double inv = 1.0 / std::sqrt(var + 1e-7); + std::vector x(n); + for (size_t i = 0; i < n; ++i) x[i] = static_cast((pcm[i] - mean) * inv); + + // Windows of kWindowFrames, stepping by kWindowFrames - 2 * kContextFrames; each + // keeps its frames away from the edges it shares with a neighbour. + const int64_t win_samples = kWindowFrames * stride + (m.info.receptive_field - stride); + const int hop = kWindowFrames - 2 * kContextFrames; + std::vector logits; + for (int f0 = 0;; f0 += hop) { + const int64_t s0 = static_cast(f0) * stride; + const int64_t len = std::min(win_samples, static_cast(n) - s0); + int wf = 0; + if (!run_window(m, x.data() + s0, len, logits, wf, err)) return false; + const bool last = s0 + len >= static_cast(n); + const int keep_from = f0 == 0 ? 0 : kContextFrames; + const int keep_to = last ? wf : std::min(wf, kWindowFrames - kContextFrames); + for (int i = keep_from; i < keep_to && f0 + i < frames; ++i) { + const float* row = logits.data() + static_cast(i) * V; + float mx = row[0]; + for (int v = 1; v < V; ++v) mx = std::max(mx, row[v]); + double sum = 0.0; + for (int v = 0; v < V; ++v) sum += std::exp(row[v] - mx); + const float lse = mx + static_cast(std::log(sum)); + float* dst = out.data() + static_cast(f0 + i) * V; + for (int v = 0; v < V; ++v) dst[v] = row[v] - lse; + } + if (last) break; + } + return true; +} diff --git a/electron/native/whisper-stt/src/ctc_aligner.h b/electron/native/whisper-stt/src/ctc_aligner.h new file mode 100644 index 000000000..80997505f --- /dev/null +++ b/electron/native/whisper-stt/src/ctc_aligner.h @@ -0,0 +1,37 @@ +// CTC acoustic model (wav2vec2 fine-tuned for CTC) run on ggml, for the word +// aligner's second pass. The helper only computes the per-frame log-probs; the +// forced alignment of whisper's words over them lives on the Node side +// (electron/stt/ctcAlign.ts). See transcription-and-captions.md § Word-level alignment. +#pragma once + +#include +#include +#include + +struct CtcModel; + +struct CtcModelInfo { + std::vector vocab; // token id -> text; "|" is the word delimiter + std::vector languages; // what the model was fine-tuned on + int blank = 0; + int stride = 320; // samples per output frame (20 ms at 16 kHz) + int receptive_field = 400; // samples the first frame sees +}; + +struct CtcModelDeleter { void operator()(CtcModel* m) const; }; +using CtcModelPtr = std::unique_ptr; + +// Loads a GGUF written by scripts/convert-wav2vec2-gguf.mjs. On the GPU when +// `use_gpu` and a GPU device is registered, else on the CPU. Null + `err` on failure. +CtcModelPtr ctc_load(const std::string& path, bool use_gpu, int threads, std::string& err); + +const CtcModelInfo& ctc_info(const CtcModel& model); + +// Name of the device the weights live on ("Vulkan0", "MTL0", "CPU"). +std::string ctc_device(const CtcModel& model); + +// Log-softmax emissions for 16 kHz mono `pcm`: `frames` rows of vocab-size +// floats. Frame i sees samples [i * stride, i * stride + receptive_field). +// Long inputs run in overlapping windows, so memory stays bounded. +bool ctc_emissions(CtcModel& model, const float* pcm, size_t n, std::vector& out, + int& frames, std::string& err); diff --git a/electron/native/whisper-stt/src/main.cpp b/electron/native/whisper-stt/src/main.cpp index f287eed2e..77440e54d 100644 --- a/electron/native/whisper-stt/src/main.cpp +++ b/electron/native/whisper-stt/src/main.cpp @@ -27,7 +27,8 @@ // "start": 0.0, "end": 11.0, // "words": [ { "word": "...", "start": 0.5, "end": 0.9, "probability": 0.9 }, ... ] // }, ... -// ] +// ], +// "speech": [ { "start": 0.4, "end": 2.1 }, ... ] // only when the VAD model loaded // } // // Concurrency: whisper contexts are not thread-safe. /inference is serialized @@ -35,6 +36,7 @@ // this is a belt-and-braces guarantee against a future bug or parallel invoker). #include "whisper.h" +#include "ctc_aligner.h" #include #include @@ -222,12 +224,57 @@ std::string detect_active_backend() { } struct Word { - double start = 0.0; - double end = 0.0; - double prob = 0.0; + double start = 0.0; + double end = 0.0; + // t_dtw of the word's first token: the end of that token, so always inside + // the word. Only used to tell which stretch of speech a word belongs to. + double anchor = 0.0; + double prob = 0.0; std::string text; }; +// One stretch of speech copied from the upload into the buffer whisper decodes. +struct Kept { + int64_t at; // first sample in that buffer + int64_t from; // first sample in the upload + int64_t len; +}; + +// A whisper time (centiseconds on the speech-only buffer) back on the upload's +// clock, in seconds. Inside a kept stretch it moves with the stretch; in the +// silence inserted between two stretches it snaps to the nearer edge, so a +// word can never land in audio that was cut out. No stretches: VAD was off. +double to_original_sec(int64_t cs, const std::vector& kept) { + if (kept.empty()) return cs / 100.0; + const int64_t x = cs * 160; // 16 kHz: 160 samples per centisecond + for (size_t i = 0; i < kept.size(); ++i) { + const Kept& k = kept[i]; + if (x < k.at) { + if (i == 0) return k.from / 16000.0; + const Kept& p = kept[i - 1]; + const bool nearer_prev = x - (p.at + p.len) <= k.at - x; + return (nearer_prev ? p.from + p.len : k.from) / 16000.0; + } + if (x < k.at + k.len) return (k.from + x - k.at) / 16000.0; + } + return (kept.back().from + kept.back().len) / 16000.0; +} + +std::string base64(const void* data, size_t n) { + static const char* abc = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + const auto* p = static_cast(data); + std::string out; + out.reserve((n + 2) / 3 * 4); + for (size_t i = 0; i < n; i += 3) { + const uint32_t v = (p[i] << 16) | (i + 1 < n ? p[i + 1] << 8 : 0) | (i + 2 < n ? p[i + 2] : 0); + out += abc[(v >> 18) & 63]; + out += abc[(v >> 12) & 63]; + out += i + 1 < n ? abc[(v >> 6) & 63] : '='; + out += i + 2 < n ? abc[v & 63] : '='; + } + return out; +} + } // namespace int main(int argc, char** argv) { @@ -312,17 +359,17 @@ int main(int argc, char** argv) { log("model loaded; backend=" + active_backend); // ---- Init VAD context (Silero VAD v6.2.0) ---- + // On the CPU, always. Asked for the GPU on Vulkan, whisper.cpp 1.9.1 puts the + // VAD weights in a Vulkan buffer, then finds no GPU for the VAD's own backend + // and aborts inside ggml (0xC0000409) — so the app relaunched the helper with + // --cpu and every transcription lost its GPU. Silero is 0.9 MB: the CPU is + // not where the time goes. struct whisper_vad_context* vctx = nullptr; if (!vad_model_path.empty()) { struct whisper_vad_context_params vad_ctx_params = whisper_vad_default_context_params(); vad_ctx_params.n_threads = threads; - vad_ctx_params.use_gpu = !force_cpu; + vad_ctx_params.use_gpu = false; vctx = whisper_vad_init_from_file_with_params(vad_model_path.c_str(), vad_ctx_params); - if (!vctx && vad_ctx_params.use_gpu) { - log("GPU VAD initialization failed; retrying with CPU inference"); - vad_ctx_params.use_gpu = false; - vctx = whisper_vad_init_from_file_with_params(vad_model_path.c_str(), vad_ctx_params); - } if (vctx) { log("VAD model loaded; path=" + vad_model_path); } else { @@ -345,13 +392,15 @@ int main(int argc, char** argv) { res.set_content(probe.dump(), "application/json"); }); - // POST /inference — multipart form with `file` (WAV) + `language` + `response_format`. - svr.Post("/inference", [&](const httplib::Request& req, httplib::Response& res) { + // The upload of /inference and /emissions: a 16 kHz mono PCM16 WAV in the + // multipart field `file`. False after answering 400. + const auto read_upload = [](const httplib::Request& req, httplib::Response& res, + std::vector& pcm) -> bool { auto it = req.files.find("file"); if (it == req.files.end()) { res.status = 400; res.set_content(R"({"error":"missing 'file' form field"})", "application/json"); - return; + return false; } const auto& file_entry = it->second; @@ -373,7 +422,6 @@ int main(int argc, char** argv) { out.write(file_entry.content.data(), static_cast(file_entry.content.size())); } - std::vector pcm; int sample_rate = 0, channels = 0; const bool ok = read_wav_pcm16(tmp_wav, pcm, sample_rate, channels); std::error_code ec; @@ -381,15 +429,22 @@ int main(int argc, char** argv) { if (!ok) { res.status = 400; res.set_content(R"({"error":"failed to parse WAV"})", "application/json"); - return; + return false; } if (sample_rate != 16000 || channels != 1) { res.status = 400; res.set_content( R"({"error":"expected 16 kHz mono PCM16 WAV"})", "application/json"); - return; + return false; } + return true; + }; + + // POST /inference — multipart form with `file` (WAV) + `language` + `response_format`. + svr.Post("/inference", [&](const httplib::Request& req, httplib::Response& res) { + std::vector pcm; + if (!read_upload(req, res, pcm)) return; // language param std::string language = "auto"; @@ -407,14 +462,52 @@ int main(int argc, char** argv) { wparams.print_realtime = false; wparams.print_timestamps = false; wparams.n_threads = threads; - if (vctx && !vad_model_path.empty()) { - wparams.vad = true; - wparams.vad_model_path = vad_model_path.c_str(); - wparams.vad_params = whisper_vad_default_params(); + + // ---- Speech only (Silero VAD), cut here rather than by whisper_full ---- + // whisper_full's own `vad` param maps SEGMENT times back onto the upload + // but not token times, so every word (t_dtw included) came back early by + // all the silence removed before it: 13 s into a 25 s clip. Cutting here + // keeps the map. The stretches also go out as `speech`, which the Node + // side anchors phrase edges on. + const auto t0 = std::chrono::steady_clock::now(); // VAD time is part of the run + std::vector speech; + std::vector kept; + nlohmann::json speech_json = nlohmann::json::array(); + if (vctx) { + whisper_vad_segments* vs = whisper_vad_segments_from_samples( + vctx, whisper_vad_default_params(), pcm.data(), static_cast(pcm.size())); + if (!vs) { + res.status = 500; + res.set_content(R"({"error":"VAD failed"})", "application/json"); + return; + } + const int64_t n_pcm = static_cast(pcm.size()); + const auto to_sample = [&](float cs) { + return std::clamp(std::llround(cs * 160.0), 0, n_pcm); + }; + const int n_vs = whisper_vad_segments_n_segments(vs); + for (int i = 0; i < n_vs; ++i) { + const int64_t from = to_sample(whisper_vad_segments_get_segment_t0(vs, i)); + const int64_t end = to_sample(whisper_vad_segments_get_segment_t1(vs, i)); + if (end <= from) continue; + speech_json.push_back({{"start", from / 16000.0}, {"end", end / 16000.0}}); + // As whisper.cpp does: 0.1 s past the detected end so a soft ending + // survives, and 0.1 s of silence between stretches. Unlike it, the + // tail never runs into the next stretch and decodes it twice. + const int64_t next = i + 1 < n_vs + ? to_sample(whisper_vad_segments_get_segment_t0(vs, i + 1)) + : n_pcm; + const int64_t to = std::max(end, std::min(end + 1600, next)); + if (!kept.empty()) speech.insert(speech.end(), 1600, 0.0f); + kept.push_back({static_cast(speech.size()), from, to - from}); + speech.insert(speech.end(), pcm.begin() + from, pcm.begin() + to); + } + whisper_vad_free_segments(vs); } + // VAD on and no speech found: nothing to decode, so nothing to report. + const std::vector& input = vctx ? speech : pcm; - const auto t0 = std::chrono::steady_clock::now(); - const int rc = whisper_full(ctx, wparams, pcm.data(), static_cast(pcm.size())); + const int rc = input.empty() ? 0 : whisper_full(ctx, wparams, input.data(), static_cast(input.size())); const auto t1 = std::chrono::steady_clock::now(); if (rc != 0) { log("whisper_full returned " + std::to_string(rc)); @@ -451,18 +544,28 @@ int main(int argc, char** argv) { }; std::vector segments; - const int n_segments = whisper_full_n_segments(ctx); + // whisper.cpp writes a token's t_dtw when the DTW path enters the decoder + // row that PREDICTS the next token (whisper_exp_compute_token_level_ + // timestamps_dtw, v1.9.1): it marks the END of the token, not its start. + // So a word runs from the t_dtw of the text token before it to the t_dtw + // of its own last token. Taking its first token's t_dtw as the start put + // every word one token late (+175 ms median, tools/stt-eval/word-timing). + // The previous token carries across segments; the request's very first + // word has none and starts on its first token's time (the Node side + // anchors it on the speech onset anyway). + double prev_tok_end = -1.0; + const int n_segments = input.empty() ? 0 : whisper_full_n_segments(ctx); for (int si = 0; si < n_segments; ++si) { Segment seg; - seg.start = whisper_full_get_segment_t0(ctx, si) / 100.0; - seg.end = whisper_full_get_segment_t1(ctx, si) / 100.0; + seg.start = to_original_sec(whisper_full_get_segment_t0(ctx, si), kept); + seg.end = to_original_sec(whisper_full_get_segment_t1(ctx, si), kept); if (const char* t = whisper_full_get_segment_text(ctx, si)) seg.text = t; - struct W { double t_dtw_first; double p_sum; int p_n; std::string text; }; + struct W { double start; double end; double anchor; double p_sum; int p_n; std::string text; }; std::vector word_buf; std::string cur_text; bool in_word = false; - double w_first_t_dtw = 0; + double w_start = 0, w_end = 0, w_anchor = 0; double w_p_sum = 0; int w_p_n = 0; const int n_tokens = whisper_full_n_tokens(ctx, si); @@ -486,33 +589,35 @@ int main(int argc, char** argv) { ++non_special_tokens; const bool starts_word = (!in_word) || (!raw.empty() && raw[0] == ' '); - const double td_dtw = (td.t_dtw >= 0 ? td.t_dtw : 0) / 100.0; + const double td_dtw = to_original_sec(td.t_dtw >= 0 ? td.t_dtw : 0, kept); if (starts_word && in_word) { - word_buf.push_back({ w_first_t_dtw, w_p_sum, w_p_n, cur_text }); + word_buf.push_back({ w_start, w_end, w_anchor, w_p_sum, w_p_n, cur_text }); w_p_sum = 0; w_p_n = 0; cur_text.clear(); } if (starts_word) { in_word = true; - w_first_t_dtw = td_dtw; + w_start = prev_tok_end >= 0 ? prev_tok_end : td_dtw; + w_anchor = td_dtw; cur_text = (!raw.empty() && raw[0] == ' ') ? raw.substr(1) : raw; } else { cur_text += raw; } + w_end = td_dtw; + prev_tok_end = td_dtw; w_p_sum += td.p; w_p_n += 1; } if (in_word) { - word_buf.push_back({ w_first_t_dtw, w_p_sum, w_p_n, cur_text }); + word_buf.push_back({ w_start, w_end, w_anchor, w_p_sum, w_p_n, cur_text }); } - for (size_t wi = 0; wi < word_buf.size(); ++wi) { + for (const W& b : word_buf) { Word w; - w.start = word_buf[wi].t_dtw_first; - w.end = (wi + 1 < word_buf.size()) - ? word_buf[wi + 1].t_dtw_first - : seg.end; - w.prob = word_buf[wi].p_sum / std::max(1, word_buf[wi].p_n); - w.text = word_buf[wi].text; + w.start = b.start; + w.end = b.end; + w.anchor = b.anchor; + w.prob = b.p_sum / std::max(1, b.p_n); + w.text = b.text; seg.words.push_back(w); } segments.push_back(std::move(seg)); @@ -547,7 +652,7 @@ int main(int argc, char** argv) { // whisper resolved — the detected one under "auto", and the forced one // otherwise, which is correct for both paths. std::string resolved_language = language; - const int lang_id = whisper_full_lang_id(ctx); + const int lang_id = input.empty() ? -1 : whisper_full_lang_id(ctx); if (lang_id >= 0) { if (const char* lang_str = whisper_lang_str(lang_id)) { resolved_language = lang_str; @@ -576,6 +681,7 @@ int main(int argc, char** argv) { {"word", w.text}, {"start", w.start}, {"end", w.end}, + {"anchor", w.anchor}, {"probability", w.prob}, }); } @@ -583,73 +689,78 @@ int main(int argc, char** argv) { segs.push_back(std::move(seg)); } reply["segments"] = std::move(segs); + if (vctx) reply["speech"] = std::move(speech_json); res.set_content(reply.dump(), "application/json"); }); - // POST /vad — multipart form with `file` (WAV: 16 kHz mono PCM16). - // Runs Silero VAD segmentation and returns speech intervals [start, end] in seconds. - svr.Post("/vad", [&](const httplib::Request& req, httplib::Response& res) { - if (!vctx) { - res.status = 503; - res.set_content(R"({"error":"VAD model was not loaded on server startup"})", "application/json"); - return; - } - auto it = req.files.find("file"); - if (it == req.files.end()) { - res.status = 400; - res.set_content(R"({"error":"missing 'file' form field"})", "application/json"); - return; - } - const auto& file_entry = it->second; - - static std::atomic tmp_vad_counter{0}; - const auto tmp_vad_id = tmp_vad_counter.fetch_add(1, std::memory_order_relaxed); - const auto tmp_vad_ts = std::chrono::high_resolution_clock::now().time_since_epoch().count(); - const std::string tmp_wav = (std::filesystem::temp_directory_path() / - ("openscreen-vad-" + std::to_string(tmp_vad_ts) + - "-" + std::to_string(tmp_vad_id) + ".wav")).string(); - { - std::ofstream out(tmp_wav, std::ios::binary); - out.write(file_entry.content.data(), - static_cast(file_entry.content.size())); - } + // POST /emissions — the CTC aligner's acoustic pass (issue #948, phase 3). + // Multipart form: `file` (the same WAV as /inference), `model` (path of a + // wav2vec2 GGUF, see ctc_aligner.h) and `regions` (JSON [[start_s, end_s], ...]). + // Answers the model's vocabulary and, per region, base64 float32 log-probs + // [frames x vocab]; frame i of a region sees the audio from + // `start + i * stride_s` for `receptive_s`. The forced alignment itself runs + // on the Node side (electron/stt/ctcAlign.ts). The model stays loaded until a + // request names another one. + CtcModelPtr aligner; + std::string aligner_path; + svr.Post("/emissions", [&](const httplib::Request& req, httplib::Response& res) { std::vector pcm; - int sample_rate = 0, channels = 0; - const bool ok = read_wav_pcm16(tmp_wav, pcm, sample_rate, channels); - std::error_code ec; - std::filesystem::remove(tmp_wav, ec); - if (!ok) { + if (!read_upload(req, res, pcm)) return; + const std::string model = req.get_file_value("model").content; + nlohmann::json regions = nlohmann::json::parse(req.get_file_value("regions").content, nullptr, false); + if (model.empty() || !regions.is_array()) { res.status = 400; - res.set_content(R"({"error":"failed to parse WAV"})", "application/json"); + res.set_content(R"({"error":"need 'model' and a JSON 'regions' array"})", "application/json"); return; } - if (sample_rate != 16000 || channels != 1) { - res.status = 400; - res.set_content( - R"({"error":"expected 16 kHz mono PCM16 WAV"})", - "application/json"); - return; - } - const std::lock_guard lk(infer_mu); - struct whisper_vad_params vad_params = whisper_vad_default_params(); - struct whisper_vad_segments* vad_segments = - whisper_vad_segments_from_samples(vctx, vad_params, pcm.data(), static_cast(pcm.size())); - - nlohmann::json reply; - reply["segments"] = nlohmann::json::array(); - if (vad_segments) { - const int n_segs = whisper_vad_segments_n_segments(vad_segments); - for (int i = 0; i < n_segs; ++i) { - const float t0 = whisper_vad_segments_get_segment_t0(vad_segments, i); - const float t1 = whisper_vad_segments_get_segment_t1(vad_segments, i); - reply["segments"].push_back({ - {"start", static_cast(t0) / 100.0}, - {"end", static_cast(t1) / 100.0} - }); + const auto t0 = std::chrono::steady_clock::now(); + if (!aligner || aligner_path != model) { + aligner.reset(); + std::string err; + aligner = ctc_load(model, cparams.use_gpu, threads, err); + if (!aligner) { + log("aligner: " + err); + res.status = 500; + res.set_content(nlohmann::json{{"error", "aligner: " + err}}.dump(), "application/json"); + return; } - whisper_vad_free_segments(vad_segments); + aligner_path = model; + log("aligner loaded on " + ctc_device(*aligner) + ": " + model); } + const CtcModelInfo& info = ctc_info(*aligner); + nlohmann::json out_regions = nlohmann::json::array(); + const int64_t n_pcm = static_cast(pcm.size()); + for (const auto& r : regions) { + if (!r.is_array() || r.size() != 2 || !r[0].is_number() || !r[1].is_number()) continue; + const int64_t from = std::clamp(std::llround(r[0].get() * 16000.0), 0, n_pcm); + const int64_t to = std::clamp(std::llround(r[1].get() * 16000.0), from, n_pcm); + std::vector lp; + int frames = 0; + std::string err; + if (!ctc_emissions(*aligner, pcm.data() + from, static_cast(to - from), lp, frames, err)) { + log("aligner: " + err); + res.status = 500; + res.set_content(nlohmann::json{{"error", "aligner: " + err}}.dump(), "application/json"); + return; + } + out_regions.push_back({ + {"start", from / 16000.0}, + {"frames", frames}, + {"logprobs", base64(lp.data(), lp.size() * sizeof(float))}, + }); + } + const double elapsed_s = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + nlohmann::json reply = { + {"vocab", info.vocab}, + {"blank", info.blank}, + {"languages", info.languages}, + {"stride_s", info.stride / 16000.0}, + {"receptive_s", info.receptive_field / 16000.0}, + {"device", ctc_device(*aligner)}, + {"elapsed_s", elapsed_s}, + {"regions", std::move(out_regions)}, + }; res.set_content(reply.dump(), "application/json"); }); diff --git a/electron/native/whisper-stt/whisper-patches/char-dtw.cpp b/electron/native/whisper-stt/whisper-patches/char-dtw.cpp new file mode 100644 index 000000000..241ac5abd --- /dev/null +++ b/electron/native/whisper-stt/whisper-patches/char-dtw.cpp @@ -0,0 +1,227 @@ +// OpenScreen (issue #948): character-level teacher-forced DTW. +// +// Spliced into whisper.cpp v1.9.1 by ../CMakeLists.txt in place of everything +// from `struct median_filter_user_data` to the end of +// whisper_exp_compute_token_level_timestamps_dtw. Same signature, same output: +// a `t_dtw` on every text token of the window, in centiseconds. +// +// Upstream teacher-forces the decoded BPE tokens once more and runs DTW on the +// cross-attention of the alignment heads. This does the same with the text cut +// into single characters ("Whisper Has an Internal Word Aligner", +// arXiv 2509.09987): one decoder row per character, so a word boundary is +// where the path reaches the row that predicts the space or the letter after +// it, not wherever a multi-letter token happens to start. On the word-timing +// harness (tools/stt-eval/word-timing) it takes the inner-boundary median from +// 31 to 16 ms. +// +// t_dtw keeps its upstream meaning: the time the path enters the row that +// predicts whatever follows the token, i.e. the END of the token. +// +// Departures from upstream, all measured on the harness: +// - the attention of the heads is averaged and each audio frame's column is +// scaled to unit L2 norm, as in the paper. Upstream's z-score + median filter +// does worse on characters (+7 ms median); +// - French drops one silent final consonant per word before aligning (vais, +// vous, plaît): its row otherwise eats the start of the next word, +60 to +// +140 ms on the words after it; +// - the decoder has n_text_ctx positions. A window too long as characters is +// aligned in several runs: one stretch of it as characters, the rest of the +// window around it as BPE tokens; +// - runs never step back in time: each t_dtw is at least the previous one. +static void whisper_exp_compute_token_level_timestamps_dtw( + struct whisper_context * ctx, + struct whisper_state * state, + struct whisper_full_params params, + int i_segment, + size_t n_segments, + int seek, + int n_frames, + int /*medfilt_width*/, + int n_threads) +{ + const int n_audio_ctx = state->exp_n_audio_ctx > 0 ? state->exp_n_audio_ctx : ctx->model.hparams.n_audio_ctx; + WHISPER_ASSERT(n_frames <= n_audio_ctx * 2); + WHISPER_ASSERT(ctx->params.dtw_aheads_preset != WHISPER_AHEADS_NONE); + + const whisper_token eot = whisper_token_eot(ctx); + const auto & vocab = ctx->vocab; + + std::vector text; + for (size_t i = i_segment; i < i_segment + n_segments; ++i) { + for (auto & t : state->result_all[i].tokens) { + if (t.id < eot) { + text.push_back(&t); + } + } + } + if (text.empty()) { + return; + } + + std::vector sot_seq = { whisper_token_sot(ctx) }; + if (whisper_is_multilingual(ctx)) { + const int lang_id = whisper_lang_id(params.language); + state->lang_id = lang_id; + sot_seq.push_back(whisper_token_lang(ctx, lang_id)); + } + sot_seq.push_back(whisper_token_not(ctx)); + + // Each text token as one token per character. A character with no token of + // its own goes as its bytes; a token that cannot be split stays whole. + const auto find_id = [&](const std::string & s, whisper_token & id) { + const auto it = vocab.token_to_id.find(s); + if (it == vocab.token_to_id.end()) return false; + id = it->second; + return true; + }; + std::vector> chars(text.size()); + for (size_t k = 0; k < text.size(); ++k) { + const std::string & s = vocab.id_to_token.at(text[k]->id); + std::vector out; + bool ok = true; + for (size_t i = 0; ok && i < s.size();) { + const unsigned char c = s[i]; + const size_t len = std::min(c < 0x80 ? 1 : (c >> 5) == 0x6 ? 2 : (c >> 4) == 0xE ? 3 : (c >> 3) == 0x1E ? 4 : 1, s.size() - i); + const std::string piece = s.substr(i, len); + i += len; + whisper_token id; + if (find_id(piece, id)) { + out.push_back(id); + continue; + } + for (size_t b = 0; ok && b < piece.size(); ++b) { + ok = find_id(piece.substr(b, 1), id); + if (ok) out.push_back(id); + } + } + chars[k] = ok ? std::move(out) : std::vector{ text[k]->id }; + } + + if (state->lang_id == whisper_lang_id("fr")) { + // One silent final consonant per word of two letters or more. A word is a + // run of letters, across tokens; a byte >= 0x80 counts as a letter. + const auto ch = [&](whisper_token id) { + const std::string & s = vocab.id_to_token.at(id); + return s.size() == 1 ? (unsigned char) s[0] : (unsigned char) 0x80; + }; + std::vector> word; // (token, character) + const auto end_word = [&]() { + if (word.size() >= 2 && strchr("stxdz", tolower(ch(chars[word.back().first][word.back().second]))) != nullptr) { + chars[word.back().first][word.back().second] = -1; + } + word.clear(); + }; + for (size_t k = 0; k < chars.size(); ++k) { + for (size_t i = 0; i < chars[k].size(); ++i) { + const unsigned char c = ch(chars[k][i]); + if (c >= 0x80 || isalpha(c)) { + word.push_back({ k, i }); + } else { + end_word(); + } + } + } + end_word(); + for (auto & cs : chars) { + cs.erase(std::remove(cs.begin(), cs.end(), -1), cs.end()); + } + } + + struct ggml_init_params gparams = { + /*.mem_size =*/ ctx->params.dtw_mem_size, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ false, + }; + + int64_t t_floor = 0; + for (int i = i_segment - 1; i >= 0 && t_floor == 0; --i) { + for (const auto & t : state->result_all[i].tokens) { + if (t.id < eot && t.t_dtw > t_floor) t_floor = t.t_dtw; + } + } + + const int n_ctx = ctx->model.hparams.n_text_ctx; + const int n_text = (int) text.size(); + for (int k0 = 0; k0 < n_text;) { + // Tokens [k0, k1) as characters, as many as fit. + int len = (int) sot_seq.size() + n_text + 1; + int k1 = k0; + while (k1 < n_text && len - 1 + (int) chars[k1].size() <= n_ctx) { + len += (int) chars[k1].size() - 1; + ++k1; + } + if (k1 == k0) { + chars[k0] = { text[k0]->id }; + continue; + } + + std::vector tokens = sot_seq; + std::vector first(n_text + 1); // each text token's first row after `not` + for (int k = 0; k < n_text; ++k) { + first[k] = (int) (tokens.size() - sot_seq.size()); + if (k >= k0 && k < k1) { + tokens.insert(tokens.end(), chars[k].begin(), chars[k].end()); + } else { + tokens.push_back(text[k]->id); + } + } + first[n_text] = (int) (tokens.size() - sot_seq.size()); + tokens.push_back(eot); + + whisper_kv_cache_clear(state->kv_self); + whisper_batch_prep_legacy(state->batch, tokens.data(), tokens.size(), 0, 0); + whisper_kv_cache_seq_rm(state->kv_self, 0, 0, -1); + if (!whisper_decode_internal(*ctx, *state, state->batch, n_threads, true, nullptr, nullptr)) { + WHISPER_LOG_INFO("DECODER FAILED\n"); + WHISPER_ASSERT(0); + } + const ggml_tensor * qk = state->aheads_cross_QKs; + WHISPER_ASSERT(qk != nullptr && qk->type == GGML_TYPE_F32 && ggml_is_contiguous(qk)); + const int64_t n_tok = qk->ne[0]; + const int64_t n_actx = qk->ne[1]; + const int64_t n_head = qk->ne[2]; + const int64_t F = n_frames / 2; // 20 ms per frame + WHISPER_ASSERT(F <= n_actx); + auto & data = state->aheads_cross_QKs_data; + data.resize(n_tok * n_actx * n_head); + ggml_backend_tensor_get(qk, data.data(), 0, sizeof(float) * data.size()); + + // Rows: `not`, then every token after it but eot. Row r predicts the + // token after it, so entering row r is where that token starts. + const int64_t row0 = (int64_t) sot_seq.size() - 1; + const int64_t R = (int64_t) tokens.size() - 1 - row0; + + struct ggml_context * gctx = ggml_init(gparams); + ggml_tensor * x = ggml_new_tensor_2d(gctx, GGML_TYPE_F32, R, F); + float * xd = (float *) x->data; + for (int64_t f = 0; f < F; ++f) { + double norm = 0.0; + for (int64_t r = 0; r < R; ++r) { + float a = 0.0f; + for (int64_t h = 0; h < n_head; ++h) { + a += data[(row0 + r) + f * n_tok + h * n_tok * n_actx]; + } + xd[r + f * R] = a; + norm += (double) a * a; + } + norm = std::sqrt(norm) + 1e-9; + for (int64_t r = 0; r < R; ++r) { + xd[r + f * R] = (float) (-xd[r + f * R] / norm); + } + } + + const ggml_tensor * path = dtw_and_backtrace(gctx, x); + std::vector entry(R, 0); + for (int64_t i = path->ne[1] - 1; i >= 0; --i) { + const int32_t r = whisper_get_i32_nd(path, 0, i, 0, 0); + if (r >= 0 && r < R) entry[r] = whisper_get_i32_nd(path, 1, i, 0, 0); + } + ggml_free(gctx); + + for (int k = k0; k < k1; ++k) { + t_floor = std::max(t_floor, (int64_t) entry[first[k + 1]] * 2 + seek); + text[k]->t_dtw = t_floor; + } + k0 = k1; + } +} diff --git a/electron/preload.ts b/electron/preload.ts index 80e583f24..feb3942df 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,11 +1,22 @@ -import { contextBridge, ipcRenderer, webUtils } from "electron"; +import { contextBridge, ipcRenderer, sharedTexture, webUtils } from "electron"; import type { NativeLinuxRecordingRequest } from "../src/lib/nativeLinuxRecording"; import type { NativeMacRecordingRequest } from "../src/lib/nativeMacRecording"; import type { NativeWindowsRecordingRequest } from "../src/lib/nativeWindowsRecording"; import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession"; import type { ShortcutBinding } from "../src/lib/shortcuts"; -import type { AiEditionChatEvent } from "../src/native/contracts"; -import { NATIVE_BRIDGE_CHANNEL, type NativeBridgeRequest } from "../src/native/contracts"; +import type { + AiEditionChatEvent, + AiEditionMcpHostRequest, + AiEditionMcpHostResponse, + CompositorSharedFrameMeta, +} from "../src/native/contracts"; +import { + AI_EDITION_MCP_HOST_CHANNEL, + AI_EDITION_MCP_REQUEST_CHANNEL, + AI_EDITION_MCP_RESPONSE_CHANNEL, + NATIVE_BRIDGE_CHANNEL, + type NativeBridgeRequest, +} from "../src/native/contracts"; import type { RecordingPrefs } from "./ipc/handlers"; import type { SttStatusEvent, @@ -24,6 +35,23 @@ const assetBaseUrl = assetBaseUrlArg ? assetBaseUrlArg.slice(ASSET_BASE_URL_ARG_ // so a synchronous read here saves the renderer's every-call IPC round-trip. const PLATFORM = process.platform; +// Preview frames the main process hands over as shared GPU textures (see +// `compositorViewService`). Electron gives up on a send after 1 s without a receiver, so it is +// registered here, at load, and forwards to whichever listener the page has set. +type CompositorFrameListener = (frame: VideoFrame, meta: CompositorSharedFrameMeta) => void; +let compositorFrameListener: CompositorFrameListener | null = null; +sharedTexture?.setSharedTextureReceiver(async ({ importedSharedTexture }, meta) => { + const frame = importedSharedTexture.getVideoFrame(); + try { + compositorFrameListener?.(frame, meta as CompositorSharedFrameMeta); + } finally { + // The page drew from its own copy (contextBridge clones the frame) and closed it. Closing + // ours and releasing the import is what lets the native ring write the slot again. + frame.close(); + importedSharedTexture.release(); + } +}); + contextBridge.exposeInMainWorld("electronAPI", { assetBaseUrl, @@ -75,6 +103,14 @@ contextBridge.exposeInMainWorld("electronAPI", { ipcRenderer.on("export:native-progress", handler); return () => ipcRenderer.off("export:native-progress", handler); }, + onCompositorFrame: (listener: CompositorFrameListener) => { + compositorFrameListener = listener; + return () => { + if (compositorFrameListener === listener) { + compositorFrameListener = null; + } + }; + }, invokeNativeBridge: (request: NativeBridgeRequest) => { return ipcRenderer.invoke(NATIVE_BRIDGE_CHANNEL, request) as Promise; }, @@ -129,6 +165,9 @@ contextBridge.exposeInMainWorld("electronAPI", { usesSystemSourcePicker: (): Promise => { return ipcRenderer.invoke("uses-system-source-picker"); }, + getLastPickedSource: (): Promise => { + return ipcRenderer.invoke("get-last-picked-source"); + }, openNotes: () => { return ipcRenderer.invoke("open-notes"); }, @@ -255,6 +294,11 @@ contextBridge.exposeInMainWorld("electronAPI", { startNativeMacRecording: (request: NativeMacRecordingRequest) => { return ipcRenderer.invoke("start-native-mac-recording", request); }, + onNativeMacSystemAudioUnavailable: (callback: () => void) => { + const listener = () => callback(); + ipcRenderer.on("native-mac-system-audio-unavailable", listener); + return () => ipcRenderer.removeListener("native-mac-system-audio-unavailable", listener); + }, pauseNativeMacRecording: () => { return ipcRenderer.invoke("pause-native-mac-recording"); }, @@ -517,6 +561,33 @@ contextBridge.exposeInMainWorld("electronAPI", { ipcRenderer.on("ai-edition.chat-event", listener); return () => ipcRenderer.removeListener("ai-edition.chat-event", listener); }, + // The editor answers the MCP server's reads and writes of the live document. + // Subscribing is what makes this window the one the server asks; the + // returned unsubscribe withdraws it. See electron/mcp/editor-document-host.ts. + onAiEditionMcpRequest: ( + callback: (request: AiEditionMcpHostRequest) => Promise, + ) => { + const listener = (_e: unknown, request: AiEditionMcpHostRequest) => { + void callback(request).then( + (result) => + ipcRenderer.send(AI_EDITION_MCP_RESPONSE_CHANNEL, { + requestId: request.requestId, + result, + }), + () => + ipcRenderer.send(AI_EDITION_MCP_RESPONSE_CHANNEL, { + requestId: request.requestId, + result: null, + }), + ); + }; + ipcRenderer.on(AI_EDITION_MCP_REQUEST_CHANNEL, listener); + ipcRenderer.send(AI_EDITION_MCP_HOST_CHANNEL, true); + return () => { + ipcRenderer.removeListener(AI_EDITION_MCP_REQUEST_CHANNEL, listener); + ipcRenderer.send(AI_EDITION_MCP_HOST_CHANNEL, false); + }; + }, stt: { transcribe: (request: SttTranscribeRequest): Promise => { return ipcRenderer.invoke("stt:transcribe", request) as Promise; diff --git a/electron/recording-source-settings.test.ts b/electron/recording-source-settings.test.ts index f56a684fa..40aefc588 100644 --- a/electron/recording-source-settings.test.ts +++ b/electron/recording-source-settings.test.ts @@ -1,8 +1,10 @@ import { describe, expect, it } from "vitest"; import { + describeMacPickerSource, describeRecordingSource, enumerationIncludesSourceKind, mergeEnumeratedSources, + rememberedPickerSourceName, resolveCurrentRecordingSource, resolveRecordingSource, restoreRecordingSourceAfterEnumeration, @@ -139,3 +141,32 @@ describe("recording source settings", () => { ).toEqual(display); }); }); + +describe("the display picked in Apple's picker, kept only to be named", () => { + const picked = { id: "mac-picker:display:5", name: "Studio Display", display_id: "5" }; + + it("describes a display pick as a screen and a window pick as a window", () => { + expect(describeMacPickerSource(picked, "display")).toEqual({ + platform: "darwin", + kind: "screen", + id: "mac-picker:display:5", + name: "Studio Display", + displayId: "5", + }); + // Not derived from the id: `recordingSourceKindFromId` only knows desktopCapturer's + // `window:` prefix and would call a picked window a screen. + expect( + describeMacPickerSource( + { id: "mac-picker:window:42", name: "Notes", display_id: "" }, + "window", + ), + ).toMatchObject({ kind: "window", displayId: null }); + }); + + it("names the last pick only where Apple's picker owns the choice", () => { + const last = describeMacPickerSource(picked, "display"); + expect(rememberedPickerSourceName(last, true)).toBe("Studio Display"); + expect(rememberedPickerSourceName(last, false)).toBeNull(); + expect(rememberedPickerSourceName(null, true)).toBeNull(); + }); +}); diff --git a/electron/recording-source-settings.ts b/electron/recording-source-settings.ts index d8d89425b..2480d24a8 100644 --- a/electron/recording-source-settings.ts +++ b/electron/recording-source-settings.ts @@ -23,6 +23,36 @@ export function describeRecordingSource( }; } +/** + * What Apple's picker returned, kept only to name it after a relaunch. It is not a + * restorable descriptor: the filter behind the pick cannot leave the helper, and rebuilding + * one needs the Screen Recording grant the picker exists to avoid. + */ +export function describeMacPickerSource( + source: { id: string; name: string; display_id: string }, + kind: "display" | "window", +): RecordingSourceDescriptor { + return { + platform: "darwin", + kind: kind === "window" ? "window" : "screen", + id: source.id, + name: source.name, + displayId: source.display_id || null, + }; +} + +/** + * The name to show for the last pick while nothing is live. Only where Apple's picker owns + * the choice: everywhere else the descriptor restores a real source, so there is nothing to + * show in its place. + */ +export function rememberedPickerSourceName( + lastSource: RecordingSourceDescriptor | null | undefined, + pickerOwnsSources: boolean, +): string | null { + return pickerOwnsSources && lastSource ? lastSource.name : null; +} + /** * A screen-only (or window-only) enumeration cannot prove a source of the * other kind is gone. Clearing the live pick in that case is a false negative. diff --git a/electron/recording/displaySleepBlocker.test.ts b/electron/recording/displaySleepBlocker.test.ts new file mode 100644 index 000000000..ff420d52c --- /dev/null +++ b/electron/recording/displaySleepBlocker.test.ts @@ -0,0 +1,52 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { start, stop } = vi.hoisted(() => { + let nextId = 1; + return { start: vi.fn(() => nextId++), stop: vi.fn() }; +}); + +vi.mock("electron", () => ({ powerSaveBlocker: { start, stop } })); + +describe("setDisplaySleepBlocked", () => { + let setDisplaySleepBlocked: (recording: boolean) => void; + + beforeEach(async () => { + vi.resetModules(); + start.mockClear(); + stop.mockClear(); + ({ setDisplaySleepBlocked } = await import("./displaySleepBlocker")); + }); + + it("starts one display-sleep blocker per recording and stops that same one", () => { + setDisplaySleepBlocked(true); + expect(start).toHaveBeenCalledOnce(); + expect(start).toHaveBeenCalledWith("prevent-display-sleep"); + + setDisplaySleepBlocked(false); + expect(stop).toHaveBeenCalledOnce(); + expect(stop).toHaveBeenCalledWith(start.mock.results[0].value); + }); + + it("never stacks a second blocker when a start is reported twice", () => { + setDisplaySleepBlocked(true); + setDisplaySleepBlocked(true); + setDisplaySleepBlocked(false); + expect(start).toHaveBeenCalledOnce(); + expect(stop).toHaveBeenCalledOnce(); + }); + + it("ignores a stop with nothing started, as a failed start then cancel reports", () => { + setDisplaySleepBlocked(false); + expect(stop).not.toHaveBeenCalled(); + }); + + it("leaves nothing running across consecutive takes", () => { + for (let take = 0; take < 3; take++) { + setDisplaySleepBlocked(true); + setDisplaySleepBlocked(false); + setDisplaySleepBlocked(false); + } + expect(start).toHaveBeenCalledTimes(3); + expect(stop.mock.calls.map(([id]) => id)).toEqual(start.mock.results.map((r) => r.value)); + }); +}); diff --git a/electron/recording/displaySleepBlocker.ts b/electron/recording/displaySleepBlocker.ts new file mode 100644 index 000000000..c58407493 --- /dev/null +++ b/electron/recording/displaySleepBlocker.ts @@ -0,0 +1,18 @@ +import { powerSaveBlocker } from "electron"; + +let blockerId: number | null = null; + +/** + * Keeps the display awake for the whole take, pauses included (#935). Driven by the + * same recording flag as the tray, which every backend's start and every stop, + * cancel and failure path already report, so it is idempotent in both directions: + * a repeated `true` never stacks a second blocker, a stray `false` is a no-op. + */ +export function setDisplaySleepBlocked(recording: boolean) { + if (recording && blockerId === null) { + blockerId = powerSaveBlocker.start("prevent-display-sleep"); + } else if (!recording && blockerId !== null) { + powerSaveBlocker.stop(blockerId); + blockerId = null; + } +} diff --git a/electron/recording/nativeWindowsCaptureStop.test.ts b/electron/recording/nativeWindowsCaptureStop.test.ts index 68ebf01c9..0def7936f 100644 --- a/electron/recording/nativeWindowsCaptureStop.test.ts +++ b/electron/recording/nativeWindowsCaptureStop.test.ts @@ -7,8 +7,13 @@ import { NATIVE_WINDOWS_SALVAGEABLE_OUTPUT_BYTES, readMicrophoneDefaulted, readMicrophoneUnavailable, + readReportedWebcamPaths, + readSecondaryWindowsApplied, readStoppedPath, + readStoppedWebcamPaths, + readUnavailableWebcamIndices, readWebcamFormat, + readWebcamFormatAt, readWebcamUnavailable, terminateNativeWindowsCapture, waitForNativeWindowsCaptureStop, @@ -125,6 +130,27 @@ describe("readMicrophoneDefaulted", () => { }); }); +describe("readSecondaryWindowsApplied", () => { + it("is true when the helper drew menus and popups into the capture", () => { + const output = + '{"event":"cursor-capture","schemaVersion":2,"requested":false,"applied":false}\n' + + '{"event":"secondary-windows","applied":true}\n' + + "Recording started\n"; + expect(readSecondaryWindowsApplied(output)).toBe(true); + }); + + it("is false on a runtime older than Windows 11 24H2", () => { + expect(readSecondaryWindowsApplied('{"event":"secondary-windows","applied":false}\n')).toBe( + false, + ); + }); + + it("is null when the helper said nothing, as for a monitor source", () => { + const output = '{"event":"cursor-capture","applied":true}\nRecording started\n'; + expect(readSecondaryWindowsApplied(output)).toBe(null); + }); +}); + describe("readMicrophoneUnavailable", () => { // The line ScreenCaptureRecorder.swift emits on macOS 13 and 14, which have no // `captureMicrophone`, before recording the take without it (#700). @@ -517,3 +543,75 @@ describe("terminateNativeWindowsCapture", () => { expect(helper.killCalls).toBe(1); }); }); + +describe("per-camera helper events", () => { + const unavailable = (i?: number) => + `{"event":"warning","code":"webcam-unavailable"${i === undefined ? "" : `,"index":${i}`},"message":"x"}`; + + it("collects unavailable cameras by index, an old event counting as camera 0", () => { + expect(readUnavailableWebcamIndices(`${unavailable(1)}\n${unavailable(3)}`)).toEqual([1, 3]); + expect(readUnavailableWebcamIndices(unavailable())).toEqual([0]); + expect(readUnavailableWebcamIndices("nothing")).toEqual([]); + }); + + it("ignores other warnings and reads a prefixed unavailable event", () => { + const other = '{"event":"warning","code":"microphone-defaulted","index":2}'; + const prefixed = `INFO: camera lost ${unavailable(2)}`; + expect(readUnavailableWebcamIndices(`${other}\n${prefixed}`)).toEqual([2]); + }); + + it("reads the format of a given camera", () => { + const out = [ + '{"event":"webcam-format","schemaVersion":2,"index":0,"width":1920,"height":1080,"fps":30,"deviceName":"A"}', + '{"event":"webcam-format","schemaVersion":2,"index":1,"width":1280,"height":720,"fps":30,"deviceName":"B"}', + ].join("\n"); + expect(readWebcamFormatAt(out, 1)?.width).toBe(1280); + expect(readWebcamFormatAt(out, 0)?.width).toBe(1920); + expect(readWebcamFormatAt(out, 2)).toBeNull(); + }); + + it("reads a format event glued to a diagnostic prefix, the last one winning", () => { + const out = [ + 'INFO: DirectShow webcam connected subtype NV12 {"event":"webcam-format","index":1,"width":640,"height":480}', + 'INFO: again {"event":"webcam-format","index":1,"width":1280,"height":720}', + ].join("\n"); + expect(readWebcamFormatAt(out, 1)?.width).toBe(1280); + }); + + it("treats a format without index as camera 0", () => { + expect(readWebcamFormatAt('{"event":"webcam-format","width":800}', 0)?.width).toBe(800); + }); + + it("reads every stopped camera path, falling back to the single legacy path", () => { + expect( + readStoppedWebcamPaths( + '{"event":"recording-stopped","webcamPath":"a.mp4","webcamPaths":["a.mp4","b.mp4"]}', + ), + ).toEqual(["a.mp4", "b.mp4"]); + expect(readStoppedWebcamPaths('{"event":"recording-stopped","webcamPath":"a.mp4"}')).toEqual([ + "a.mp4", + ]); + expect(readStoppedWebcamPaths('{"event":"recording-stopped"}')).toEqual([]); + }); + + it("tells a reported webcamPaths list apart from its absence", () => { + expect( + readReportedWebcamPaths('{"event":"recording-stopped","webcamPaths":["a.mp4","b.mp4"]}'), + ).toEqual(["a.mp4", "b.mp4"]); + expect(readReportedWebcamPaths('{"event":"recording-stopped","webcamPaths":[]}')).toEqual([]); + // An old helper sends only the legacy path; that is "unknown", not "none". + expect( + readReportedWebcamPaths('{"event":"recording-stopped","webcamPath":"a.mp4"}'), + ).toBeNull(); + expect(readReportedWebcamPaths("Recording stopped. Output path: s.mp4")).toBeNull(); + }); + + it("reads JSON-escaped Windows paths behind a prefix", () => { + const line = + 'INFO: x {"event":"recording-stopped","schemaVersion":2,"screenPath":"C:\\\\Users\\\\me\\\\s.mp4","webcamPath":"C:\\\\Users\\\\me\\\\s-webcam.mp4","webcamPaths":["C:\\\\Users\\\\me\\\\s-webcam.mp4","C:\\\\Users\\\\me\\\\s-webcam-2.mp4"]}'; + expect(readStoppedWebcamPaths(line)).toEqual([ + "C:\\Users\\me\\s-webcam.mp4", + "C:\\Users\\me\\s-webcam-2.mp4", + ]); + }); +}); diff --git a/electron/recording/nativeWindowsCaptureStop.ts b/electron/recording/nativeWindowsCaptureStop.ts index 9f2e2db33..05c2a0f1f 100644 --- a/electron/recording/nativeWindowsCaptureStop.ts +++ b/electron/recording/nativeWindowsCaptureStop.ts @@ -122,6 +122,20 @@ export function readMicrophoneDefaulted(output: string) { return output.includes('"code":"microphone-defaulted"'); } +/** + * Did the helper draw menus and popups into a window recording? + * + * `true` on Windows 11 24H2+, `false` where the runtime cannot (the recording is + * the window alone, as before), `null` when the helper never said: a monitor + * source, the kill switch, or an older helper (getopenscreen/openscreen#894). + */ +export function readSecondaryWindowsApplied(output: string) { + if (output.includes('"event":"secondary-windows","applied":true')) { + return true; + } + return output.includes('"event":"secondary-windows","applied":false') ? false : null; +} + /** * Did the macOS helper record without the microphone it was asked for? * @@ -204,6 +218,93 @@ export function readWebcamFormat(output: string) { } } +type HelperEvent = Record; + +/** + * Every `{"event":""…}` object in the output, in order. + * + * Locates each object start and slices it with `findObjectEnd` instead of + * parsing whole lines, because diagnostics can be glued in front of an event + * (see `readWebcamFormat`). Slices that do not parse are skipped. + */ +function readHelperEvents(output: string, name: string): HelperEvent[] { + const needle = `{"event":"${name}"`; + const events: HelperEvent[] = []; + let from = 0; + for (;;) { + const start = output.indexOf(needle, from); + if (start === -1) { + return events; + } + const end = findObjectEnd(output, start); + if (end === -1) { + return events; + } + from = end + 1; + try { + const parsed: unknown = JSON.parse(output.slice(start, end + 1)); + if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) { + events.push(parsed as HelperEvent); + } + } catch { + // Not a complete JSON object; keep looking. + } + } +} + +/** A helper event's camera index; an event without one comes from an old helper (camera 0). */ +function eventCameraIndex(event: HelperEvent) { + return typeof event.index === "number" ? event.index : 0; +} + +/** Indices of every camera the helper reported as unavailable (no index = camera 0). */ +export function readUnavailableWebcamIndices(output: string): number[] { + return readHelperEvents(output, "warning") + .filter((event) => event.code === "webcam-unavailable") + .map(eventCameraIndex); +} + +/** The last `webcam-format` event of the given camera (no index = camera 0), or null. */ +export function readWebcamFormatAt( + output: string, + index: number, +): ReturnType { + const match = readHelperEvents(output, "webcam-format") + .filter((event) => eventCameraIndex(event) === index) + .at(-1); + return (match as ReturnType) ?? null; +} + +/** + * The `webcamPaths` list of the last `recording-stopped` event, or null when + * there is no such event or it carried no `webcamPaths` key. + * + * Null is the signal that matters: only a helper that sends the key says which + * cameras were still recording at stop, so only then may a camera missing from + * it be called one that stopped early. An old helper, or a stop without the + * event, must read as "unknown", never as "every camera stopped". + */ +export function readReportedWebcamPaths(output: string): string[] | null { + const event = readHelperEvents(output, "recording-stopped").at(-1); + if (!event || !Array.isArray(event.webcamPaths)) { + return null; + } + return event.webcamPaths.filter((entry): entry is string => typeof entry === "string"); +} + +/** + * Camera files the helper reported at stop: `webcamPaths`, else the single + * legacy `webcamPath`, else none. + */ +export function readStoppedWebcamPaths(output: string): string[] { + const reported = readReportedWebcamPaths(output); + if (reported) { + return reported; + } + const event = readHelperEvents(output, "recording-stopped").at(-1); + return typeof event?.webcamPath === "string" && event.webcamPath ? [event.webcamPath] : []; +} + /** * The most useful line of a failed helper run, for a toast. * diff --git a/electron/recording/nativeWindowsWebcams.test.ts b/electron/recording/nativeWindowsWebcams.test.ts new file mode 100644 index 000000000..f7187851c --- /dev/null +++ b/electron/recording/nativeWindowsWebcams.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, it } from "vitest"; +import { + additionalWebcamLabels, + buildHelperWebcamConfig, + collectStoppedWebcams, + dedupeAdditionalWebcams, + isWebcamSidecarFile, + labelsOfUnavailableAdditionalWebcams, + labelsOfWebcamsStoppedEarly, + stripWebcamSuffix, + webcamOutputPath, +} from "./nativeWindowsWebcams"; + +describe("nativeWindowsWebcams", () => { + it("names camera files", () => { + expect(webcamOutputPath("C:\\r", "rec-", 7, 1)).toMatch(/rec-7-webcam\.mp4$/); + expect(webcamOutputPath("C:\\r", "rec-", 7, 3)).toMatch(/rec-7-webcam-3\.mp4$/); + }); + + it("recognizes every camera file of a recording and nothing else", () => { + for (const f of ["rec-7-webcam.mp4", "rec-7-webcam-2.mp4", "rec-7-webcam-4.webm"]) { + expect(isWebcamSidecarFile(f)).toBe(true); + } + for (const f of ["rec-7.mp4", "rec-7-webcamera.mp4", "rec-7-webcam-x.mp4"]) { + expect(isWebcamSidecarFile(f)).toBe(false); + } + expect(stripWebcamSuffix("rec-7-webcam-2")).toBe("rec-7"); + expect(stripWebcamSuffix("rec-7-webcam")).toBe("rec-7"); + expect(stripWebcamSuffix("rec-7")).toBe("rec-7"); + }); + + it("dedupes a device that equals camera 1, duplicates, and caps at three", () => { + const extras = [ + { deviceId: "a", deviceName: "Front" }, + { deviceId: "b", deviceName: "Desk" }, + { deviceId: "b", deviceName: "Desk" }, + { deviceName: "Side" }, + { deviceName: "Top" }, + { deviceName: "Fifth" }, + ]; + expect( + dedupeAdditionalWebcams({ deviceId: "a", deviceName: "Front" }, extras).map( + (e) => e.deviceName, + ), + ).toEqual(["Desk", "Side", "Top"]); + }); + + it("drops extras that name no device at all", () => { + expect(dedupeAdditionalWebcams(null, [{ deviceName: " " }, { deviceName: "Desk" }])).toEqual([ + { deviceName: "Desk" }, + ]); + }); + + it("skips malformed entries from IPC", () => { + const junk = [ + null, + { deviceName: 3 }, + { deviceId: 4, deviceName: "X" }, + { deviceName: "Desk" }, + ]; + expect(dedupeAdditionalWebcams(null, junk as unknown as Array<{ deviceName: string }>)).toEqual( + [{ deviceName: "Desk" }], + ); + }); + + it("never matches an extra with an id to an id-less camera 1 by name", () => { + // Two cameras of the same model: same name, and camera 1 came without an id. + expect( + dedupeAdditionalWebcams({ deviceName: "USB Camera" }, [ + { deviceId: "b", deviceName: "USB Camera" }, + ]), + ).toEqual([{ deviceId: "b", deviceName: "USB Camera" }]); + // Without an id on either side the name is all there is, and it still matches. + expect( + dedupeAdditionalWebcams({ deviceName: "USB Camera" }, [{ deviceName: "USB Camera" }]), + ).toEqual([]); + }); + + describe("labelsOfWebcamsStoppedEarly", () => { + const front = String.raw`C:\Rec\r-webcam.mp4`; + const desk = String.raw`C:\Rec\r-webcam-2.mp4`; + const side = String.raw`C:\Rec\r-webcam-3.mp4`; + const requested = [ + { path: front, label: "Front" }, + { path: desk, label: "Desk" }, + { path: side, label: "Side" }, + ]; + const sizes = new Map([ + [front, 100], + [desk, 100], + [side, 0], + ]); + + it("names a kept camera the helper no longer listed, camera 1 included", () => { + expect( + labelsOfWebcamsStoppedEarly({ + requested, + sizes, + helperWebcamPaths: [desk], + }), + ).toEqual(["Front"]); + }); + + it("leaves out a camera whose file was not kept: that one was not recorded", () => { + expect(labelsOfWebcamsStoppedEarly({ requested, sizes, helperWebcamPaths: [] })).toEqual([ + "Front", + "Desk", + ]); + }); + + it("flags nothing without a webcamPaths key", () => { + expect(labelsOfWebcamsStoppedEarly({ requested, sizes, helperWebcamPaths: null })).toEqual( + [], + ); + }); + + it("matches paths regardless of case and separator", () => { + expect( + labelsOfWebcamsStoppedEarly({ + requested, + sizes, + helperWebcamPaths: ["c:/rec/R-WEBCAM.mp4", String.raw`c:\REC//r-webcam-2.MP4`], + }), + ).toEqual([]); + }); + }); + + it("drops an empty additional camera file and names it", () => { + const r = collectStoppedWebcams({ + camera1Enabled: true, + requested: [ + { path: "w.mp4", label: "Front" }, + { path: "w-2.mp4", label: "Desk" }, + { path: "w-3.mp4", label: "Side" }, + ], + sizes: new Map([ + ["w.mp4", 100], + ["w-2.mp4", 0], + ["w-3.mp4", 50], + ]), + }); + expect(r).toEqual({ + camera1: "w.mp4", + additional: [{ path: "w-3.mp4", label: "Side" }], + dropped: ["Desk"], + }); + }); + + it("keeps camera 1 and reports a camera whose file never appeared as not recorded", () => { + const r = collectStoppedWebcams({ + camera1Enabled: true, + requested: [ + { path: "w.mp4", label: "Front" }, + { path: "w-2.mp4", label: "Desk" }, + ], + sizes: new Map([["w.mp4", 100]]), + }); + expect(r).toEqual({ camera1: "w.mp4", additional: [], dropped: ["Desk"] }); + }); + + it("keeps extras when camera 1 is lost, leaving camera 1 out of dropped", () => { + const r = collectStoppedWebcams({ + camera1Enabled: true, + requested: [ + { path: "w.mp4", label: "Front" }, + { path: "w-2.mp4", label: "Desk" }, + ], + sizes: new Map([ + ["w.mp4", 0], + ["w-2.mp4", 10], + ]), + }); + expect(r).toEqual({ additional: [{ path: "w-2.mp4", label: "Desk" }], dropped: [] }); + }); + + it("maps unavailable helper indices to the labels of the extras", () => { + const requested = [ + { path: "w.mp4", label: "Front" }, + { path: "w-2.mp4", label: "Desk" }, + { path: "w-3.mp4", label: "Side" }, + ]; + expect(labelsOfUnavailableAdditionalWebcams(requested, [0, 2, 2, 9])).toEqual(["Side"]); + }); + + it("labels extras by device name, else Camera ", () => { + expect(additionalWebcamLabels("Front", [{ deviceName: "Desk" }, { deviceName: " " }])).toEqual( + ["Desk", "Camera 3"], + ); + }); + + it("tells cameras with the same name apart by occurrence, counting camera 1", () => { + expect( + additionalWebcamLabels("USB Camera", [ + { deviceName: "USB Camera" }, + { deviceName: "Desk" }, + { deviceName: " USB Camera " }, + ]), + ).toEqual(["USB Camera (2)", "Desk", "USB Camera (3)"]); + expect( + additionalWebcamLabels(undefined, [ + { deviceName: "USB Camera" }, + { deviceName: "USB Camera" }, + ]), + ).toEqual(["USB Camera", "USB Camera (2)"]); + }); + + it("builds a start config with the legacy fields and a list of every camera", () => { + const config = buildHelperWebcamConfig({ + camera1: { + enabled: true, + deviceId: "id-1", + deviceName: "Front", + width: 1280, + height: 720, + fps: 30, + }, + camera1Clsid: "{c1}", + camera1Path: "C:\\r\\rec-7-webcam.mp4", + extras: [ + { deviceId: "id-2", deviceName: "Desk", clsid: null, path: "C:\\r\\rec-7-webcam-2.mp4" }, + ], + }); + const parsed = JSON.parse(JSON.stringify(config)); + expect(parsed).toMatchObject({ + webcamEnabled: true, + webcamDeviceId: "id-1", + webcamDeviceName: "Front", + webcamDirectShowClsid: "{c1}", + webcamWidth: 1280, + webcamHeight: 720, + webcamFps: 30, + }); + expect(parsed.webcams).toEqual([ + { + camDeviceId: "id-1", + camDeviceName: "Front", + camClsid: "{c1}", + camWidth: 1280, + camHeight: 720, + camFps: 30, + camPath: "C:\\r\\rec-7-webcam.mp4", + }, + { + camDeviceId: "id-2", + camDeviceName: "Desk", + camClsid: null, + camWidth: 1280, + camHeight: 720, + camFps: 30, + camPath: "C:\\r\\rec-7-webcam-2.mp4", + }, + ]); + }); + + it("sends no camera list while camera 1 is off", () => { + const config = buildHelperWebcamConfig({ + camera1: { enabled: false, width: 1280, height: 720, fps: 30 }, + camera1Clsid: null, + camera1Path: "C:\\r\\rec-7-webcam.mp4", + extras: [{ deviceName: "Desk", clsid: null, path: "C:\\r\\rec-7-webcam-2.mp4" }], + }); + expect(config.webcamEnabled).toBe(false); + expect(config.webcams).toEqual([]); + }); +}); diff --git a/electron/recording/nativeWindowsWebcams.ts b/electron/recording/nativeWindowsWebcams.ts new file mode 100644 index 000000000..9c8b462b5 --- /dev/null +++ b/electron/recording/nativeWindowsWebcams.ts @@ -0,0 +1,269 @@ +/** + * Pure helpers for recording several cameras with the native Windows helper. + * + * Camera 1 stays exactly what it always was: the helper's legacy `webcam*` + * fields, the file `-webcam.mp4` and the session's + * `webcamVideoPath`. Cameras 2-4 ride along in the helper's `webcams` list and + * in the session's `additionalWebcams`. Kept out of `electron/ipc/handlers.ts` + * because that module cannot be loaded from a test. + */ +import path from "node:path"; +import { type AdditionalWebcam, MAX_ADDITIONAL_WEBCAMS } from "../../src/lib/recordingSession"; + +/** `-webcam` (camera 1) or `-webcam-<2..9>` at the end of a file's base name. */ +const WEBCAM_SUFFIX = /-webcam(?:-[2-9])?$/; + +/** Camera 1 → `-webcam.mp4`, camera n ≥ 2 → `-webcam-.mp4`. */ +export function webcamOutputPath( + dir: string, + prefix: string, + recordingId: number, + cameraNumber: number, +): string { + const suffix = cameraNumber <= 1 ? "-webcam" : `-webcam-${cameraNumber}`; + return path.join(dir, `${prefix}${recordingId}${suffix}.mp4`); +} + +/** Whether a file name is one of a recording's camera files (any extension). */ +export function isWebcamSidecarFile(fileName: string): boolean { + return WEBCAM_SUFFIX.test(path.parse(fileName).name); +} + +/** The recording's base name for a camera file's base name; other names pass through. */ +export function stripWebcamSuffix(baseName: string): string { + return baseName.replace(WEBCAM_SUFFIX, ""); +} + +/** One camera in the helper config's `webcams` list (keys are the helper's). */ +export interface HelperWebcamEntry { + camDeviceId: string | null; + camDeviceName: string; + camClsid: string | null; + camWidth: number; + camHeight: number; + camFps: number; + camPath: string; +} + +type DeviceRef = { deviceId?: string; deviceName?: string }; + +/** + * Same device: by id when both sides carry one, otherwise by name — but only + * when neither carries an id. Two webcams of the same model share a name, so an + * id on one side and none on the other says nothing about whether they are the + * same device, and matching by name there would silently drop the second one. + */ +function isSameDevice(a: DeviceRef, b: DeviceRef) { + if (a.deviceId && b.deviceId) { + return a.deviceId === b.deviceId; + } + if (a.deviceId || b.deviceId) { + return false; + } + const name = a.deviceName?.trim(); + return Boolean(name) && name === b.deviceName?.trim(); +} + +/** + * The additional cameras worth asking the helper for: no malformed entry, no + * entry that names no device, none equal to camera 1, no duplicates, and at most + * {@link MAX_ADDITIONAL_WEBCAMS}. + */ +export function dedupeAdditionalWebcams( + camera1: DeviceRef | null, + extras: T[], +): T[] { + const kept: T[] = []; + for (const extra of extras) { + if (kept.length >= MAX_ADDITIONAL_WEBCAMS) { + break; + } + // The list crosses IPC, so a malformed entry is skipped rather than trusted. + if ( + !extra || + typeof extra.deviceName !== "string" || + (extra.deviceId !== undefined && typeof extra.deviceId !== "string") + ) { + continue; + } + if (!extra.deviceId && !extra.deviceName.trim()) { + continue; + } + if (camera1 && isSameDevice(camera1, extra)) { + continue; + } + if (kept.some((other) => isSameDevice(other, extra))) { + continue; + } + kept.push(extra); + } + return kept; +} + +/** + * Labels of the additional cameras, in order: the device name, else + * `Camera ` (n counts camera 1, so the first extra is camera 2). + * + * Two webcams of the same model report the same name, and a label is all the + * user is told about a camera that could not be opened or was dropped. So a + * label already used — by camera 1 or an earlier extra — gets " (2)", " (3)" + * by occurrence. Camera 1's own label is never changed. + */ +export function additionalWebcamLabels( + camera1Name: string | undefined, + extras: Array<{ deviceName: string }>, +): string[] { + const seen = new Map(); + const camera1Label = camera1Name?.trim(); + if (camera1Label) { + seen.set(camera1Label, 1); + } + return extras.map((extra, i) => { + const label = extra.deviceName.trim() || `Camera ${i + 2}`; + const occurrence = (seen.get(label) ?? 0) + 1; + seen.set(label, occurrence); + return occurrence > 1 ? `${label} (${occurrence})` : label; + }); +} + +/** + * The camera part of the helper config: the unchanged legacy `webcam*` fields + * for camera 1 plus the `webcams` list (camera 1 first, then the extras). + * + * Extras are recorded only while camera 1 is on (the HUD's camera toggle + * governs every camera), so with camera 1 off the list is empty. Every extra + * uses camera 1's requested size and rate: the quality setting is one for all. + */ +export function buildHelperWebcamConfig(input: { + camera1: { + enabled: boolean; + deviceId?: string; + deviceName?: string; + width: number; + height: number; + fps: number; + }; + camera1Clsid: string | null; + camera1Path: string; + extras: Array<{ deviceId?: string; deviceName: string; clsid: string | null; path: string }>; +}) { + const { camera1 } = input; + const entry = ( + device: { deviceId?: string; deviceName?: string }, + clsid: string | null, + camPath: string, + ): HelperWebcamEntry => ({ + camDeviceId: device.deviceId ?? null, + camDeviceName: device.deviceName ?? "", + camClsid: clsid, + camWidth: camera1.width, + camHeight: camera1.height, + camFps: camera1.fps, + camPath, + }); + const webcams: HelperWebcamEntry[] = camera1.enabled + ? [ + entry(camera1, input.camera1Clsid, input.camera1Path), + ...input.extras.map((extra) => entry(extra, extra.clsid, extra.path)), + ] + : []; + return { + webcamEnabled: camera1.enabled, + webcamDeviceId: camera1.deviceId ?? null, + webcamDeviceName: camera1.deviceName ?? null, + webcamDirectShowClsid: input.camera1Clsid, + webcamWidth: camera1.width, + webcamHeight: camera1.height, + webcamFps: camera1.fps, + webcams, + }; +} + +/** + * Labels of the additional cameras the helper reported unavailable. The + * helper's index is the position in the `webcams` list, which is the position + * in `requested` (camera 1 first). Index 0 is camera 1 and is left out: the + * caller reports camera 1 through its own `webcamUnavailable` flag. The labels + * come from our own list because the helper's `deviceName` can be empty. + */ +export function labelsOfUnavailableAdditionalWebcams( + requested: Array<{ label: string }>, + indices: number[], +): string[] { + const labels: string[] = []; + for (const index of new Set(indices)) { + const camera = index > 0 ? requested[index] : undefined; + if (camera) { + labels.push(camera.label); + } + } + return labels; +} + +/** + * Which requested cameras made it into the take. + * + * A camera is kept when its file exists with size > 0. The helper's own list + * at stop is deliberately not consulted: it names the cameras still recording + * at that moment, so a camera that died mid-take with a playable partial file + * would be missing from it, and an old helper names only camera 1. A file that + * never appeared is simply absent from `sizes`. + * + * `camera1Enabled` says whether `requested[0]` is camera 1. Camera 1 comes back + * as `camera1` (undefined when lost) and is never in `dropped`, which names + * only the additional cameras not kept — the caller reports camera 1 through + * its own `webcamDropped` flag. + */ +export function collectStoppedWebcams(input: { + camera1Enabled: boolean; + requested: Array<{ path: string; label: string }>; + sizes: Map; +}): { camera1?: string; additional: AdditionalWebcam[]; dropped: string[] } { + const isKept = (filePath: string) => (input.sizes.get(filePath) ?? 0) > 0; + const [first, ...rest] = input.requested; + const camera1 = input.camera1Enabled && first && isKept(first.path) ? first.path : undefined; + const extras = input.camera1Enabled ? rest : input.requested; + const additional: AdditionalWebcam[] = []; + const dropped: string[] = []; + for (const camera of extras) { + if (isKept(camera.path)) { + additional.push({ path: camera.path, label: camera.label }); + } else { + dropped.push(camera.label); + } + } + return { ...(camera1 ? { camera1 } : {}), additional, dropped }; +} + +/** A path compared the way Windows does: case-insensitive, either separator. */ +function comparablePath(filePath: string) { + return filePath.replace(/[\\/]+/g, "\\").toLowerCase(); +} + +/** + * Labels of the cameras that stopped early: their file was kept (size > 0, the + * same test as {@link collectStoppedWebcams}) but the helper did not list it in + * `recording-stopped.webcamPaths`, the cameras still recording at stop. That is + * a camera the helper disabled mid-take; its partial file stays in the take, + * and the user is told which camera it was. + * + * `helperWebcamPaths` is null when the event carried no `webcamPaths` key (an + * old helper, or no event at all) — then nothing is known and nothing is + * flagged. Camera 1 counts like any other, so its entry needs a real label. + */ +export function labelsOfWebcamsStoppedEarly(input: { + requested: Array<{ path: string; label: string }>; + sizes: Map; + helperWebcamPaths: string[] | null; +}): string[] { + if (!input.helperWebcamPaths) { + return []; + } + const stillRecording = new Set(input.helperWebcamPaths.map(comparablePath)); + return input.requested + .filter( + (camera) => + (input.sizes.get(camera.path) ?? 0) > 0 && !stillRecording.has(comparablePath(camera.path)), + ) + .map((camera) => camera.label); +} diff --git a/electron/stt/ctcAlign.test.ts b/electron/stt/ctcAlign.test.ts new file mode 100644 index 000000000..579f97a37 --- /dev/null +++ b/electron/stt/ctcAlign.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it } from "vitest"; +import { + type AlignableWord, + alignWordsOnEmissions, + type CtcEmissions, + ctcViterbi, + END_OFFSET_SEC, + emissionRegions, + PAUSE_END_OFFSET_SEC, + PAUSE_START_OFFSET_SEC, + parseEmissions, + START_OFFSET_SEC, + wordTokens, +} from "./ctcAlign"; + +const VOCAB = ["", "|", "a", "b", "c", "é", "'"]; +const index = new Map(VOCAB.map((t, i) => [t, i])); +const id = (t: string) => index.get(t) as number; + +/** Log-probs where frame t is sure of `tokens[t]` (a vocab entry, "" for blank). */ +function logprobs(tokens: string[]): Float32Array { + const V = VOCAB.length; + const out = new Float32Array(tokens.length * V).fill(-12); + tokens.forEach((t, f) => { + out[f * V + (t ? id(t) : 0)] = -0.01; + }); + return out; +} + +const STRIDE = 0.02; +const emissions = (tokens: string[], startSec = 0): CtcEmissions => ({ + vocab: VOCAB, + blank: 0, + strideSec: STRIDE, + regions: [{ startSec, frames: tokens.length, logprobs: logprobs(tokens) }], +}); +/** `n` blank frames. */ +const gap = (n: number) => Array(n).fill(""); +const word = ( + text: string, + startSec: number, + endSec: number, + anchorSec = startSec, +): AlignableWord => ({ + word: text, + startSec, + endSec, + anchorSec, +}); +const r = (x: number) => Math.round(x * 1000) / 1000; + +describe("wordTokens", () => { + it("spells a word in the vocabulary's case, dropping silent punctuation", () => { + const upper = new Map(["", "|", "A", "B", "C", "'"].map((t, i) => [t, i])); + expect(wordTokens("Cab,", upper)).toEqual([4, 2, 3]); + expect(wordTokens("ab’c", upper)).toEqual([2, 3, 5, 4]); + expect(wordTokens("cab", index)).toEqual([id("c"), id("a"), id("b")]); + }); + + it("keeps an accented letter the vocabulary has and strips the accent otherwise", () => { + expect(wordTokens("É", index)).toEqual([id("é")]); + expect(wordTokens("àbc", index)).toEqual([id("a"), id("b"), id("c")]); + }); + + it("gives nothing to say for punctuation, and no spelling for digits and symbols", () => { + expect(wordTokens("?", index)).toEqual([]); + expect(wordTokens("...", index)).toEqual([]); + expect(wordTokens("1080p", index)).toBeNull(); + expect(wordTokens("€", index)).toBeNull(); + expect(wordTokens("日本", index)).toBeNull(); + }); +}); + +describe("ctcViterbi", () => { + it("finds the frames each label is spoken on", () => { + const tokens = ["", "a", "a", "", "b", ""]; + expect(ctcViterbi(logprobs(tokens), 6, VOCAB.length, 0, [id("a"), id("b")])).toEqual([ + [1, 2], + [4, 4], + ]); + }); + + it("needs a blank between two repeats of a label, and fails when the frames are too few", () => { + const aba = logprobs(["a", "", "a"]); + expect(ctcViterbi(aba, 3, VOCAB.length, 0, [id("a"), id("a")])).toEqual([ + [0, 0], + [2, 2], + ]); + expect(ctcViterbi(logprobs(["a", "a"]), 2, VOCAB.length, 0, [id("a"), id("a")])).toBeNull(); + }); + + it("scores a wildcard as whatever is spoken", () => { + const spans = ctcViterbi(logprobs(["a", "", "c", "c", "", "b"]), 6, VOCAB.length, 0, [ + id("a"), + -1, + id("b"), + ]); + expect(spans?.[1]).toEqual([2, 3]); + }); +}); + +describe("emissionRegions", () => { + it("pads each stretch of speech and merges the ones that touch", () => { + expect( + emissionRegions( + [ + { startSec: 0.1, endSec: 1 }, + { startSec: 1.4, endSec: 2 }, + { startSec: 5, endSec: 6 }, + ], + 6.1, + ).map(([a, b]) => [r(a), r(b)]), + ).toEqual([ + [0, 2.3], + [4.7, 6.1], + ]); + }); +}); + +describe("alignWordsOnEmissions", () => { + const speech = [{ startSec: 0.2, endSec: 1 }]; + + it("times words on their letters, calibrated outward, meeting mid-way inside speech", () => { + // "ab" on frames 20-21, the delimiter on 22, "c" on 24: continuous speech. + const tokens = [...gap(20), "a", "b", "|", "", "c", ...gap(40)]; + const out = alignWordsOnEmissions( + [word("ab", 0.3, 0.5, 0.45), word("c", 0.5, 0.7, 0.6)], + speech, + emissions(tokens), + ); + const abEnd = 22 * STRIDE + END_OFFSET_SEC; + const cStart = 24 * STRIDE + START_OFFSET_SEC; + const boundary = abEnd > cStart ? (abEnd + cStart) / 2 : null; + expect(r(out[0].startSec)).toBe(r(20 * STRIDE + PAUSE_START_OFFSET_SEC)); + expect(r(out[0].endSec)).toBe(r(boundary ?? abEnd)); + expect(r(out[1].startSec)).toBe(r(boundary ?? cStart)); + expect(r(out[1].endSec)).toBe(r(25 * STRIDE + PAUSE_END_OFFSET_SEC)); + expect(out[0].anchorSec).toBe(0.45); + }); + + it("keeps a pause between two words as a pause", () => { + // "a" on frame 12, "c" on frame 40: 0.54 s apart. + const tokens = [...gap(12), "a", ...gap(27), "c", ...gap(40)]; + const out = alignWordsOnEmissions( + [word("a", 0.2, 0.4, 0.3), word("c", 0.4, 0.9, 0.85)], + speech, + emissions(tokens), + ); + expect(r(out[0].endSec)).toBe(r(13 * STRIDE + PAUSE_END_OFFSET_SEC)); + expect(r(out[1].startSec)).toBe(r(40 * STRIDE + PAUSE_START_OFFSET_SEC)); + }); + + it("gives a word it cannot spell the speech between its neighbours", () => { + const tokens = [...gap(15), "a", "|", "c", "b", "c", "|", "b", ...gap(40)]; + const out = alignWordsOnEmissions( + [word("a", 0.3, 0.32, 0.31), word("42", 0.32, 0.4, 0.35), word("b", 0.4, 0.5, 0.42)], + speech, + emissions(tokens), + ); + expect(out[1].startSec).toBeGreaterThanOrEqual(out[0].endSec); + expect(out[1].endSec).toBeLessThanOrEqual(out[2].startSec); + expect(r(out[1].startSec)).toBeLessThan(0.34); + expect(r(out[2].startSec)).toBeLessThan(0.42); + }); + + it("puts punctuation on the end of the word before it", () => { + const tokens = [...gap(15), "a", "b", ...gap(40)]; + const out = alignWordsOnEmissions( + [word("ab", 0.3, 0.5, 0.35), word("!", 0.5, 0.6, 0.55)], + speech, + emissions(tokens), + ); + expect(out[1].startSec).toBe(out[0].endSec); + expect(out[1].endSec).toBe(out[0].endSec); + }); + + it("aligns a stretch that runs to the end of the upload, past the last frame", () => { + // 1 s of audio gives 49 frames (0.98 s): the stretch ends after them. + const tokens = [...gap(30), "a", "b", ...gap(17)]; + const out = alignWordsOnEmissions( + [word("ab", 0.65, 1, 0.7)], + [{ startSec: 0.5, endSec: 1 }], + emissions(tokens), + ); + expect(r(out[0].startSec)).toBe(r(30 * STRIDE + PAUSE_START_OFFSET_SEC)); + }); + + it("aligns a sentence's last word in the stretch it closes, even anchored in the pause", () => { + // "b." is said on frame 25 (0.5 s), but DTW anchored it at 0.75 s, past the + // first stretch's tail (issue #948). It is aligned there, not in "c"'s audio. + const tokens = [...gap(15), "a", "|", ...gap(8), "b", ...gap(49), "c", ...gap(24)]; + const out = alignWordsOnEmissions( + [word("a", 0.25, 0.35, 0.3), word("b.", 0.35, 0.75, 0.75), word("c", 0.75, 1.6, 1.55)], + [ + { startSec: 0.2, endSec: 0.6 }, + { startSec: 1.4, endSec: 1.8 }, + ], + emissions(tokens), + ); + expect(r(out[1].startSec)).toBe(r(25 * STRIDE + PAUSE_START_OFFSET_SEC)); + expect(r(out[2].startSec)).toBe(r(75 * STRIDE + PAUSE_START_OFFSET_SEC)); + }); + + it("keeps the helper's times where it cannot align", () => { + const words = [word("ab", 0.3, 0.5, 0.35), word("c", 1.5, 1.7, 1.6)]; + // The second word is outside every stretch; the first one's letters do not fit. + const out = alignWordsOnEmissions(words, speech, emissions(gap(2))); + expect(out).toEqual(words); + // No region covers the stretch at all. + expect(alignWordsOnEmissions(words, speech, emissions(gap(60), 5))).toEqual(words); + }); +}); + +describe("parseEmissions", () => { + const floats = new Float32Array([-0.1, -2, -3, -4, -5, -6, -7]); + const logprobs = Buffer.from(floats.buffer).toString("base64"); + + it("decodes the helper's base64 log-probs", () => { + const parsed = parseEmissions({ + vocab: VOCAB, + blank: 0, + stride_s: 0.02, + regions: [{ start: 1.5, frames: 1, logprobs }], + }); + expect(parsed?.regions[0].startSec).toBe(1.5); + expect([...(parsed?.regions[0].logprobs ?? [])]).toEqual([...floats]); + }); + + it("refuses a reply whose sizes do not add up", () => { + expect( + parseEmissions({ + vocab: VOCAB, + blank: 0, + stride_s: 0.02, + regions: [{ start: 0, frames: 2, logprobs }], + }), + ).toBeNull(); + }); +}); diff --git a/electron/stt/ctcAlign.ts b/electron/stt/ctcAlign.ts new file mode 100644 index 000000000..4e142671e --- /dev/null +++ b/electron/stt/ctcAlign.ts @@ -0,0 +1,294 @@ +// Second alignment pass: re-times whisper's words on a CTC acoustic model +// (issue #948, phase 3). +// +// whisper's DTW times words within ~30 ms median inside continuous speech, but +// its 20 ms frames and diffuse cross-attention cap it there (P90 ~110 ms). A +// wav2vec2 model fine-tuned for CTC scores every 20 ms frame against every +// letter, so a Viterbi pass that forces whisper's own words through those +// scores puts each letter where the audio has it. The helper computes the +// scores (`/emissions`, electron/native/whisper-stt/src/ctc_aligner.cpp); this +// file spells whisper's words in the model's letters, aligns them, and maps the +// letters back to word times. +// +// Words the model cannot spell (digits, symbols, another script) become one +// wildcard token that matches any speech, so they hold their place and their +// neighbours stay aligned. A stretch the model cannot fit keeps the times it +// came with, and so does every word when there is no aligner for the language. +// +// Self-contained on purpose: tools/stt-eval/word-timing imports this file +// straight into Node, which resolves no extensionless import. + +import { ownWords } from "./snapWordBoundaries.ts"; +import type { SttVadSegment } from "./transcriptionContract"; + +/** What `/emissions` answers, decoded. */ +export interface CtcEmissions { + vocab: string[]; + blank: number; + /** Seconds between two frames. */ + strideSec: number; + /** Per requested region: its first frame's time and `frames * vocab` log-probs. */ + regions: Array<{ startSec: number; frames: number; logprobs: Float32Array }>; +} + +/** The fields of a helper word this pass reads and rewrites. */ +export interface AlignableWord { + word: string; + startSec: number; + endSec: number; + /** Always inside the word; decides which stretch of speech owns it. */ + anchorSec: number; +} + +// Calibration, fitted on tools/stt-eval/word-timing (TTS and LibriSpeech agree). +// The model is sure of a letter a little after the sound starts and before it +// ends, so CTC shrinks words: the edges move outward. Inside continuous speech +// the word boundary lies between the last letter of one word and the first of +// the next; where the two estimates cross, the boundary is their midpoint. +/** Added to the frame of a word's first letter. */ +export const START_OFFSET_SEC = -0.045; +/** Added to the end of the frame of a word's last letter. */ +export const END_OFFSET_SEC = 0.025; +/** A gap between two words' letters this long is a pause, not a boundary. */ +export const PAUSE_SEC = 0.1; +/** After a pause, a word's first letter trails its onset further (breath, closure, soft attack). */ +export const PAUSE_START_OFFSET_SEC = -0.06; +/** Before a pause, a word decays past its last letter; early there is audible, late is silence. */ +export const PAUSE_END_OFFSET_SEC = 0.15; + +/** Audio kept on each side of a stretch of speech, so its first and last letters have context. */ +export const REGION_MARGIN_SEC = 0.3; + +/** As in snapWordBoundaries.ts: the audio the helper keeps past each speech offset. */ +const TAIL_SEC = 0.1; + +const WILDCARD = -1; + +/** Seconds of the upload the helper should score: the speech plus a margin, merged. */ +export function emissionRegions( + speech: SttVadSegment[], + durationSec: number, +): Array<[number, number]> { + const out: Array<[number, number]> = []; + for (const s of speech) { + const a = Math.max(0, s.startSec - REGION_MARGIN_SEC); + const b = Math.min(durationSec, s.endSec + REGION_MARGIN_SEC); + const last = out[out.length - 1]; + if (last && a <= last[1]) last[1] = Math.max(last[1], b); + else if (b > a) out.push([a, b]); + } + return out; +} + +/** + * A word as the model's token ids: `[]` when there is nothing to say + * (punctuation), `null` when a letter is not in the vocabulary (digits, symbols, + * another script). Case follows the vocabulary, an accented letter the + * vocabulary lacks falls back to its base letter, and the apostrophe is + * normalised. + */ +export function wordTokens(word: string, index: Map): number[] | null { + const upper = index.has("A") && !index.has("a"); + let text = word.normalize("NFC").replace(/[’‘`´]/g, "'"); + text = upper ? text.toUpperCase() : text.toLowerCase(); + const out: number[] = []; + for (const ch of text) { + const id = index.get(ch); + if (id !== undefined) { + out.push(id); + continue; + } + if (/\p{S}/u.test(ch)) return null; // "%", "+", "€": said aloud, but not spelled so + if (/[\p{P}\s]/u.test(ch)) continue; // silent, apostrophes and hyphens included + const base = [...ch.normalize("NFD").replace(/\p{M}/gu, "")]; + const ids = base.map((c) => index.get(c)); + if (ids.length === 0 || ids.some((x) => x === undefined)) return null; + out.push(...(ids as number[])); + } + return out; +} + +/** + * CTC Viterbi forced alignment of `labels` over `T` frames of `V` log-probs. + * `WILDCARD` scores as the frame's best non-blank token. Returns, per label, the + * first and last frame the best path spends on it, or null when the labels + * cannot fit in the frames. + */ +export function ctcViterbi( + logprobs: Float32Array, + T: number, + V: number, + blank: number, + labels: number[], +): Array<[number, number]> | null { + const L = labels.length; + const S = 2 * L + 1; + if (L === 0 || T === 0) return null; + const wild = new Float32Array(T); + if (labels.includes(WILDCARD)) { + for (let t = 0; t < T; t++) { + let best = Number.NEGATIVE_INFINITY; + for (let v = 0; v < V; v++) if (v !== blank) best = Math.max(best, logprobs[t * V + v]); + wild[t] = best; + } + } + const lab = (s: number) => (s % 2 === 0 ? blank : labels[(s - 1) >> 1]); + const em = (t: number, s: number) => { + const l = lab(s); + return l === WILDCARD ? wild[t] : logprobs[t * V + l]; + }; + const NEG = Number.NEGATIVE_INFINITY; + let prev = new Float64Array(S).fill(NEG); + let cur = new Float64Array(S); + const back = new Uint8Array(T * S); // how far back the best predecessor is: 0, 1 or 2 + prev[0] = em(0, 0); + prev[1] = em(0, 1); + for (let t = 1; t < T; t++) { + for (let s = 0; s < S; s++) { + let best = prev[s]; + let from = 0; + if (s >= 1 && prev[s - 1] > best) { + best = prev[s - 1]; + from = 1; + } + // A label may follow the one before the blank directly, unless it repeats it. + if (s >= 2 && s % 2 === 1 && lab(s) !== lab(s - 2) && prev[s - 2] > best) { + best = prev[s - 2]; + from = 2; + } + cur[s] = best === NEG ? NEG : best + em(t, s); + back[t * S + s] = from; + } + [prev, cur] = [cur, prev]; + } + let s = prev[S - 2] > prev[S - 1] ? S - 2 : S - 1; + if (prev[s] === NEG) return null; + const spans: Array<[number, number]> = labels.map(() => [-1, -1]); + for (let t = T - 1; t >= 0; t--) { + if (s % 2 === 1) { + const k = (s - 1) >> 1; + spans[k][0] = t; + if (spans[k][1] < 0) spans[k][1] = t; + } + s -= back[t * S + s]; + } + return spans; +} + +/** + * Re-time `words` on `emissions`. Each speech stretch owns the words + * `ownWords` (snapWordBoundaries.ts) gives it; they are aligned on the frames + * between the neighbouring stretches, with the vocabulary's word delimiter + * between words when it has one. Words outside every stretch, stretches the + * model cannot fit, and a vocabulary with no word in it keep the helper's + * times. Punctuation collapses onto the end of the word before it. + */ +export function alignWordsOnEmissions( + words: W[], + speech: SttVadSegment[], + emissions: CtcEmissions, +): W[] { + const out = words.map((w) => ({ ...w })); + const index = new Map(emissions.vocab.map((t, i) => [t, i])); + const delimiter = index.get("|"); + const V = emissions.vocab.length; + const { strideSec } = emissions; + const ranges = ownWords(words, speech); + for (let i = 0; i < speech.length; i++) { + const { startSec: onset, endSec: offset } = speech[i]; + const [from, to] = ranges[i]; + const owned = Array.from({ length: to - from }, (_, n) => from + n); + // A region's frames stop up to one receptive field (25 ms) short of the audio + // it was cut from, so a stretch that runs to the end of the upload ends + // past them; two frames of slack keep it. `hi` below stays on the frames. + const region = emissions.regions.find( + (r) => r.startSec <= onset && r.startSec + (r.frames + 2) * strideSec >= offset, + ); + if (!region || owned.length === 0) continue; + // The frames of this stretch: its margin, stopping at the neighbours' speech. + const lo = Math.max( + region.startSec, + (speech[i - 1]?.endSec ?? Number.NEGATIVE_INFINITY) + TAIL_SEC, + onset - REGION_MARGIN_SEC, + ); + const hi = Math.min( + region.startSec + region.frames * strideSec, + speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY, + offset + REGION_MARGIN_SEC, + ); + const f0 = Math.max(0, Math.ceil((lo - region.startSec) / strideSec)); + const f1 = Math.min(region.frames, Math.floor((hi - region.startSec) / strideSec)); + if (f1 <= f0) continue; + + const labels: number[] = []; + const timed: Array<{ j: number; first: number; last: number }> = []; + for (const j of owned) { + const toks = wordTokens(out[j].word, index); + if (toks?.length === 0) continue; + if (labels.length > 0 && delimiter !== undefined) labels.push(delimiter); + timed.push({ j, first: labels.length, last: labels.length + (toks?.length ?? 1) - 1 }); + labels.push(...(toks ?? [WILDCARD])); + } + const spans = ctcViterbi( + region.logprobs.subarray(f0 * V, f1 * V), + f1 - f0, + V, + emissions.blank, + labels, + ); + if (!spans) continue; + const at = (f: number) => region.startSec + (f0 + f) * strideSec; + const s = timed.map((w) => at(spans[w.first][0])); + const e = timed.map((w) => at(spans[w.last][1] + 1)); + timed.forEach(({ j }, q) => { + const prevEnd = e[q - 1] ?? Number.NEGATIVE_INFINITY; + const nextStart = s[q + 1] ?? Number.POSITIVE_INFINITY; + out[j].startSec = + s[q] - prevEnd >= PAUSE_SEC + ? Math.max(s[q] + PAUSE_START_OFFSET_SEC, (prevEnd + s[q]) / 2) + : s[q] + START_OFFSET_SEC; + out[j].endSec = + nextStart - e[q] >= PAUSE_SEC + ? Math.min(e[q] + PAUSE_END_OFFSET_SEC, (e[q] + nextStart) / 2) + : e[q] + END_OFFSET_SEC; + const before = q > 0 ? out[timed[q - 1].j] : null; + if (before && before.endSec > out[j].startSec) { + const mid = (before.endSec + out[j].startSec) / 2; + before.endSec = mid; + out[j].startSec = mid; + } + }); + for (let n = 1; n < owned.length; n++) { + const j = owned[n]; + if (timed.some((w) => w.j === j)) continue; + out[j].startSec = out[owned[n - 1]].endSec; + out[j].endSec = out[j].startSec; + } + } + return out; +} + +/** The helper's `/emissions` JSON. */ +export interface EmissionsJson { + vocab: string[]; + blank: number; + stride_s: number; + regions: Array<{ start: number; frames: number; logprobs: string }>; + /** Seconds the helper spent, model load included. */ + elapsed_s?: number; +} + +/** Decodes the helper's `/emissions` JSON; null when it is not one. */ +export function parseEmissions(json: EmissionsJson): CtcEmissions | null { + if (!Array.isArray(json?.vocab) || !Array.isArray(json.regions) || !(json.stride_s > 0)) + return null; + const V = json.vocab.length; + const regions: CtcEmissions["regions"] = []; + for (const r of json.regions) { + // Copied: a pooled Buffer can start on an offset Float32Array refuses. + const bytes = Uint8Array.from(Buffer.from(r.logprobs, "base64")); + if (bytes.byteLength !== r.frames * V * 4) return null; + regions.push({ startSec: r.start, frames: r.frames, logprobs: new Float32Array(bytes.buffer) }); + } + return { vocab: json.vocab, blank: json.blank, strideSec: json.stride_s, regions }; +} diff --git a/electron/stt/extractAudio.test.ts b/electron/stt/extractAudio.test.ts index 24e9694c8..c437b1ae9 100644 --- a/electron/stt/extractAudio.test.ts +++ b/electron/stt/extractAudio.test.ts @@ -8,10 +8,11 @@ const resolveFfmpegMock = vi.fn<() => string | null>(); vi.mock("node:child_process", () => ({ spawn: (...args: unknown[]) => spawnMock(...args) })); vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => resolveFfmpegMock() })); -const { extractMono16kPcm, FfmpegUnavailableError, NoAudioTrackError } = await import( - "./extractAudio" +const { extractMono16kPcm, FfmpegUnavailableError, MediaUnreadableError, NoAudioTrackError } = + await import("./extractAudio"); +const { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } = await import( + "./transcriptionContract" ); -const { STT_NATIVE_EXTRACTION_UNAVAILABLE } = await import("./transcriptionContract"); /** A stand-in for the ffmpeg child: two pipes and a close event, nothing more. */ function fakeChild() { @@ -95,6 +96,60 @@ describe("extractMono16kPcm", () => { await expect(promise).rejects.toBeInstanceOf(NoAudioTrackError); }); + // Issue #968: the renderer explains these as "check folder access" and retries + // them, so both must carry the marker, which is all that survives the IPC. + it("reports a read the OS refused as unreadable media, not as no audio", async () => { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4"); + child.stderr.end("/Users/me/Downloads/a.mp4: Operation not permitted"); + child.stdout.end(); + child.emit("close", 1); + + await expect(promise).rejects.toBeInstanceOf(MediaUnreadableError); + await expect(promise).rejects.toThrow(STT_MEDIA_UNREADABLE); + }); + + it("reports an ffmpeg that hangs opening the file as unreadable media", async () => { + vi.useFakeTimers(); + try { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4"); + const settled = expect(promise).rejects.toThrow(STT_MEDIA_UNREADABLE); + await vi.advanceTimersByTimeAsync(60_000); + await settled; + expect(child.kill).toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + + it("reports a timeout AFTER audio came out as a plain stall, not unreadable media", async () => { + // The file was readable; pointing the user at folder access would be wrong. + vi.useFakeTimers(); + try { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4"); + const settled = promise.then( + () => { + throw new Error("expected a rejection"); + }, + (error: Error) => error, + ); + child.stdout.write(f32le([0.5])); + await vi.advanceTimersByTimeAsync(60_000); + const error = await settled; + expect(error).not.toBeInstanceOf(MediaUnreadableError); + expect(error.message).not.toContain(STT_MEDIA_UNREADABLE); + expect(error.message).toBe("ffmpeg timed out after 60000ms on /Users/me/Downloads/a.mp4"); + expect(child.kill).toHaveBeenCalledWith("SIGKILL"); + } finally { + vi.useRealTimers(); + } + }); + it("keeps the samples when ffmpeg exits non-zero AFTER writing audio", async () => { // A truncated file still yields usable audio; throwing it away would lose a // transcript over a trailing byte. diff --git a/electron/stt/extractAudio.ts b/electron/stt/extractAudio.ts index 08168d21e..28dd9f177 100644 --- a/electron/stt/extractAudio.ts +++ b/electron/stt/extractAudio.ts @@ -21,7 +21,7 @@ import { spawn } from "node:child_process"; import { resolveFfmpeg } from "../media/audioPeaks"; -import { STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract"; +import { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract"; /** What whisper.cpp wants, and what `decodePeaks` already asks ffmpeg for. */ const SAMPLE_RATE = 16_000; @@ -50,6 +50,22 @@ export class NoAudioTrackError extends Error { } } +/** + * The media file could not be read at all: the OS refused it, or ffmpeg hung + * opening it. Unlike `NoAudioTrackError` this is NOT permanent: it clears once + * the user grants access, so it carries `STT_MEDIA_UNREADABLE` for the renderer + * to explain rather than remember. + */ +export class MediaUnreadableError extends Error { + constructor(filePath: string, detail: string) { + super(`${STT_MEDIA_UNREADABLE}: cannot read ${filePath}: ${detail}`); + this.name = "MediaUnreadableError"; + } +} + +/** What ffmpeg's stderr says when the OS refused to open the input. */ +const ACCESS_DENIED = /operation not permitted|permission denied/i; + /** * Decode `filePath` to mono 16 kHz float samples. * @@ -105,8 +121,16 @@ export async function extractMono16kPcm( const timer = setTimeout(() => { child.kill("SIGKILL"); + // Only a hang before the first byte says the file could not be opened (a + // pending macOS folder-access prompt produces exactly that, issue #968). Once + // audio has come out the media was readable, and the stall is ffmpeg's. + const nothingRead = total === 0 && carry === null; finish(() => - reject(new Error(`ffmpeg timed out after ${EXTRACT_TIMEOUT_MS}ms on ${filePath}`)), + reject( + nothingRead + ? new MediaUnreadableError(filePath, `ffmpeg timed out after ${EXTRACT_TIMEOUT_MS}ms`) + : new Error(`ffmpeg timed out after ${EXTRACT_TIMEOUT_MS}ms on ${filePath}`), + ), ); }, EXTRACT_TIMEOUT_MS); @@ -137,7 +161,13 @@ export async function extractMono16kPcm( // caller treats both the same way — this asset will not transcribe — so they // share an error type; `stderr` carries which it was. if (code !== 0 && total === 0) { - finish(() => reject(new NoAudioTrackError(filePath, stderr.trim()))); + // Except a read the OS refused: that also exits non-zero with nothing + // decoded, but remembering it as "no audio" would never retry the file + // once access is granted. + const error = ACCESS_DENIED.test(stderr) + ? new MediaUnreadableError(filePath, stderr.trim()) + : new NoAudioTrackError(filePath, stderr.trim()); + finish(() => reject(error)); return; } const out = new Float32Array(total); diff --git a/electron/stt/index.test.ts b/electron/stt/index.test.ts index 69ef1723b..c5af87fe9 100644 --- a/electron/stt/index.test.ts +++ b/electron/stt/index.test.ts @@ -34,6 +34,9 @@ vi.mock("./whisperServer", () => { vi.mock("./modelManager", () => ({ ensureModels: vi.fn(async () => undefined), + ensureAligner: vi.fn(async ({ language }: { language: string }) => `/fake/${language}.gguf`), + cachedAligner: vi.fn(async () => null), + alignerPath: vi.fn((base: string, language: string): string | null => `${base}/${language}.gguf`), modelPaths: (base: string) => ({ whisper: `${base}/whisper-ggml/ggml-small-q8_0.bin`, }), @@ -470,6 +473,110 @@ describe("SttManager", () => { expect(fakeWhisperServer.start).toHaveBeenCalledOnce(); }); + describe("word aligner", () => { + /** Chunks report what `alignerFor("fr")` gave them, as the helper would use it. */ + function recordAligner() { + const asked: Array = []; + fakeWhisperServer.transcribe.mockImplementation( + async (opts: { alignerFor?: (language: string) => Promise }) => { + asked.push((await opts.alignerFor?.("fr")) ?? null); + return { + segments: [], + wordSegments: [], + detectedLanguage: "fr", + backend: "whispercpp-cpu" as const, + }; + }, + ); + return asked; + } + const long = new Float32Array(16_000 * 400); + + async function mocks() { + const mm = await import("./modelManager"); + const ensure = vi.mocked(mm.ensureAligner); + const cached = vi.mocked(mm.cachedAligner); + ensure.mockReset(); + cached.mockReset(); + cached.mockResolvedValue(null); + return { ensure, cached }; + } + + it("never holds a chunk on the download, and uses the aligner once it lands", async () => { + const { ensure } = await mocks(); + let land: (file: string) => void = () => undefined; + ensure.mockImplementation(() => new Promise((resolve) => (land = resolve))); + const asked = recordAligner(); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + + // The download never finishes during this run: every chunk goes on without it. + await mgr.transcribe({ samples: long, language: "fr" }); + expect(asked.length).toBeGreaterThan(1); + expect(asked.every((x) => x === null)).toBe(true); + expect(ensure).toHaveBeenCalledOnce(); + + land("/fake/fr.gguf"); + await new Promise((resolve) => setTimeout(resolve, 0)); + asked.length = 0; + await mgr.transcribe({ samples: new Float32Array(16_000), language: "fr" }); + expect(asked).toEqual(["/fake/fr.gguf"]); + expect(ensure).toHaveBeenCalledOnce(); + }); + + it("uses a verified copy already on disk without downloading", async () => { + const { ensure, cached } = await mocks(); + cached.mockResolvedValue("/fake/fr.gguf"); + const asked = recordAligner(); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + await mgr.transcribe({ samples: long, language: "fr" }); + expect(asked.every((x) => x === "/fake/fr.gguf")).toBe(true); + expect(cached).toHaveBeenCalledOnce(); + expect(ensure).not.toHaveBeenCalled(); + }); + + it("aborts the download on cancel and on shutdown, then retries on a later run", async () => { + const { ensure } = await mocks(); + const signals: AbortSignal[] = []; + ensure.mockImplementation( + ({ signal }: { signal?: AbortSignal }) => + new Promise((_, reject) => { + if (signal) signals.push(signal); + signal?.addEventListener("abort", () => reject(signal.reason)); + }), + ); + recordAligner(); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + await mgr.transcribe({ samples: long, language: "fr" }); + expect(signals).toHaveLength(1); + + mgr.cancel(); + expect(signals[0].aborted).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 0)); + + // Not cached as failed for good: the next transcription starts it again. + await mgr.transcribe({ samples: long, language: "fr" }); + expect(signals).toHaveLength(2); + expect(signals[1].aborted).toBe(false); + await mgr.shutdown(); + expect(signals[1].aborted).toBe(true); + }); + + it("gives a language without an aligner nothing, and downloads nothing", async () => { + const { ensure } = await mocks(); + const mm = await import("./modelManager"); + vi.mocked(mm.alignerPath).mockReturnValueOnce(null); + const asked = recordAligner(); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + await mgr.transcribe({ samples: new Float32Array(16_000), language: "fr" }); + expect(asked).toEqual([null]); + expect(ensure).not.toHaveBeenCalled(); + }); + }); + it("fans status out to every sink, and detaching one leaves the others", async () => { const mgr = new SttManager(); const a = vi.fn<(e: SttStatusEvent) => void>(); diff --git a/electron/stt/index.ts b/electron/stt/index.ts index 656d7bf6d..11c760f0b 100644 --- a/electron/stt/index.ts +++ b/electron/stt/index.ts @@ -2,17 +2,20 @@ import path from "node:path"; import { app, type IpcMain } from "electron"; import { planChunks } from "./chunking"; import { extractMono16kPcm } from "./extractAudio"; -import { ensureModels, modelPaths } from "./modelManager"; import { - STT_VAD_UNAVAILABLE, - type SttPhraseSegment, - type SttStatusEvent, - type SttTiming, - type SttTranscribeRequest, - type SttTranscribeResponse, - type SttVadResponse, - type SttVadSegment, - type SttWordSegment, + alignerPath, + cachedAligner, + ensureAligner, + ensureModels, + modelPaths, +} from "./modelManager"; +import type { + SttPhraseSegment, + SttStatusEvent, + SttTiming, + SttTranscribeRequest, + SttTranscribeResponse, + SttWordSegment, } from "./transcriptionContract"; import { WhisperServerManager } from "./whisperServer"; @@ -108,6 +111,60 @@ export class SttManager { */ private cancelEpoch = 0; + /** + * The word aligner per language (ctcAlign.ts). A chunk never waits for a + * download: it is an improvement, and a 350 MB fetch inside a chunk would hold + * the helper's queue for as long as the network takes, past any request + * ceiling and out of Cancel's reach. So: + * - a copy already on disk is verified in place the first time a language is + * asked for in a session (a local read, no network), then reused; + * - otherwise the download starts in the background and the chunks keep + * whisper's times until it lands; later chunks and transcriptions use it; + * - Cancel and quit abort it, and so does a stall (`ensureAligner`). A failed + * one is not retried for the rest of the run, only on the next transcription. + */ + private readonly alignersReady = new Map(); + private readonly alignersChecked = new Set(); + private readonly alignerDownloads = new Map(); + private readonly alignersFailed = new Set(); + + private readonly alignerFor = async (language: string): Promise => { + const ready = this.alignersReady.get(language); + if (ready) return ready; + if (!alignerPath(this.getModelsDir(), language)) return null; + if (!this.alignersChecked.has(language)) { + this.alignersChecked.add(language); + const cached = await cachedAligner(this.getModelsDir(), language); + if (cached) { + this.alignersReady.set(language, cached); + return cached; + } + } + if (!this.alignersFailed.has(language) && !this.alignerDownloads.has(language)) { + this.downloadAligner(language); + } + return null; + }; + + private downloadAligner(language: string): void { + const controller = new AbortController(); + this.alignerDownloads.set(language, controller); + console.info(`[stt] downloading the "${language}" word aligner in the background`); + ensureAligner({ baseDir: this.getModelsDir(), language, signal: controller.signal }) + .then((file) => { + if (file) this.alignersReady.set(language, file); + console.info(`[stt] "${language}" word aligner ready`); + }) + .catch((error: unknown) => { + this.alignersFailed.add(language); + console.warn( + `[stt] no word aligner for "${language}", keeping whisper's word times: ` + + `${error instanceof Error ? error.message : String(error)}`, + ); + }) + .finally(() => this.alignerDownloads.delete(language)); + } + /** * The extraction in flight, if any. `cancelEpoch` alone stops the CHUNK loop, which is * checked between chunks — so a cancel during the decode left ffmpeg running to @@ -147,6 +204,7 @@ export class SttManager { cancel(): void { this.cancelEpoch++; this.extraction?.abort(); + for (const download of this.alignerDownloads.values()) download.abort(cancelledError()); } /** @@ -238,7 +296,7 @@ export class SttManager { for (let attempt = 1; attempt <= CHUNK_ATTEMPTS; attempt++) { if (this.shuttingDown) throw cancelledError(); try { - return await this.server.transcribe({ samples, language }); + return await this.server.transcribe({ samples, language, alignerFor: this.alignerFor }); } catch (error) { lastError = error; if (this.shuttingDown) throw cancelledError(); @@ -276,6 +334,7 @@ export class SttManager { await this.init(); const epoch = this.cancelEpoch; + this.alignersFailed.clear(); // Extraction is part of the run, and on a long file it is the part the user used // to watch the editor freeze through. Doing it here means the renderer hands over // a path and gets segments back, holding none of the audio. No new status phase: @@ -440,26 +499,9 @@ export class SttManager { async shutdown(): Promise { if (this.shuttingDown) return; this.shuttingDown = true; - this.cancelEpoch++; + this.cancel(); await this.server.shutdown(); } - - /** True when the helper reported Silero VAD loaded and ready. */ - isVadAvailable(): boolean { - return this.server.status.vadAvailable; - } - - /** - * Run Voice Activity Detection (Silero VAD) to detect speech segments in samples. - */ - async detectSpeech(samples: Float32Array): Promise { - if (this.shuttingDown) throw cancelledError(); - await this.init(); - if (!this.server.status.vadAvailable) { - throw new Error(STT_VAD_UNAVAILABLE); - } - return this.server.detectVadSegments({ samples }); - } } let singleton: SttManager | null = null; @@ -518,13 +560,6 @@ export function registerSttIpc(ipcMain: IpcMain): void { } }, ); - ipcMain.handle( - "stt:vad", - async (_event, req: { samples: Float32Array }): Promise => { - const segments = await manager.detectSpeech(req.samples); - return { segments }; - }, - ); ipcMain.handle("stt:cancel", () => { manager.cancel(); }); diff --git a/electron/stt/modelManager.test.ts b/electron/stt/modelManager.test.ts index 441906234..dcee6a24f 100644 --- a/electron/stt/modelManager.test.ts +++ b/electron/stt/modelManager.test.ts @@ -4,7 +4,16 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises" import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { areModelsPresent, ensureModels, modelPaths, STT_MODELS } from "./modelManager"; +import { + alignerPath, + areModelsPresent, + CTC_ALIGNERS, + cachedAligner, + ensureAligner, + ensureModels, + modelPaths, + STT_MODELS, +} from "./modelManager"; describe("modelManager", () => { let dir: string; @@ -254,4 +263,116 @@ describe("modelManager", () => { STT_MODELS.whisper.files[0].expectedSha256 = originalSha; } }); + + describe("CTC aligners", () => { + it("pins every aligner to a digest and names its upstream revision", () => { + for (const file of Object.values(CTC_ALIGNERS)) { + expect(file.expectedSha256).toMatch(/^[0-9a-f]{64}$/); + expect(file.source).toMatch(/@[0-9a-f]{40}$/); + // Hosted under a `v0.0.0-*` release tag, never a moving one. + expect(file.url).toMatch(/\/releases\/download\/v0\.0\.0-[^/]+\//); + } + }); + + it("has none for a language it does not cover, prototype keys included", async () => { + expect(alignerPath(dir, "de")).toBeNull(); + expect(alignerPath(dir, "constructor")).toBeNull(); + expect(await ensureAligner({ baseDir: dir, language: "de" })).toBeNull(); + }); + + it("downloads, verifies and returns the aligner of a covered language", async () => { + const bytes = Buffer.from("gguf weights"); + const original = CTC_ALIGNERS.fr.expectedSha256; + CTC_ALIGNERS.fr.expectedSha256 = createHash("sha256").update(bytes).digest("hex"); + try { + const file = await ensureAligner({ + baseDir: dir, + language: "fr", + fetcher: async () => new Response(bytes, { status: 200 }), + }); + expect(file).toBe(alignerPath(dir, "fr")); + expect(await readFile(file as string)).toEqual(bytes); + } finally { + CTC_ALIGNERS.fr.expectedSha256 = original; + } + }); + + it("refuses a download whose digest does not match", async () => { + await expect( + ensureAligner({ + baseDir: dir, + language: "en", + fetcher: async () => new Response("tampered", { status: 200 }), + }), + ).rejects.toThrow(/SHA-256 mismatch/); + expect(existsSync(alignerPath(dir, "en") as string)).toBe(false); + }); + + /** A body that sends a few bytes, then nothing, ever. */ + const stalledBody = () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(8)); + }, + }), + { status: 200 }, + ); + + it("aborts a download that stops sending, and leaves no partial file", async () => { + const file = alignerPath(dir, "fr") as string; + await expect( + ensureAligner({ + baseDir: dir, + language: "fr", + stallMs: 50, + fetcher: async () => stalledBody(), + }), + ).rejects.toThrow(/stalled/); + expect(existsSync(file)).toBe(false); + expect(existsSync(`${file}.partial`)).toBe(false); + }); + + it("stops when its signal aborts, mid-body or between attempts", async () => { + const controller = new AbortController(); + const download = ensureAligner({ + baseDir: dir, + language: "fr", + signal: controller.signal, + fetcher: async () => stalledBody(), + }); + setTimeout(() => controller.abort(new Error("cancelled")), 20); + await expect(download).rejects.toThrow("cancelled"); + + // A 503 puts it in a backoff of seconds: the abort must not wait for it. + const again = new AbortController(); + const started = Date.now(); + const retrying = ensureAligner({ + baseDir: dir, + language: "fr", + signal: again.signal, + fetcher: async () => new Response("busy", { status: 503 }), + }); + setTimeout(() => again.abort(new Error("quit")), 20); + await expect(retrying).rejects.toThrow("quit"); + expect(Date.now() - started).toBeLessThan(1000); + }); + + it("verifies a cached copy without the network", async () => { + expect(await cachedAligner(dir, "fr")).toBeNull(); + const file = alignerPath(dir, "fr") as string; + const bytes = Buffer.from("gguf weights"); + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, bytes); + // Present but not the pinned bytes: not usable. + expect(await cachedAligner(dir, "fr")).toBeNull(); + const original = CTC_ALIGNERS.fr.expectedSha256; + CTC_ALIGNERS.fr.expectedSha256 = createHash("sha256").update(bytes).digest("hex"); + try { + expect(await cachedAligner(dir, "fr")).toBe(file); + } finally { + CTC_ALIGNERS.fr.expectedSha256 = original; + } + }); + }); }); diff --git a/electron/stt/modelManager.ts b/electron/stt/modelManager.ts index 959551237..d9d4aef32 100644 --- a/electron/stt/modelManager.ts +++ b/electron/stt/modelManager.ts @@ -20,8 +20,11 @@ import { pipeline } from "node:stream/promises"; * The file is verified by SHA-256 and written atomically (via .partial rename) * to prevent partial downloads from being treated as complete. * - * Word timestamps come from whisper.cpp's native DTW token timestamps, so no - * separate VAD model is required. See `technical-documentation/architecture/transcription-and-captions.md`. + * Word timestamps come from whisper.cpp's native DTW token timestamps. The + * Silero VAD model only decides which audio whisper decodes, and gives the + * speech edges phrases are anchored on. The CTC aligners (`CTC_ALIGNERS`) re-time + * the words for the languages that have one, and are fetched only when a + * transcription detects such a language. See `technical-documentation/architecture/transcription-and-captions.md`. */ export type SttModelId = "whisper" | "silero-vad"; @@ -93,6 +96,89 @@ export const STT_MODELS: Record = { }, }; +/** + * The CTC aligners that re-time whisper's words (electron/stt/ctcAlign.ts), per + * language whisper reports. A language missing here keeps whisper's DTW times. + * + * Each file is a wav2vec2 CTC model converted to GGUF by + * `scripts/convert-wav2vec2-gguf.mjs` (deterministic: re-running it on the source + * named in `source` reproduces the digest below), then published under a `v0.0.0-*` + * release tag like the other binaries that need a permanent URL but are not a + * product version (see scripts/fetch-onnxruntime.mjs). Apache-2.0, both of them. + */ +const ALIGNER_RELEASE = + "https://github.com/getopenscreen/openscreen/releases/download/v0.0.0-ctc-aligners-1"; + +export interface CtcAlignerFile extends SttModelFile { + /** HuggingFace repo and revision the file was converted from. */ + source: string; +} + +export const CTC_ALIGNERS: Record = { + en: { + name: "w2v-en-base-q8_0.gguf", + url: `${ALIGNER_RELEASE}/w2v-en-base-q8_0.gguf`, + expectedSha256: "b7f21a97208f368d3505bd9a7bc9ff3795b1169d8e3b036028082eb25eb464ae", + approximateBytes: 109_040_064, + source: "facebook/wav2vec2-base-960h@22aad52d435eb6dbaf354bdad9b0da84ce7d6156", + }, + fr: { + name: "w2v-fr-large-q8_0.gguf", + url: `${ALIGNER_RELEASE}/w2v-fr-large-q8_0.gguf`, + expectedSha256: "e3c284da3e27564db07ac226bf6402a4d7856b806455b3e0f5283f29f9495f48", + approximateBytes: 348_037_120, + // The repo's safetensors conversion PR: main only has pytorch_model.bin. + source: "jonatasgrosman/wav2vec2-large-xlsr-53-french@70db24a266633ffcc8edce4e72f3a5cb69d602d6", + }, +}; + +const ALIGNER_DIR = "ctc-aligner"; + +/** Where the aligner for `language` lives, or null when there is none for it. */ +export function alignerPath(baseDir: string, language: string): string | null { + const file = Object.keys(CTC_ALIGNERS).includes(language) ? CTC_ALIGNERS[language] : null; + return file ? path.join(baseDir, ALIGNER_DIR, file.name) : null; +} + +/** + * The aligner for `language` when it is already on disk and intact, without + * touching the network: null when it is missing, corrupt, or the language has + * none. A local read, so a transcription can afford to wait for it. + */ +export async function cachedAligner(baseDir: string, language: string): Promise { + const filePath = alignerPath(baseDir, language); + if (!filePath || !existsSync(filePath)) return null; + const actual = await sha256OfFile(filePath).catch(() => ""); + return actual === CTC_ALIGNERS[language].expectedSha256 ? filePath : null; +} + +/** Abort an aligner download that receives no bytes for this long. */ +export const ALIGNER_STALL_MS = 30_000; + +/** + * Make sure the aligner for `language` is on disk, downloading and verifying it + * like the whisper model. Null when the language has none; throws when the + * download fails, stalls for `stallMs` or is aborted through `signal`, which the + * caller turns into "keep whisper's times". + */ +export async function ensureAligner(opts: { + baseDir: string; + language: string; + signal?: AbortSignal; + stallMs?: number; + fetcher?: typeof fetch; +}): Promise { + const filePath = alignerPath(opts.baseDir, opts.language); + if (!filePath) return null; + const file = CTC_ALIGNERS[opts.language]; + await ensureFile(filePath, file.url, file.expectedSha256, { + fetcher: opts.fetcher, + signal: opts.signal, + stallMs: opts.stallMs ?? ALIGNER_STALL_MS, + }); + return filePath; +} + export function modelPaths(baseDir: string): Record { return { whisper: path.join(baseDir, STT_MODELS.whisper.cacheDir, MODEL_FILE), @@ -131,8 +217,19 @@ export async function sha256OfFile(filePath: string): Promise { const MAX_ATTEMPTS = 6; const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]); -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); +function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(signal.reason); + const timer = setTimeout(resolve, ms); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + reject(signal.reason); + }, + { once: true }, + ); + }); } function backoffMs(attempt: number, retryAfter: string | null): number { @@ -145,19 +242,34 @@ function backoffMs(attempt: number, retryAfter: string | null): number { return Math.min(60_000, 2_000 * 2 ** (attempt - 1)) + Math.floor(Math.random() * 1000); } -async function fetchWithRetry(url: string, fetcher: typeof fetch): Promise { +/** + * `signal` aborts the request and the waits between attempts; `waiting` is told + * when a backoff starts and ends, so a stall timer does not count it. + */ +async function fetchWithRetry( + url: string, + fetcher: typeof fetch, + signal?: AbortSignal, + waiting: (yes: boolean) => void = () => undefined, +): Promise { let lastErr: unknown; + const backoff = async (ms: number) => { + waiting(true); + await sleep(ms, signal); + waiting(false); + }; for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { try { const res = await fetcher(url, { headers: { "user-agent": "openscreen-stt" }, + signal, }); if (res.ok && res.body) return res; if (res.status >= 400 && res.status < 500 && !RETRYABLE_STATUS.has(res.status)) { throw new Error(`Failed to download ${url}: HTTP ${res.status} ${res.statusText}`); } if (RETRYABLE_STATUS.has(res.status) && attempt < MAX_ATTEMPTS) { - await sleep(backoffMs(attempt, res.headers.get("retry-after"))); + await backoff(backoffMs(attempt, res.headers.get("retry-after"))); continue; } throw new Error(`Failed to download ${url}: HTTP ${res.status} ${res.statusText}`); @@ -166,8 +278,8 @@ async function fetchWithRetry(url: string, fetcher: typeof fetch): Promise= MAX_ATTEMPTS) throw err; - await sleep(backoffMs(attempt, null)); + if (signal?.aborted || attempt >= MAX_ATTEMPTS) throw signal?.reason ?? err; + await backoff(backoffMs(attempt, null)); } } throw lastErr; @@ -178,6 +290,10 @@ export interface DownloadOptions { onProgress?: (bytes: number) => void; /** Override fetch (for tests); defaults to `globalThis.fetch`. */ fetcher?: typeof fetch; + /** Aborts the download (and the waits between attempts). */ + signal?: AbortSignal; + /** Abort when no byte arrives for this long, backoffs aside. Off when unset. */ + stallMs?: number; } /** @@ -211,17 +327,44 @@ async function ensureFile( await mkdir(path.dirname(filePath), { recursive: true }); const fetcher = options.fetcher ?? fetch; - const res = await fetchWithRetry(fileUrl, fetcher); const tmp = `${filePath}.partial`; - let downloaded = 0; - - const source = Readable.fromWeb(res.body as never); - source.on("data", (chunk: Buffer | Uint8Array) => { - downloaded += chunk.length; - options.onProgress?.(downloaded); - }); - const { createWriteStream } = await import("node:fs"); - await pipeline(source, createWriteStream(tmp)); + // One controller for the caller's abort and the stall timer: a connection that + // stops sending leaves `pipeline` pending forever, with nothing else to end it. + const controller = new AbortController(); + const onAbort = () => controller.abort(options.signal?.reason); + if (options.signal?.aborted) onAbort(); + options.signal?.addEventListener("abort", onAbort, { once: true }); + let stall: ReturnType | undefined; + const arm = (on = true) => { + clearTimeout(stall); + if (!on || !options.stallMs) return; + const ms = options.stallMs; + stall = setTimeout( + () => controller.abort(new Error(`download stalled: no data for ${ms / 1000}s`)), + ms, + ); + }; + try { + arm(); + const res = await fetchWithRetry(fileUrl, fetcher, controller.signal, (waiting) => + arm(!waiting), + ); + let downloaded = 0; + const source = Readable.fromWeb(res.body as never); + source.on("data", (chunk: Buffer | Uint8Array) => { + arm(); + downloaded += chunk.length; + options.onProgress?.(downloaded); + }); + const { createWriteStream } = await import("node:fs"); + await pipeline(source, createWriteStream(tmp), { signal: controller.signal }); + } catch (error) { + await rm(tmp, { force: true }).catch(() => undefined); + throw controller.signal.aborted ? (controller.signal.reason ?? error) : error; + } finally { + arm(false); + options.signal?.removeEventListener("abort", onAbort); + } if (expectedSha256) { const actual = await sha256OfFile(tmp); diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.ts index 2668c6966..f3272b38d 100644 --- a/electron/stt/snapWordBoundaries.test.ts +++ b/electron/stt/snapWordBoundaries.test.ts @@ -1,76 +1,192 @@ import { describe, expect, it } from "vitest"; -import { snapWordBoundariesToAudio } from "./snapWordBoundaries"; +import { anchorWordsOnSpeech, type HelperWord, ownWords } from "./snapWordBoundaries"; import type { SttWordSegment } from "./transcriptionContract"; -const SAMPLE_RATE = 16_000; - -/** Mono 16 kHz buffer that is loud everywhere except the given silent spans. */ -function audioWithSilences(durationSec: number, silences: Array<[number, number]>): Float32Array { - const samples = new Float32Array(Math.round(durationSec * SAMPLE_RATE)); - for (let i = 0; i < samples.length; i++) { - const t = i / SAMPLE_RATE; - const silent = silences.some(([from, to]) => t >= from && t < to); - // Alternating ±0.5 gives a flat, non-zero RMS without needing a real tone. - samples[i] = silent ? 0 : i % 2 === 0 ? 0.5 : -0.5; - } - return samples; -} - -const word = (w: Partial = {}): SttWordSegment => ({ - word: "w", - startSec: 0, - endSec: 0.1, - ...w, +/** A helper word; its anchor defaults to its start, as for a request's first word. */ +const word = ( + text: string, + startSec: number, + endSec: number, + anchorSec = startSec, +): HelperWord => ({ + word: text, + startSec, + endSec, + anchorSec, }); +const ms = (sec: number) => Math.round(sec * 1000) / 1000; +const times = (words: SttWordSegment[]) => words.map((w) => [w.word, ms(w.startSec), ms(w.endSec)]); -describe("snapWordBoundariesToAudio", () => { - it("pulls a late boundary back into the silence that precedes it", () => { - // Speech stops at 1.0 and resumes at 1.2; whisper reports the next word - // starting at 1.3 — 100 ms after the audio actually resumed. - const samples = audioWithSilences(3, [[1.0, 1.2]]); - const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.8 })], samples); - expect(snapped.startSec).toBeGreaterThanOrEqual(1.0); - expect(snapped.startSec).toBeLessThan(1.2); +describe("anchorWordsOnSpeech", () => { + it("returns the helper's times without speech intervals, minus the anchor", () => { + const out = anchorWordsOnSpeech([word("a", 1, 1.4, 1.2), word("b", 1.4, 1.4, 1.4)]); + expect(out).toEqual([ + { word: "a", startSec: 1, endSec: 1.4 }, + { word: "b", startSec: 1.4, endSec: 1.42 }, + ]); }); - it("leaves a boundary alone when nothing quieter precedes it", () => { - // A word ending a phrase: whisper is already right, the frames before the - // boundary are all speech, so the quietest frame in the window is the - // boundary itself and it must not drift. - const samples = audioWithSilences(3, [[1.5, 2.0]]); - const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.0, endSec: 1.5 })], samples); - expect(snapped.endSec).toBeCloseTo(1.5, 2); + it("puts a phrase's first word on its onset whether DTW put it late or early", () => { + // "Salut" opens the request, so it has no previous token and starts late, + // on its own first token. "Bah" starts where the previous token ended, at + // the end of the first phrase: in the pause, before its own speech. + const words = [ + word("Salut", 2.15, 2.5), + word("!", 2.5, 2.6, 2.6), + word("Bah", 2.6, 3.8, 3.61), + word("voilà", 3.8, 4.35, 4.01), + ]; + const speech = [ + { startSec: 1.57, endSec: 2.56 }, + { startSec: 3.36, endSec: 4.35 }, + ]; + expect(times(anchorWordsOnSpeech(words, speech))).toEqual([ + ["Salut", 1.57, 2.56], + ["!", 2.56, 2.56], + ["Bah", 3.36, 3.8], + ["voilà", 3.8, 4.35], + ]); }); - it("never moves a boundary more than the lookback window", () => { - const samples = audioWithSilences(3, [[0.0, 1.0]]); - const [snapped] = snapWordBoundariesToAudio([word({ startSec: 2.0, endSec: 2.5 })], samples); - expect(snapped.startSec).toBeGreaterThanOrEqual(2.0 - 0.15); + it("ends each phrase's last word where its speech stops, stretched or cut back", () => { + const words = [ + word("tic,", 1.95, 2.59), + word("tac", 2.59, 2.79, 2.7), + word("!", 2.79, 2.9, 2.9), + ]; + const speech = [ + { startSec: 1.95, endSec: 2.37 }, + { startSec: 2.59, endSec: 3.04 }, + ]; + expect(times(anchorWordsOnSpeech(words, speech))).toEqual([ + ["tic,", 1.95, 2.37], + ["tac", 2.59, 3.04], + ["!", 3.04, 3.04], + ]); }); - it("keeps a boundary shared by two words shared", () => { - const samples = audioWithSilences(3, [[1.0, 1.2]]); - const [first, second] = snapWordBoundariesToAudio( - [word({ startSec: 0.5, endSec: 1.3 }), word({ startSec: 1.3, endSec: 1.8 })], - samples, + it("leaves an edge alone when the word is too far past it to be that edge", () => { + // 1.2 s after the onset and 1.1 s before the offset: more likely a neighbour + // of words whisper dropped than the phrase's own edges. + const tooFar = [word("w", 2.2, 2.5)]; + expect(times(anchorWordsOnSpeech(tooFar, [{ startSec: 1, endSec: 3.6 }]))).toEqual( + times(tooFar), ); - expect(first.endSec).toBeCloseTo(second.startSec, 6); }); - it("leaves boundaries that fall outside the decoded audio alone", () => { - // Clamping these into range would collapse every boundary onto the end of - // the buffer instead of leaving the unmeasurable ones untouched. - const samples = audioWithSilences(0.1, []); - const words = [word({ startSec: 5.51, endSec: 6.85 })]; - expect(snapWordBoundariesToAudio(words, samples)).toEqual(words); + it("gives a word to the stretch its anchor falls in, the kept tail included", () => { + // "b" is anchored in the first stretch's 0.1 s tail: it closes that phrase. + // "c" starts in the first stretch (the previous token's end) but is + // anchored in the second, so it opens the second. + const words = [ + word("a", 1, 1.9, 1.5), + word("b", 1.9, 2.05, 2.05), + word("c", 2.05, 3.5, 3.2), + word("d", 3.5, 4, 3.8), + ]; + const speech = [ + { startSec: 1, endSec: 2 }, + { startSec: 3, endSec: 4 }, + ]; + expect(times(anchorWordsOnSpeech(words, speech))).toEqual([ + ["a", 1, 1.9], + ["b", 1.9, 2], + ["c", 3, 3.5], + ["d", 3.5, 4], + ]); + }); + + // Issue #948, measured on a Zira TTS clip: "…as if nothing happened. The + // transcript…" with speech stopping at 30.59 and resuming at 31.33. DTW + // anchored "happened." at 30.69, past the first stretch's tail. + const pause = [ + { startSec: 28.13, endSec: 30.59 }, + { startSec: 31.33, endSec: 35.23 }, + ]; + + it("keeps a sentence's last word in the stretch it closes when DTW anchors it in the pause", () => { + const words = [ + word("and", 27.97, 28.35, 28.35), + word("nothing", 29.65, 30.03, 30.03), + word("happened.", 30.03, 30.69, 30.69), + word("The", 30.69, 31.47, 31.47), + word("transcript", 31.47, 32.13, 32.13), + ]; + expect(times(anchorWordsOnSpeech(words, pause))).toEqual([ + ["and", 28.13, 28.35], + ["nothing", 29.65, 30.03], + ["happened.", 30.03, 30.59], + ["The", 31.33, 31.47], + ["transcript", 31.47, 32.13], + ]); + expect(ownWords(words, pause)).toEqual([ + [0, 3], + [3, 5], + ]); + }); + + it("keeps the punctuation whisper split off a sentence's last word with it", () => { + const words = [ + word("et", 28.2, 28.4, 28.4), + word("vraiment", 28.4, 30.69, 30.69), + word("?", 30.69, 30.7, 30.7), + word("Oui", 30.7, 31.6, 31.5), + ]; + expect(ownWords(words, pause)).toEqual([ + [0, 3], + [3, 4], + ]); + }); + + it("does not read an opening bracket after a sentence's last word as part of its end", () => { + // "b." is anchored in the pause, then whisper split "(" off the next word: + // "b.(" is no sentence end, so "b." would have opened the next stretch. + const words = [ + word("a", 28.2, 30.03, 30.03), + word("b.", 30.03, 30.69, 30.69), + word("(", 30.69, 31.4, 31.4), + word("c)", 31.4, 32, 32), + ]; + expect(ownWords(words, pause)).toEqual([ + [0, 2], + [2, 4], + ]); }); - it("keeps degenerate words non-empty and passes words through without audio", () => { - const samples = audioWithSilences(3, [[1.0, 1.2]]); - const [degenerate] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.3 })], samples); - expect(degenerate.endSec).toBeGreaterThan(degenerate.startSec); + it("lets a one-word sentence open a stretch when it is anchored in its speech", () => { + const words = [ + word("Agreed?", 28.13, 30.5, 30.4), + word("Yes.", 30.5, 31.6, 31.5), + word("Good", 31.6, 32, 32), + ]; + expect(ownWords(words, pause)).toEqual([ + [0, 1], + [1, 3], + ]); + }); + + it("gives a sentence's last word to the next stretch when the one before has no words", () => { + const words = [word("Done.", 31, 31.6, 31.2), word("Next", 31.6, 32, 32)]; + expect(ownWords(words, pause)).toEqual([ + [0, 0], + [0, 2], + ]); + }); - const untouched = [word({ startSec: 1.3, endSec: 1.8 })]; - expect(snapWordBoundariesToAudio(untouched, new Float32Array(0))).toEqual(untouched); + it("puts the words in order, none inverted, whatever stretch owns them", () => { + // The aligner-on shape of the report, as the aligner left it when the next + // stretch owned "happened": a 20 ms word after the pause and "The" earlier. + const aligned = [ + word("nothing", 29.68, 30.59, 30.03), + word("happened", 31.33, 31.35, 30.69), + word("The", 30.69, 31.48, 31.47), + word("transcript", 31.48, 32.12, 32.13), + ]; + const out = anchorWordsOnSpeech(aligned, pause); + for (const [j, w] of out.entries()) { + expect(w.endSec).toBeGreaterThanOrEqual(w.startSec); + if (j > 0) expect(w.startSec).toBeGreaterThanOrEqual(out[j - 1].startSec); + } + expect(out.map((w) => w.word)).toEqual(["nothing", "happened", "The", "transcript"]); }); }); diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.ts index d5d0b6f10..0cbcb6991 100644 --- a/electron/stt/snapWordBoundaries.ts +++ b/electron/stt/snapWordBoundaries.ts @@ -1,100 +1,165 @@ -// Pulls whisper.cpp's DTW word boundaries back onto the audio they describe. +// Puts the edges of every phrase on the speech the helper's VAD found. // -// whisper.cpp reports one time per token and derives word spans from it, so a -// word's `start` is the point where the decoder *emitted* the token, not where -// the speaker started saying it. Measured on real recordings, that lands -// consistently 80–150 ms late, and because consecutive words share a boundary -// (`word[i].end === word[i+1].start`) the whole transcript is dragged right by -// roughly a syllable. +// The helper times a word from the end of the token before it to the end of its +// own last token (whisper.cpp's t_dtw marks where a token ENDS, see +// whisper-stt/src/main.cpp). Inside continuous speech that lands within ~30 ms +// of the audio (measured on a TTS corpus with exact word times, +// tools/stt-eval/word-timing), so inner boundaries are left alone: an RMS snap +// on top of them would drag correct boundaries early. // -// That is invisible in captions but not in the transcript editor: deleting -// words there turns the selection into a trim of exactly -// `[firstWord.startSec, lastWord.endSec]`, so a late boundary leaves the attack -// of the first removed word audible and bites into the following kept word. +// The edges of a phrase are the exception. The token before a phrase's first +// word is the previous phrase's last one, so that word "starts" at the end of +// the previous phrase, in the pause, or DTW puts it late when the helper +// starts a new decode window there. Its last word ends on whatever token DTW +// last aligned, short of the speech or past it. When the helper sent its speech +// intervals, both are put on the edges of the stretch of speech they belong to. // -// The correction is to look at the audio instead of guessing an offset: each -// boundary moves back to the quietest 10 ms frame within the preceding -// `LOOKBACK_SEC`. It is self-limiting — on a decaying tail (a word that ends a -// phrase, where whisper is already right) the quietest frame IS the reported -// one, so the boundary doesn't move at all. +// That matters because deleting words in the transcript editor trims exactly +// `[firstWord.startSec, lastWord.endSec]`: a late boundary leaves the attack of +// the first removed word audible, an early one bites into the kept word. -import type { SttWordSegment } from "./transcriptionContract"; +import type { SttVadSegment, SttWordSegment } from "./transcriptionContract"; -/** Matches `writeSamplesAsWav` — the samples handed to whisper are mono 16 kHz. */ -const SAMPLE_RATE = 16_000; +/** A word as the helper reports it: `anchorSec` is always inside the word. */ +export interface HelperWord extends SttWordSegment { + /** End of the word's first token. Only tells which stretch of speech owns the word. */ + anchorSec: number; +} -/** RMS envelope resolution; finer than any boundary error worth correcting. */ -const FRAME_SEC = 0.01; +/** Keep degenerate words non-empty. */ +const MIN_WORD_SEC = 0.02; /** - * How far back a boundary may travel — the calibration knob for this - * correction. Sized to the measured DTW lag (~80–150 ms). Widen it and - * boundaries inside continuous speech start snapping onto the *previous* - * syllable's trough; narrow it and the lag survives. + * How far a phrase's first word may start after its speech, or its last word + * end before it, and still be stretched onto that edge — the calibration knob of + * the anchoring step. Past it, the word is more likely a neighbour of one + * whisper dropped, and stretching it over that audio would be a guess. Pulling + * a word forward to the onset, or cutting it back to where the speech stops, + * needs no such bound: the VAD says nothing is said outside it. */ -const LOOKBACK_SEC = 0.15; +const MAX_ANCHOR_SEC = 1; -/** Keep degenerate words (whisper sometimes reports end <= start) non-empty. */ -const MIN_WORD_SEC = 0.02; +/** + * Audio the helper keeps past each speech offset, as whisper.cpp does, so a soft + * ending survives. A word whose anchor falls in there belongs to the stretch it + * trails. + */ +const TAIL_SEC = 0.1; -/** Per-frame RMS of the mono signal — the cheapest usable "is this speech" proxy. */ -function rmsEnvelope(samples: Float32Array): Float32Array { - const frameLength = Math.round(SAMPLE_RATE * FRAME_SEC); - const frameCount = Math.floor(samples.length / frameLength); - const envelope = new Float32Array(frameCount); - for (let f = 0; f < frameCount; f++) { - const start = f * frameLength; - let sum = 0; - for (let i = start; i < start + frameLength; i++) { - const v = samples[i] ?? 0; - sum += v * v; - } - envelope[f] = Math.sqrt(sum / frameLength); +/** French puts a space before `!`, `?`, `:` and `;`, so whisper emits them as words. */ +const isPunctuation = (word: string) => /^[\p{P}\p{S}]+$/u.test(word); + +/** An opening bracket or quote starts what follows, so it never closes a sentence. */ +const opens = (word: string) => /^[\p{Ps}\p{Pi}]+$/u.test(word); + +/** `.`, `!`, `?`, `…` or a full-width form, maybe inside a closing quote or bracket. */ +const endsSentence = (text: string) => /[.!?…。！？．｡][\p{Pe}\p{Pf}"']*$/u.test(text); + +/** End of the audio the helper kept for stretch `i`: its speech, plus a tail short of the next. */ +const keptUntil = (speech: SttVadSegment[], i: number) => + Math.min(speech[i].endSec + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY); + +/** + * The words each speech stretch owns, as `[from, to)` ranges of `words`: those + * anchored before the end of the audio the helper kept for it. Both the aligner + * (ctcAlign.ts) and the edge anchoring below read this, so a word is aligned in + * the stretch whose edges it is put on. + * + * One exception. DTW can anchor a sentence's last word in the pause after it, + * past the kept tail: the next stretch then put it on its onset, after the + * words that follow it, and the aligner looked for it in the wrong audio (issue + * #948). So a word that ends a sentence and is anchored in the pause, before the + * next stretch's speech, closes the stretch before it, with its punctuation, + * when that stretch has words. One anchored inside the next speech still opens + * it: on LibriSpeech, nine one-word sentences ("Heaven!", "What?") open a + * stretch that way, and giving them to the stretch before cost phrase deletes. + */ +export function ownWords( + words: ReadonlyArray<{ word: string; anchorSec: number }>, + speech: SttVadSegment[], +): Array<[number, number]> { + const out: Array<[number, number]> = []; + let k = 0; + for (let i = 0; i < speech.length; i++) { + const from = k; + const until = keptUntil(speech, i); + while (k < words.length && words[k].anchorSec < until) k++; + // The word that would open the next stretch, and the punctuation after it. + let first = k; + while (first < words.length && isPunctuation(words[first].word)) first++; + let end = first + 1; + while (end < words.length && isPunctuation(words[end].word) && !opens(words[end].word)) end++; + if ( + i + 1 < speech.length && + first < words.length && + words[first].anchorSec < speech[i + 1].startSec && + words.slice(from, k).some((w) => !isPunctuation(w.word)) && + endsSentence( + words + .slice(first, end) + .map((w) => w.word) + .join(""), + ) + ) + k = end; + out.push([from, k]); } - return envelope; + return out; } /** - * Move every word boundary back to the quietest frame in the `LOOKBACK_SEC` - * preceding it. Boundaries shared by two words snap identically (same input - * time), so the transcript stays gap-free where whisper made it gap-free. - * Returns the words unchanged when there is no audio to measure against. + * Put the first and last word of every speech stretch on the stretch's edges. + * The onset already carries the VAD's 30 ms pad, so a cut there lands just + * before the attack. Words own the speech and no pause: the last word ends on + * the offset, and the punctuation closing the phrase collapses to a point + * there, wherever DTW dropped it. Without `speech` (the helper ran without its + * VAD model), the words come back as the helper timed them. */ -export function snapWordBoundariesToAudio( - words: SttWordSegment[], - samples: Float32Array, +export function anchorWordsOnSpeech( + words: HelperWord[], + speech?: SttVadSegment[], ): SttWordSegment[] { - const envelope = rmsEnvelope(samples); - if (envelope.length === 0) return words; + const out: SttWordSegment[] = words.map(({ anchorSec: _, ...w }) => ({ + ...w, + endSec: Math.max(w.endSec, w.startSec + MIN_WORD_SEC), + })); + if (speech) anchorEdges(out, words, speech); + // Whatever the stretches did, words come out in order and never inverted. A + // start past the next word's comes back to it rather than pushing the next + // one: the onset clamp is what overshoots (a word whisper invented, put on + // the onset after the real one the aligner found earlier). + for (let j = out.length - 1; j >= 0; j--) { + if (j + 1 < out.length) out[j].startSec = Math.min(out[j].startSec, out[j + 1].startSec); + out[j].endSec = Math.max(out[j].endSec, out[j].startSec); + } + return out; +} - const lookbackFrames = Math.round(LOOKBACK_SEC / FRAME_SEC); - const snap = (timeSec: number): number => { - const hi = Math.round(timeSec / FRAME_SEC); - // A boundary outside the audio we can measure (whisper occasionally reports - // times past the end of the samples) has nothing to snap to — clamping it - // into range would drag it to the end of the buffer instead. - if (hi <= 0 || hi >= envelope.length) return timeSec; - const lo = Math.max(0, hi - lookbackFrames); - let bestFrame = hi; - let bestRms = envelope[hi]; - // Strict `<` while walking backwards keeps the frame CLOSEST to the - // reported boundary when a whole stretch is equally quiet, so digital - // silence can't drag a boundary the full window. - for (let f = hi - 1; f >= lo; f--) { - if (envelope[f] < bestRms) { - bestRms = envelope[f]; - bestFrame = f; +function anchorEdges(out: SttWordSegment[], words: HelperWord[], speech: SttVadSegment[]) { + const owned = ownWords(words, speech); + for (let i = 0; i < speech.length; i++) { + const { startSec: onset, endSec: offset } = speech[i]; + let first = -1; + let last = -1; + for (let k = owned[i][0]; k < owned[i][1]; k++) { + if (isPunctuation(out[k].word)) continue; + if (first < 0) first = k; + last = k; + } + if (first < 0) continue; + if (out[first].startSec - onset <= MAX_ANCHOR_SEC) { + out[first].startSec = onset; + out[first].endSec = Math.max(out[first].endSec, onset + MIN_WORD_SEC); + for (let j = first - 1; j >= 0 && out[j].endSec > onset; j--) { + out[j].endSec = onset; + out[j].startSec = Math.min(out[j].startSec, onset); } } - return bestFrame * FRAME_SEC; - }; - - return words.map((w) => { - const startSec = snap(w.startSec); - return { - ...w, - startSec, - endSec: Math.max(snap(w.endSec), startSec + MIN_WORD_SEC), - }; - }); + if (offset - out[last].endSec > MAX_ANCHOR_SEC) continue; + out[last].endSec = Math.max(offset, out[last].startSec + MIN_WORD_SEC); + for (let j = last + 1; j < out.length && isPunctuation(out[j].word); j++) { + out[j].startSec = out[last].endSec; + out[j].endSec = out[last].endSec; + } + } } diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.ts index c9b7d4850..4a2563a78 100644 --- a/electron/stt/transcriptionContract.ts +++ b/electron/stt/transcriptionContract.ts @@ -141,7 +141,17 @@ export interface SttTranscribeRequest { * a literal typed twice is a fallback that silently stops working. */ export const STT_NATIVE_EXTRACTION_UNAVAILABLE = "stt:native-extraction-unavailable"; -export const STT_VAD_UNAVAILABLE = "stt:vad-unavailable"; + +/** + * Marker carried in the error message when audio extraction could not read the + * `sourcePath` itself: the OS refused the read, or ffmpeg hung opening the file + * (macOS holds the read behind a pending folder-access prompt, issue #968). + * + * Only `extractAudio.ts` sets it, so the renderer can tell "this file's folder is + * not readable" from an engine failure that happens to mention a permission (model + * loading, the whisper helper). Same IPC reason as above for being a string. + */ +export const STT_MEDIA_UNREADABLE = "stt:media-unreadable"; /** IPC response: main → renderer. */ export interface SttTranscribeResponse { @@ -165,8 +175,3 @@ export interface SttVadSegment { startSec: number; endSec: number; } - -/** Response from the VAD speech detection pass. */ -export interface SttVadResponse { - segments: SttVadSegment[]; -} diff --git a/electron/stt/whisperServer.test.ts b/electron/stt/whisperServer.test.ts index 314926e58..773235577 100644 --- a/electron/stt/whisperServer.test.ts +++ b/electron/stt/whisperServer.test.ts @@ -3,9 +3,8 @@ import { mkdtemp, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { STT_VAD_UNAVAILABLE } from "./transcriptionContract"; import { writeSamplesAsWav } from "./wav"; -import { MAX_VAD_CHUNK_SAMPLES, mergeVadIntervals, WhisperServerManager } from "./whisperServer"; +import { WhisperServerManager } from "./whisperServer"; vi.mock("node:child_process", async (importOriginal) => { // `node:child_process` is CJS, so the ESM namespace also carries a `default` @@ -350,6 +349,148 @@ describe("WhisperServerManager", () => { } }); + it("anchors a phrase's first word on the onset of the speech its anchor falls in", async () => { + // "Salut" starts where the previous token ended, in the first stretch's + // tail, but its anchor puts it in the second stretch. + const fakeJson = { + segments: [ + { + text: " Salut", + start: 1.57, + end: 2.56, + words: [{ word: " Salut", start: 1.05, end: 2.56, anchor: 1.8 }], + }, + ], + speech: [ + { start: 0.2, end: 1 }, + { start: 1.57, end: 2.56 }, + ], + backend: "whispercpp-cpu", + }; + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(JSON.stringify(fakeJson), { status: 200 })), + ); + try { + const mgr = new WhisperServerManager(); + (mgr as unknown as { process: unknown; port: number }).process = {}; + (mgr as unknown as { process: unknown; port: number }).port = 9999; + const result = await mgr.transcribe({ samples: new Float32Array(16_000 * 3) }); + expect(result.wordSegments).toEqual([{ word: "Salut", startSec: 1.57, endSec: 2.56 }]); + } finally { + vi.unstubAllGlobals(); + } + }); + + describe("CTC word aligner", () => { + const inference = { + segments: [ + { + text: " ab ba", + start: 0.2, + end: 1, + words: [ + { word: " ab", start: 0.3, end: 0.5, anchor: 0.4 }, + { word: " ba", start: 0.5, end: 0.8, anchor: 0.6 }, + ], + }, + ], + speech: [{ start: 0.2, end: 1 }], + detected_language: "fr", + backend: "whispercpp-vulkan", + timing: { elapsed_s: 0.5, audio_s: 2, rtf: 0.25 }, + }; + // "ab" spoken on frames 20-21, the word delimiter on 22, "ba" on 24-25. + const vocab = ["", "|", "a", "b"]; + const frames = Array(60).fill(0); + frames[20] = 2; + frames[21] = 3; + frames[22] = 1; + frames[24] = 3; + frames[25] = 2; + const logprobs = new Float32Array(60 * vocab.length).fill(-12); + frames.forEach((tok, f) => { + logprobs[f * vocab.length + tok] = -0.01; + }); + const emissions = { + vocab, + blank: 0, + stride_s: 0.02, + elapsed_s: 0.1, + regions: [ + { start: 0, frames: 60, logprobs: Buffer.from(logprobs.buffer).toString("base64") }, + ], + }; + + async function run(emissionsReply: Response, reply: object = inference) { + const fetchMock = vi.fn(async (url: string, _init?: RequestInit) => + url.endsWith("/emissions") + ? emissionsReply + : new Response(JSON.stringify(reply), { status: 200 }), + ); + vi.stubGlobal("fetch", fetchMock); + try { + const mgr = new WhisperServerManager(); + (mgr as unknown as { process: unknown; port: number }).process = {}; + (mgr as unknown as { process: unknown; port: number }).port = 9999; + const alignerFor = vi.fn(async (language: string) => `/models/${language}.gguf`); + const result = await mgr.transcribe({ + samples: new Float32Array(16_000 * 2), + alignerFor, + }); + return { result, alignerFor, fetchMock }; + } finally { + vi.unstubAllGlobals(); + } + } + + it("re-times the words on the aligner of the detected language, and counts its cost", async () => { + const { result, alignerFor, fetchMock } = await run( + new Response(JSON.stringify(emissions), { status: 200 }), + ); + expect(alignerFor).toHaveBeenCalledWith("fr"); + const body = fetchMock.mock.calls[1][1]?.body as FormData; + expect(body.get("model")).toBe("/models/fr.gguf"); + expect(JSON.parse(String(body.get("regions")))).toEqual([[0, 1.3]]); + // The phrase edges stay on the speech; the inner boundary moves to the letters. + const ms = (x: number) => Math.round(x * 1000); + expect(result.wordSegments.map((w) => [w.word, ms(w.startSec), ms(w.endSec)])).toEqual([ + ["ab", 200, 450], + ["ba", 450, 1000], + ]); + expect(result.timing?.elapsedSec).toBeCloseTo(0.6); + expect(result.timing?.rtf).toBeCloseTo(0.3); + }); + + it("leaves the aligner out on the CPU, without even asking for it", async () => { + const { result, alignerFor, fetchMock } = await run( + new Response(JSON.stringify(emissions), { status: 200 }), + { ...inference, backend: "whispercpp-cpu" }, + ); + expect(alignerFor).not.toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalledOnce(); + expect(result.wordSegments.map((w) => [w.word, w.startSec, w.endSec])).toEqual([ + ["ab", 0.2, 0.5], + ["ba", 0.5, 1], + ]); + }); + + it("keeps whisper's times when the aligner fails", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + try { + const { result } = await run(new Response("not found", { status: 404 })); + expect(result.wordSegments.map((w) => [w.word, w.startSec, w.endSec])).toEqual([ + ["ab", 0.2, 0.5], + ["ba", 0.5, 1], + ]); + expect(result.timing?.elapsedSec).toBe(0.5); + expect(warn).toHaveBeenCalledWith(expect.stringMatching(/word aligner failed/)); + } finally { + warn.mockRestore(); + } + }); + }); + it("spawns whisper-stt-server with --model", async () => { const fs = await import("node:fs/promises"); const { spawn } = await import("node:child_process"); @@ -729,175 +870,4 @@ describe("WhisperServerManager", () => { await rm(dir, { recursive: true, force: true }); } }); - - it("detectVadSegments extracts segments from /vad response", async () => { - const fakeJson = { - segments: [ - { start: 0.25, end: 1.5 }, - { start: 2.1, end: 4.8 }, - ], - }; - vi.stubGlobal( - "fetch", - vi.fn(async () => new Response(JSON.stringify(fakeJson), { status: 200 })), - ); - try { - const mgr = new WhisperServerManager(); - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {}; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable = - true; - - const segments = await mgr.detectVadSegments({ samples: new Float32Array(1600) }); - expect(segments).toEqual([ - { startSec: 0.25, endSec: 1.5 }, - { startSec: 2.1, endSec: 4.8 }, - ]); - } finally { - vi.unstubAllGlobals(); - } - }); - - it("detectVadSegments surfaces server error from /vad endpoint", async () => { - vi.stubGlobal( - "fetch", - vi.fn(async () => new Response("VAD model was not loaded", { status: 400 })), - ); - try { - const mgr = new WhisperServerManager(); - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {}; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable = - true; - - await expect(mgr.detectVadSegments({ samples: new Float32Array(1600) })).rejects.toThrow( - /whisper-stt-server \/vad HTTP 400/, - ); - } finally { - vi.unstubAllGlobals(); - } - }); - - it("filters out /vad segments missing end that map to zero-length intervals", async () => { - const fakeJson = { - segments: [ - { start: 1.5 }, // missing end -> maps to startSec: 1.5, endSec: 1.5 - ], - }; - vi.stubGlobal( - "fetch", - vi.fn(async () => new Response(JSON.stringify(fakeJson), { status: 200 })), - ); - try { - const mgr = new WhisperServerManager(); - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {}; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable = - true; - - const segments = await mgr.detectVadSegments({ samples: new Float32Array(1600) }); - expect(segments).toEqual([]); - } finally { - vi.unstubAllGlobals(); - } - }); - - it("handles malformed WAV /vad HTTP 400 without crashing helper availability", async () => { - vi.stubGlobal( - "fetch", - vi.fn( - async () => new Response(JSON.stringify({ error: "failed to parse WAV" }), { status: 400 }), - ), - ); - try { - const mgr = new WhisperServerManager(); - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = { - pid: 999, - }; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable = - true; - - await expect(mgr.detectVadSegments({ samples: new Float32Array(100) })).rejects.toThrow( - /whisper-stt-server \/vad HTTP 400/, - ); - expect(mgr.status.running).toBe(true); - } finally { - vi.unstubAllGlobals(); - } - }); - - it("rejects detectVadSegments when VAD is unavailable", async () => { - const mgr = new WhisperServerManager(); - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {}; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable = - false; - - await expect(mgr.detectVadSegments({ samples: new Float32Array(1600) })).rejects.toThrow( - STT_VAD_UNAVAILABLE, - ); - }); - - describe("mergeVadIntervals", () => { - it("returns empty array for empty input", () => { - expect(mergeVadIntervals([])).toEqual([]); - }); - - it("returns single interval as-is", () => { - expect(mergeVadIntervals([{ startSec: 1, endSec: 2 }])).toEqual([{ startSec: 1, endSec: 2 }]); - }); - - it("merges overlapping and adjacent intervals", () => { - const input = [ - { startSec: 1, endSec: 3 }, - { startSec: 2.5, endSec: 5 }, - { startSec: 5, endSec: 6.2 }, - { startSec: 8, endSec: 10 }, - ]; - expect(mergeVadIntervals(input)).toEqual([ - { startSec: 1, endSec: 6.2 }, - { startSec: 8, endSec: 10 }, - ]); - }); - - it("drops inverted intervals where endSec <= startSec", () => { - const input = [ - { startSec: 2, endSec: 1 }, - { startSec: 3, endSec: 4 }, - ]; - expect(mergeVadIntervals(input)).toEqual([{ startSec: 3, endSec: 4 }]); - }); - - it("drops singleton intervals where endSec <= startSec", () => { - expect(mergeVadIntervals([{ startSec: 2, endSec: 1 }])).toEqual([]); - expect(mergeVadIntervals([{ startSec: 2, endSec: 2 }])).toEqual([]); - }); - }); - - it("chunks long audio across MAX_VAD_CHUNK_SAMPLES and offsets timestamps", async () => { - const fetchMock = vi.fn(async () => { - const callIndex = fetchMock.mock.calls.length - 1; - const segments = callIndex === 0 ? [{ start: 10, end: 180 }] : [{ start: 0, end: 20 }]; - return new Response(JSON.stringify({ segments }), { status: 200 }); - }); - vi.stubGlobal("fetch", fetchMock); - try { - const mgr = new WhisperServerManager(); - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {}; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable = - true; - - // Total duration = 200 seconds @ 16 kHz = 3,200,000 samples (> MAX_VAD_CHUNK_SAMPLES = 2,880,000) - const samples = new Float32Array(MAX_VAD_CHUNK_SAMPLES + 16_000 * 20); - const segments = await mgr.detectVadSegments({ samples }); - - expect(fetchMock).toHaveBeenCalledTimes(2); - // 180s in chunk 0 + 20s in chunk 1 with offset 180s -> merged [10, 200] - expect(segments).toEqual([{ startSec: 10, endSec: 200 }]); - } finally { - vi.unstubAllGlobals(); - } - }); }); diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.ts index a9698674f..2543a79bb 100644 --- a/electron/stt/whisperServer.ts +++ b/electron/stt/whisperServer.ts @@ -6,15 +6,19 @@ import os from "node:os"; import path from "node:path"; import type { Readable } from "node:stream"; -import { resolveBinaryPath } from "./gpuDetector"; -import { snapWordBoundariesToAudio } from "./snapWordBoundaries"; import { - STT_VAD_UNAVAILABLE, - type SttBackend, - type SttPhraseSegment, - type SttTiming, - type SttVadSegment, - type SttWordSegment, + alignWordsOnEmissions, + type EmissionsJson, + emissionRegions, + parseEmissions, +} from "./ctcAlign"; +import { resolveBinaryPath } from "./gpuDetector"; +import { anchorWordsOnSpeech } from "./snapWordBoundaries"; +import type { + SttBackend, + SttPhraseSegment, + SttTiming, + SttWordSegment, } from "./transcriptionContract"; import { cleanupWav, writeSamplesAsWav } from "./wav"; @@ -76,7 +80,13 @@ export interface WhisperServerStatus { backend: SttBackend | null; startedAtMs: number | null; lastError: string | null; - vadAvailable: boolean; +} + +export interface TranscribeOptions { + samples: Float32Array; + language?: string; + /** The aligner model for a language, or null for none (see `transcribe`). */ + alignerFor?: (language: string) => Promise; } /** Per-word entry inside a whisper-stt-server `/inference` JSON segment. */ @@ -84,6 +94,8 @@ interface WhisperJsonWord { word?: string; start?: number; end?: number; + /** End of the word's first token: always inside the word (see snapWordBoundaries.ts). */ + anchor?: number; probability?: number; } @@ -115,6 +127,8 @@ interface WhisperJsonResponse { detected_language?: string; backend?: string; timing?: WhisperJsonTiming; + /** Silero VAD's speech intervals; absent when the helper runs without the VAD model. */ + speech?: Array<{ start?: number | string; end?: number | string }>; } export class WhisperServerManager { @@ -125,11 +139,7 @@ export class WhisperServerManager { private lastError: string | null = null; private startedAtMs: number | null = null; private inFlight: Promise = Promise.resolve(); - private starting: Promise<{ - port: number; - backend: SttBackend; - vadAvailable: boolean; - }> | null = null; + private starting: Promise<{ port: number; backend: SttBackend }> | null = null; /** Buffered stderr from the helper; surfaced on shutdown + poll failures. */ private stderrTail = ""; @@ -154,12 +164,12 @@ export class WhisperServerManager { }); } - /** Check the server's HTTP root for a 200; resolves once responsive. Returns VAD availability. */ + /** Check the server's HTTP root for a 200; resolves once responsive. */ private static async pollUntilReady( baseUrl: string, timeoutMs = 30_000, shouldContinue: () => boolean = () => true, - ): Promise<{ vad: boolean }> { + ): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { if (!shouldContinue()) throw new Error("whisper-stt-server exited before readiness"); @@ -170,10 +180,7 @@ export class WhisperServerManager { ); try { const res = await fetch(baseUrl, { method: "GET", signal: controller.signal }); - if (res.ok) { - const body = typeof res.json === "function" ? await res.json().catch(() => null) : null; - return { vad: Boolean((body as { vad?: boolean } | null)?.vad) }; - } + if (res.ok) return; } catch { // not up yet } finally { @@ -208,8 +215,6 @@ export class WhisperServerManager { if (log) console.error(`[stt] ${message}`); } - private vadAvailable = false; - /** True when a process is alive and a model is loaded. */ get status(): WhisperServerStatus { return { @@ -219,22 +224,15 @@ export class WhisperServerManager { backend: this.backend, startedAtMs: this.startedAtMs, lastError: this.lastError, - vadAvailable: this.vadAvailable, }; } - get isVadAvailable(): boolean { - return this.vadAvailable; - } - /** * Spawn the helper if not running and return once `/` returns 200. Idempotent — * if a server is already up we just return its port so the caller never pays * the cold-start cost twice. */ - async start( - options: WhisperServerStartOptions, - ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> { + async start(options: WhisperServerStartOptions): Promise<{ port: number; backend: SttBackend }> { if (this.starting) return this.starting; this.starting = this.startImpl(options).finally(() => { this.starting = null; @@ -244,7 +242,7 @@ export class WhisperServerManager { private async startImpl( options: WhisperServerStartOptions, - ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> { + ): Promise<{ port: number; backend: SttBackend }> { if (this.shuttingDown) { throw new Error("whisper-stt-server manager is shutting down"); } @@ -252,7 +250,6 @@ export class WhisperServerManager { return { port: this.port, backend: this.backend ?? options.backend ?? "whispercpp-cpu", - vadAvailable: this.vadAvailable, }; } @@ -281,9 +278,7 @@ export class WhisperServerManager { throw new Error(`Whisper GGML model not found at ${options.modelPath}`); } - const launch = async ( - forceCpu: boolean, - ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> => { + const launch = async (forceCpu: boolean): Promise<{ port: number; backend: SttBackend }> => { const port = await WhisperServerManager.pickFreePort(); if (this.shuttingDown) { throw new Error("whisper-stt-server manager is shutting down"); @@ -331,7 +326,6 @@ export class WhisperServerManager { this.process = null; this.port = null; this.startedAtMs = null; - this.vadAvailable = false; } }); child.once("error", (err) => { @@ -347,7 +341,6 @@ export class WhisperServerManager { this.process = null; this.port = null; this.startedAtMs = null; - this.vadAvailable = false; } }); @@ -362,9 +355,8 @@ export class WhisperServerManager { }); child.once("error", reject); }); - let readyInfo: { vad: boolean }; try { - readyInfo = await Promise.race([ + await Promise.race([ WhisperServerManager.pollUntilReady( `http://127.0.0.1:${port}`, 30_000, @@ -384,8 +376,7 @@ export class WhisperServerManager { this.recordError(message, { log: !alreadyLogged }); throw new Error(message); } - this.vadAvailable = Boolean(readyInfo?.vad); - return { port, backend: activeBackend, vadAvailable: this.vadAvailable }; + return { port, backend: activeBackend }; }; try { @@ -411,7 +402,6 @@ export class WhisperServerManager { /** Send SIGTERM and wait for the helper to exit. Resolves even if it was already down. */ async stop(): Promise { - this.vadAvailable = false; if (!this.process) { this.port = null; this.startedAtMs = null; @@ -464,6 +454,33 @@ export class WhisperServerManager { } } + /** + * The CTC aligner's letter scores for `regions` of the same upload + * (`/emissions`, see ctcAlign.ts). Throws on any failure, including a helper + * built before the endpoint existed (404): the caller keeps whisper's times. + */ + private async runEmissions(opts: { + wavPath: string; + modelPath: string; + regions: Array<[number, number]>; + }): Promise { + const form = new FormData(); + const fileBuffer = await readFile(opts.wavPath); + form.set("file", new Blob([fileBuffer], { type: "audio/wav" }), path.basename(opts.wavPath)); + form.set("model", opts.modelPath); + form.set("regions", JSON.stringify(opts.regions)); + const res = await fetch(`${this.baseUrl()}/emissions`, { + method: "POST", + body: form, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`whisper-stt-server /emissions HTTP ${res.status}: ${text.slice(0, 512)}`); + } + return (await res.json()) as EmissionsJson; + } + private async runMultipartInfer(opts: { wavPath: string; language?: string; @@ -564,8 +581,16 @@ export class WhisperServerManager { return { elapsedSec, audioSec, rtf }; } - /** Run one transcription; serializes concurrent callers. */ - async transcribe(opts: { samples: Float32Array; language?: string }): Promise<{ + /** + * Run one transcription; serializes concurrent callers. + * + * `alignerFor` resolves the CTC aligner model for the language whisper + * detected, or null when there is none or it is not ready yet (it must not + * wait on the network: no download runs inside a chunk, see SttManager). With one, + * the words are re-timed on it before their phrase edges go on the speech + * (ctcAlign.ts); any failure there keeps whisper's own times. + */ + async transcribe(opts: TranscribeOptions): Promise<{ segments: SttPhraseSegment[]; wordSegments: SttWordSegment[]; detectedLanguage: string; @@ -578,7 +603,7 @@ export class WhisperServerManager { return task; } - private async transcribeImpl(opts: { samples: Float32Array; language?: string }): Promise<{ + private async transcribeImpl(opts: TranscribeOptions): Promise<{ segments: SttPhraseSegment[]; wordSegments: SttWordSegment[]; detectedLanguage: string; @@ -597,142 +622,69 @@ export class WhisperServerManager { return { text, startSec, endSec: Math.max(endSec, startSec + 0.05) }; }) .filter((s) => s.text.length > 0); - // whisper.cpp's DTW boundaries run ~80–150 ms behind the audio, which the - // transcript editor turns into imprecise trims (see snapWordBoundaries.ts). - // Re-anchor them on the same samples whisper was given. - const wordSegments: SttWordSegment[] = snapWordBoundariesToAudio( - raw - .flatMap((seg) => - (seg.words ?? []).map((w) => { - const word = (w.word ?? "").trim(); - const startSec = this.toSec(w.start, 0); - const endSec = this.toSec(w.end, startSec + 0.05); - const confidence = typeof w.probability === "number" ? w.probability : undefined; - return { word, startSec, endSec: Math.max(startSec + 0.02, endSec), confidence }; - }), - ) - .filter((w) => w.word.length > 0), - opts.samples, - ); + const speech = json.speech?.map((s) => ({ + startSec: this.toSec(s.start, 0), + endSec: this.toSec(s.end, 0), + })); const detectedLanguage = json.detected_language ?? json.language ?? "auto"; + let words = raw + .flatMap((seg) => + (seg.words ?? []).map((w) => { + const word = (w.word ?? "").trim(); + const startSec = this.toSec(w.start, 0); + const endSec = this.toSec(w.end, startSec + 0.05); + const anchorSec = this.toSec(w.anchor, startSec); + const confidence = typeof w.probability === "number" ? w.probability : undefined; + return { + word, + startSec, + endSec: Math.max(startSec + 0.02, endSec), + anchorSec, + confidence, + }; + }), + ) + .filter((w) => w.word.length > 0); + let alignSec = 0; const backend = this.toBackend(json.backend); - const timing = this.toTiming(json.timing); - return { segments, wordSegments, detectedLanguage, backend, timing }; - } finally { - await cleanupWav(wavPath); - } - } - - /** Run Voice Activity Detection (Silero VAD) to detect speech intervals. */ - async detectVadSegments(opts: { samples: Float32Array }): Promise { - const task = this.inFlight.then(() => this.detectVadSegmentsImpl(opts)); - this.inFlight = task.catch(() => undefined); - return task; - } - - private async detectVadChunk(samples: Float32Array): Promise { - const wavPath = await writeSamplesAsWav(samples); - try { - const url = `${this.baseUrl()}/vad`; - const form = new FormData(); - const fileBuffer = await readFile(wavPath); - const blob = new Blob([fileBuffer], { type: "audio/wav" }); - form.set("file", blob, path.basename(wavPath)); - - let res: Response; - try { - res = await fetch(url, { - method: "POST", - body: form, - signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), - }); - } catch (error) { - throw Object.assign( - new Error( - `whisper-stt-server /vad failed: ` + - `${error instanceof Error ? error.message : String(error)}; ` + - `stderr=${this.stderrTail.slice(-256)}`, - ), - { cause: error }, - ); + // GPU only. On the CPU the aligner's forward pass costs +29% (English) to + // +58% (French) of a transcription, against +9% to +14% on Vulkan, and + // character-level DTW alone is already within 16 ms median there + // (tools/stt-eval/word-timing, issue #948). Not asking also means a + // CPU-only machine never downloads the model. + if (speech?.length && words.length && opts.alignerFor && backend !== "whispercpp-cpu") { + try { + const modelPath = await opts.alignerFor(detectedLanguage); + if (modelPath) { + const reply = await this.runEmissions({ + wavPath, + modelPath, + regions: emissionRegions(speech, opts.samples.length / 16_000), + }); + const emissions = parseEmissions(reply); + if (!emissions) throw new Error("unreadable /emissions reply"); + words = alignWordsOnEmissions(words, speech, emissions); + alignSec = this.toSec(reply.elapsed_s, 0); + } + } catch (error) { + console.warn( + `[stt] word aligner failed, keeping whisper's word times: ${error instanceof Error ? error.message : String(error)}`, + ); + } } - - if (!res.ok) { - const text = await res.text().catch(() => ""); - throw new Error(`whisper-stt-server /vad HTTP ${res.status}: ${text.slice(0, 512)}`); + // Phrase edges go on the helper's speech intervals, aligner or not: the + // VAD is the better judge of where speech starts after a pause (see + // snapWordBoundaries.ts). + const wordSegments: SttWordSegment[] = anchorWordsOnSpeech(words, speech); + const timing = this.toTiming(json.timing); + // The aligner is part of what this chunk cost. + if (timing && alignSec > 0) { + timing.elapsedSec += alignSec; + timing.rtf = timing.elapsedSec / timing.audioSec; } - - const json = (await res.json().catch((error: unknown) => { - throw Object.assign( - new Error( - `whisper-stt-server /vad response was unreadable: ` + - `${error instanceof Error ? error.message : String(error)}`, - ), - { cause: error }, - ); - })) as { segments?: Array<{ start?: number | string; end?: number | string }> }; - - const raw = json.segments ?? []; - return raw.map((seg) => { - const startSec = this.toSec(seg.start, 0); - const endSec = this.toSec(seg.end, startSec); - return { startSec, endSec }; - }); + return { segments, wordSegments, detectedLanguage, backend, timing }; } finally { await cleanupWav(wavPath); } } - - private async detectVadSegmentsImpl(opts: { samples: Float32Array }): Promise { - await this.ensureReady(); - if (!this.vadAvailable) { - throw new Error(STT_VAD_UNAVAILABLE); - } - if (opts.samples.length === 0) return []; - - const allSegments: SttVadSegment[] = []; - const totalSamples = opts.samples.length; - for (let offset = 0; offset < totalSamples; offset += MAX_VAD_CHUNK_SAMPLES) { - const chunkSamples = opts.samples.subarray( - offset, - Math.min(totalSamples, offset + MAX_VAD_CHUNK_SAMPLES), - ); - const chunkSegments = await this.detectVadChunk(chunkSamples); - const offsetSec = offset / 16_000; - for (const seg of chunkSegments) { - allSegments.push({ - startSec: Math.round((seg.startSec + offsetSec) * 1000) / 1000, - endSec: Math.round((seg.endSec + offsetSec) * 1000) / 1000, - }); - } - } - - return mergeVadIntervals(allSegments); - } -} - -/** - * Silero VAD processes audio quickly, but buffering hours of 16 kHz audio as - * single WAV blobs can exhaust Node memory. We bound individual VAD passes to - * chunks of at most 3 minutes (180s = 2,880,000 samples @ 16 kHz), keeping - * memory under ~6 MB per WAV, and merge the resulting intervals. - */ -export const MAX_VAD_CHUNK_SAMPLES = 16_000 * 180; - -/** - * Merges contiguous or overlapping speech segments into non-overlapping intervals. - */ -export function mergeVadIntervals(segments: SttVadSegment[]): SttVadSegment[] { - if (segments.length <= 1) return segments.filter((seg) => seg.endSec > seg.startSec); - const merged: SttVadSegment[] = []; - for (const seg of segments) { - if (seg.endSec <= seg.startSec) continue; - const last = merged[merged.length - 1]; - if (last && seg.startSec <= last.endSec) { - last.endSec = Math.max(last.endSec, seg.endSec); - } else { - merged.push({ startSec: seg.startSec, endSec: seg.endSec }); - } - } - return merged; } diff --git a/electron/windows.ts b/electron/windows.ts index 0faaef02f..72f2fe938 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -505,6 +505,10 @@ export function createEditorWindow(query: Record = {}): BrowserW contextIsolation: true, webSecurity: false, backgroundThrottling: false, + // Media track lists: the preview's hidden

[--bin ] +// +// --source id of an existing project with at least two cameras on its first clip +// (e.g. proj_7bc8…). It is NOT modified: the run copies it to a new project id +// with a fresh `updatedAt`, so the editor opens the copy, and deletes the copy at +// the end. Back up %APPDATA%\openscreen\projects first anyway: the editor may +// touch other state while it runs. +// --out directory for screenshots, the Electron log and results.json. +// --bin native bin dir holding compositor_view.node and the ffmpeg DLLs +// (default electron/native/bin/win32-). +// +// The app is launched with the repo root as its entry (not dist-electron/main.js, which would +// move userData to Roaming\Electron) and with OPENSCREEN_DISABLE_CONTENT_PROTECTION=1. +// Exit code 1 when any check failed. +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { _electron as electron } from "playwright"; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +function arg(name, fallback) { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : fallback; +} + +const SOURCE_ID = arg("source"); +const OUT = arg("out"); +const BIN = arg("bin", path.join(ROOT, "electron", "native", "bin", `win32-${process.arch}`)); +if (!SOURCE_ID || !OUT) { + console.error("usage: node scripts/editor-multicam-smoke.mjs --source --out "); + process.exit(2); +} +fs.mkdirSync(OUT, { recursive: true }); + +const PROJECTS = path.join(process.env.APPDATA ?? "", "openscreen", "projects"); +const COPY_ID = `proj_${randomUUID()}`; +const COPY_FILE = path.join(PROJECTS, `${COPY_ID}.openscreen`); + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +const results = []; +function note(name, ok, detail = "") { + results.push({ name, ok, detail }); + console.log(`${ok ? "PASS" : "FAIL"} ${name} ${detail}`); +} + +function readDoc() { + return JSON.parse(fs.readFileSync(COPY_FILE, "utf8")); +} +function cameraState(doc = readDoc()) { + const legacy = doc.legacyEditor ?? {}; + return { + layouts: legacy.cameraLayoutRegions ?? [], + fulls: legacy.cameraFullscreenRegions ?? [], + settings: legacy.cameraSettings ?? null, + }; +} +// Saves are asynchronous: poll the file until the predicate holds or the time is up. +async function waitState(predicate, timeoutMs = 8000) { + const end = Date.now() + timeoutMs; + let state = cameraState(); + while (Date.now() < end) { + state = cameraState(); + try { + if (predicate(state)) return { ok: true, state }; + } catch { + // a half-written state: keep polling + } + await sleep(200); + } + return { ok: false, state }; +} +/** + * The part of a camera section the viewer sees, in ruler seconds: its span minus the trims of + * its clip (trims are in source seconds; a clip maps source to ruler time by a fixed offset). + * The longest remaining piece, or null when a trim hides the whole section. + */ +function visibleSpan(row, doc) { + const clips = doc.timeline.clips; + const clip = + clips.find((c) => c.id === row.clipId) ?? + clips.find( + (c) => c.timelineStartSec * 1000 <= row.startMs && row.startMs < c.timelineEndSec * 1000, + ); + let pieces = [[row.startMs / 1000, row.endMs / 1000]]; + for (const trim of doc.timeline.trimRanges ?? []) { + if (!clip || trim.assetId !== clip.assetId) continue; + if (trim.clipId && trim.clipId !== clip.id) continue; + const offset = clip.timelineStartSec - clip.sourceStartSec; + const [cutStart, cutEnd] = [trim.startSec + offset, trim.endSec + offset]; + pieces = pieces + .flatMap(([start, end]) => [ + [start, Math.min(end, cutStart)], + [Math.max(start, cutEnd), end], + ]) + .filter(([start, end]) => end - start > 1e-3); + } + pieces.sort((a, b) => b[1] - b[0] - (a[1] - a[0])); + return pieces[0] ?? null; +} +/** Where a steady-state capture of a section goes: the middle of its visible span. */ +function steadyMid(row, doc = readDoc()) { + const span = visibleSpan(row, doc); + return span ? (span[0] + span[1]) / 2 : (row.startMs + row.endMs) / 2000; +} +/** The first time from `fromSec` on with `lengthSec` free of the first clip's trims. */ +function clearOfTrims(fromSec, lengthSec, doc = readDoc()) { + const clip = doc.timeline.clips[0]; + const offset = clip.timelineStartSec - clip.sourceStartSec; + const trims = [...(doc.timeline.trimRanges ?? [])].sort((a, b) => a.startSec - b.startSec); + let at = fromSec; + for (const trim of trims) { + if (trim.assetId !== clip.assetId || (trim.clipId && trim.clipId !== clip.id)) continue; + const [cutStart, cutEnd] = [trim.startSec + offset, trim.endSec + offset]; + if (at < cutEnd && at + lengthSec > cutStart) at = Math.ceil(cutEnd) + 1; + } + return at; +} + +const brief = (rows) => + rows + .map( + (r) => + `${r.template ?? "full"}[${(r.slots ?? []).map((s) => s.camera).join(",")}]@${( + r.startMs / 1000 + ).toFixed(2)}-${(r.endMs / 1000).toFixed(2)}`, + ) + .join(" "); + +// 1. The copy the editor opens: newest `updatedAt`, fresh id, otherwise the source verbatim. +const source = JSON.parse(fs.readFileSync(path.join(PROJECTS, `${SOURCE_ID}.openscreen`), "utf8")); +source.project.id = COPY_ID; +source.project.title = `Multi-camera smoke ${new Date().toISOString()}`; +source.project.updatedAt = new Date().toISOString(); +fs.writeFileSync(COPY_FILE, JSON.stringify(source)); +console.log(`copy: ${COPY_FILE}`); + +const log = fs.createWriteStream(path.join(OUT, "electron.log")); +const consoleErrors = []; + +async function launch() { + const app = await electron.launch({ + args: [ROOT, "--lang=en-US"], + cwd: ROOT, + env: { + ...process.env, + OPENSCREEN_COMPOSITOR_VIEW_NODE: path.join(BIN, "compositor_view.node"), + OPENSCREEN_DISABLE_CONTENT_PROTECTION: "1", + PATH: `${BIN};${process.env.PATH}`, + }, + timeout: 60_000, + }); + const proc = app.process(); + proc.stdout?.on("data", (d) => log.write(d)); + proc.stderr?.on("data", (d) => log.write(d)); + const hud = await app.firstWindow({ timeout: 60_000 }); + await hud.waitForLoadState("domcontentloaded"); + // Switching closes the HUD window under the evaluate, so its rejection is expected. + await hud.evaluate(() => window.electronAPI.switchToEditor()).catch(() => undefined); + let editor = null; + for (let i = 0; i < 60 && !editor; i++) { + editor = app.windows().find((w) => w.url().includes("windowType=editor")) ?? null; + if (!editor) await sleep(500); + } + if (!editor) throw new Error("editor window did not open"); + editor.on("console", (m) => m.type() === "error" && consoleErrors.push(m.text())); + editor.on("pageerror", (e) => consoleErrors.push(`pageerror: ${e.message}`)); + await editor.waitForLoadState("domcontentloaded"); + const preview = editor.getByTestId("preview"); + await preview.waitFor({ state: "visible", timeout: 30_000 }); + await sleep(8000); // project load + first composed frame + return { app, proc, editor, preview }; +} + +async function close({ app, proc }) { + await Promise.race([app.close(), sleep(8000)]); + try { + proc.kill(); + } catch { + // already gone + } +} + +function driver(editor, preview) { + const time = async () => Number(await preview.getAttribute("data-current-time-sec")); + const canvas = editor.locator('[class*="_tlRulerRow_"] [class*="_tlCanvas_"]').first(); + let totalSec = null; + const clickRulerAt = async (frac) => { + const box = await canvas.boundingBox(); + if (!box) throw new Error("no ruler"); + await editor.mouse.click(box.x + box.width * frac, box.y + box.height / 2); + await sleep(400); + }; + // The ruler maps x linearly onto [0, total]; one probe click measures the total. + const seekTo = async (sec) => { + if (totalSec === null) { + await clickRulerAt(0.5); + totalSec = (await time()) / 0.5; + } + await clickRulerAt(sec / totalSec); + await sleep(1200); // the compositor's frame for the new time + return time(); + }; + const xOf = async (sec) => { + const box = await canvas.boundingBox(); + return box.x + (box.width * sec) / totalSec; + }; + // Clicks the camera-lane pill under `sec`: Full Camera and layout sections share that lane. + const selectPillAt = async (sec) => { + const x = await xOf(sec); + const pills = editor.locator('[class*="_lanePill_"][class*="_laneCameraFullscreen_"]'); + const n = await pills.count(); + for (let i = 0; i < n; i++) { + const b = await pills.nth(i).boundingBox(); + if (b && x >= b.x && x <= b.x + b.width) { + await editor.mouse.click(x, b.y + b.height / 2); + await sleep(500); + return true; + } + } + return false; + }; + const shot = async (name) => { + const file = path.join(OUT, `${name}.png`); + await preview.screenshot({ path: file }); + return file; + }; + const cameraPillCount = () => + editor.locator('[class*="_lanePill_"][class*="_laneCameraFullscreen_"]').count(); + // The composed frame alone (the preview element also holds the pane around it), decoded in + // the page: the four outermost columns and rows on the left and top, each as RGB pixels. + const frameEdges = async (name) => { + const file = path.join(OUT, `${name}-frame.png`); + const png = await preview.locator("canvas").first().screenshot({ path: file }); + const measured = await editor.evaluate(async (b64) => { + const img = new Image(); + img.src = `data:image/png;base64,${b64}`; + await img.decode(); + const c = document.createElement("canvas"); + c.width = img.width; + c.height = img.height; + const g = c.getContext("2d"); + g.drawImage(img, 0, 0); + const px = g.getImageData(0, 0, c.width, c.height).data; + const at = (x, y) => { + const i = (y * c.width + x) * 4; + return [px[i], px[i + 1], px[i + 2]]; + }; + const column = (x) => Array.from({ length: c.height }, (_, y) => at(x, y)); + const row = (y) => Array.from({ length: c.width }, (_, x) => at(x, y)); + return { + size: [c.width, c.height], + left: [0, 1, 2, 3].map(column), + top: [0, 1, 2, 3].map(row), + }; + }, png.toString("base64")); + return { file, ...measured }; + }; + return { time, seekTo, selectPillAt, shot, cameraPillCount, frameEdges }; +} + +const colourDistance = (a, b) => Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2]); +const meanColour = (pixels) => + [0, 1, 2].map((k) => Math.round(pixels.reduce((sum, p) => sum + p[k], 0) / pixels.length)); + +/** + * How much wallpaper a frame shows along its left and top edges, measured against a plain + * frame (no camera section) where the wallpaper lies at exactly those pixels. Per line (edge + * columns 0-3, rows 0-3): the share of its pixels within a small colour distance of the + * plain frame's pixel at the same place; the 5 % nearest each corner are left out (rounded + * corners). A line that is not part of the picture in the plain frame either (a resampling + * seam: it jumps away from its inner neighbour there, while a wallpaper gradient does not) is + * reported and skipped. + */ +function wallpaperAtEdges(frame, plain) { + const report = {}; + for (const side of ["left", "top"]) { + report[side] = frame[side].map((line, i) => { + const ref = plain[side][i]; + const inner = plain[side][i + 1] ?? plain[side][i - 1]; + const cut = Math.round(line.length * 0.05); + const at = Array.from({ length: line.length - 2 * cut }, (_, k) => k + cut); + const jump = at.reduce((sum, k) => sum + colourDistance(ref[k], inner[k]), 0) / at.length; + const wallpaper = at.filter((k) => colourDistance(line[k], ref[k]) < 24).length; + return { + line: i, + seam: jump > 40, + wallpaperShare: Number((wallpaper / at.length).toFixed(3)), + mean: meanColour(at.map((k) => line[k])), + plainMean: meanColour(at.map((k) => ref[k])), + }; + }); + } + return report; +} + +const shots = []; +let exitCode = 0; +let session = null; +try { + session = await launch(); + const { editor, preview } = session; + const d = driver(editor, preview); + const before = cameraState(); + note( + "the copy opens with no layout sections", + before.layouts.length === 0 && before.fulls.length === 1, + `fulls ${brief(before.fulls)}`, + ); + note("one camera-lane pill on open", (await d.cameraPillCount()) === 1); + + // The wallpaper along the frame's edges, read where no section is (the plain screen layout, + // whose padding shows the wallpaper). The project's own Full Camera sits at ~17-19 s. + const plainAt = clearOfTrims(12, 1); + await d.seekTo(plainAt); + const plain = await d.frameEdges("0-plain"); + note("a plain frame is captured as the wallpaper reference", true, `t=${plainAt}`); + + // A. Camera + inset from the layout menu, clear of the project's trims (a trim inside the + // section would move its visible start, see the header). + const startA = clearOfTrims(3, 2); + await d.seekTo(startA); + await editor.getByRole("button", { name: /^Add layout/ }).click(); + await editor.getByRole("button", { name: /Camera \+ inset/ }).click(); + let r = await waitState( + (s) => s.layouts.length === 1 && s.layouts[0].template === "camera-full-pip", + ); + note("A: the layout menu adds a camera-full-pip section", r.ok, brief(r.state.layouts)); + const sectionA = r.state.layouts[0]; + const midA = sectionA ? steadyMid(sectionA) : startA + 1; + const visibleA = sectionA ? visibleSpan(sectionA, readDoc()) : null; + note( + "A: the section lies clear of the project's trims", + Boolean(visibleA) && + Math.abs(visibleA[0] - sectionA.startMs / 1000) < 1e-3 && + Math.abs(visibleA[1] - sectionA.endMs / 1000) < 1e-3, + `visible ${JSON.stringify(visibleA)}, captures at ${midA.toFixed(3)}`, + ); + note( + "A: its two places show the clip's two cameras", + Boolean(sectionA) && + sectionA.slots.length === 2 && + new Set(sectionA.slots.map((s) => s.camera)).size === 2, + JSON.stringify(sectionA?.slots), + ); + const tA = await d.seekTo(midA); + shots.push(await d.shot("A-camera-full-pip")); + await editor.screenshot({ path: path.join(OUT, "A-window.png") }); + const boxes = await editor.evaluate(() => { + const box = (el) => { + const r = el?.getBoundingClientRect(); + return r ? [r.x, r.y, r.width, r.height].map(Math.round) : null; + }; + const preview = document.querySelector('[data-testid="preview"]'); + return { + window: [window.innerWidth, window.innerHeight], + preview: box(preview), + frame: box(preview?.querySelector('[class*="_previewFrame_"]')), + canvas: box(preview?.querySelector("canvas")), + }; + }); + fs.writeFileSync(path.join(OUT, "A-boxes.json"), JSON.stringify(boxes, null, 2)); + note("A: preview captured inside the section", Math.abs(tA - midA) < 0.3, `t=${tA.toFixed(2)}`); + + // B. Choose cameras in the inspector: camera 1 into the large place, then camera 2 again. + note("B: the section is selectable on the timeline", await d.selectPillAt(midA)); + const place1 = editor.getByRole("combobox", { name: /^Place 1/ }); + await place1.waitFor({ state: "visible", timeout: 5000 }); + for (const camera of [0, 1]) { + await place1.selectOption(String(camera)); + r = await waitState( + (s) => + s.layouts.length === 1 && + s.layouts[0].slots[0].camera === camera && + s.layouts[0].slots[1].camera === 1 - camera, + ); + note( + `B: the inspector puts camera ${camera + 1} into place 1 (the other moves to place 2)`, + r.ok, + JSON.stringify(r.state.layouts[0]?.slots), + ); + // Steady state at the middle of the visible span, captured with nothing selected (the + // ruler click clears the selection, so the section is picked again afterwards). + await d.seekTo(midA); + shots.push(await d.shot(`B-camera${camera + 1}-large`)); + const edges = wallpaperAtEdges(await d.frameEdges(`B-camera${camera + 1}-large`), plain); + fs.writeFileSync( + path.join(OUT, `B-camera${camera + 1}-large-edges.json`), + JSON.stringify(edges, null, 2), + ); + const lines = [...edges.left, ...edges.top].filter((l) => !l.seam); + const describe = (side) => + edges[side] + .map((l) => (l.seam ? `${l.line}:seam` : `${l.line}:${l.wallpaperShare}`)) + .join(" "); + note( + `B: camera ${camera + 1} large fills the frame edge to edge`, + lines.length >= 6 && lines.every((l) => l.wallpaperShare < 0.1), + `wallpaper share per line, left ${describe("left")} | top ${describe("top")}`, + ); + await d.selectPillAt(midA); + await place1.waitFor({ state: "visible", timeout: 5000 }); + } + + // C. A Full Camera section (key C) at 9 s, converted to screen + camera and back. + await d.seekTo(clearOfTrims(9, 2)); + await editor.keyboard.press("c"); + r = await waitState((s) => s.fulls.length === 2); + note("C: key C adds a Full Camera section", r.ok, brief(r.state.fulls)); + const fullC = r.state.fulls.find((f) => f.startMs > 8000 && f.startMs < 10_000); + const midC = fullC ? steadyMid(fullC) : 10; + await d.seekTo(midC); + shots.push(await d.shot("C1-full-camera")); + note("C: the Full Camera section is selectable", await d.selectPillAt(midC)); + const templates = editor.getByRole("group", { name: "Layout" }); + await templates.getByRole("button", { name: /Screen \+ camera/ }).click(); + r = await waitState( + (s) => + s.fulls.length === 1 && + s.layouts.length === 2 && + s.layouts.some((l) => l.template === "screen-pip" && Math.abs(l.startMs - fullC.startMs) < 1), + ); + note( + "C: Full Camera → screen-pip moves the section into the layout list", + r.ok, + brief(r.state.layouts), + ); + await sleep(1000); + shots.push(await d.shot("C2-screen-pip")); + await templates.getByRole("button", { name: /^Full camera/ }).click(); + r = await waitState( + (s) => + s.fulls.length === 2 && + s.layouts.length === 1 && + s.fulls.some((f) => Math.abs(f.startMs - fullC.startMs) < 1), + ); + note("C: screen-pip → Full Camera moves it back", r.ok, `fulls ${brief(r.state.fulls)}`); + await sleep(1000); + shots.push(await d.shot("C3-full-camera-again")); + + // D. Undo and redo the conversion back. + await editor.keyboard.press("Control+z"); + r = await waitState( + (s) => s.fulls.length === 1 && s.layouts.some((l) => l.template === "screen-pip"), + ); + note("D: Ctrl+Z restores the screen-pip section", r.ok, brief(r.state.layouts)); + await editor.keyboard.press("Control+Shift+z"); + r = await waitState((s) => s.fulls.length === 2 && s.layouts.length === 1); + note("D: Ctrl+Shift+Z redoes the Full Camera", r.ok, `fulls ${brief(r.state.fulls)}`); + + // E. Drag the inset window of section A in the preview. + await d.seekTo(midA); + await d.selectPillAt(midA); + const place = editor.getByTestId("layout-place").first(); + await place.waitFor({ state: "visible", timeout: 5000 }); + note( + "E: the selected section shows its PiP window in the preview", + true, + `${await editor.getByTestId("layout-place").count()} place(s)`, + ); + const pb = await place.boundingBox(); + const rectBefore = cameraState().layouts[0]?.slots[1]?.rect ?? null; + await editor.mouse.move(pb.x + pb.width / 2, pb.y + pb.height / 2); + await editor.mouse.down(); + for (let i = 1; i <= 10; i++) { + await editor.mouse.move(pb.x + pb.width / 2 - 12 * i, pb.y + pb.height / 2 - 6 * i); + await sleep(30); + } + await editor.mouse.up(); + r = await waitState((s) => { + const rect = s.layouts[0]?.slots[1]?.rect; + return Boolean(rect) && JSON.stringify(rect) !== JSON.stringify(rectBefore); + }); + note( + "E: dragging the PiP window stores its rect", + r.ok, + JSON.stringify(r.state.layouts[0]?.slots[1]?.rect), + ); + await sleep(1000); + shots.push(await d.shot("E-pip-dragged")); + + // F. Calibration: perspective of camera 2, from the camera list. + await d.seekTo(midA); // the ruler click also clears the selection + const rail = editor.getByRole("button", { name: "Camera layout", exact: true }); + if ((await rail.getAttribute("aria-pressed")) !== "true") await rail.click(); + await editor + .getByTestId("camera-row-1") + .getByRole("button", { name: /Correct perspective/ }) + .click(); + const dialog = editor.getByRole("dialog"); + await dialog.waitFor({ state: "visible", timeout: 5000 }); + await dialog + .getByText(/Loading the camera picture/) + .waitFor({ state: "detached", timeout: 15_000 }); + note( + "F: the dialog loads the camera still", + !(await dialog.getByText(/could not be loaded/).count()), + ); + await dialog.getByRole("button", { name: "Detect markers" }).click(); + const status = dialog.getByRole("status"); + const statusText = (await status.innerText()).trim(); + note("F: marker detection reports a result", statusText.length > 0, statusText); + const h0 = dialog.getByTestId("calibration-handle-0"); + await h0.focus(); + for (let i = 0; i < 4; i++) await editor.keyboard.press("Shift+ArrowRight"); + for (let i = 0; i < 4; i++) await editor.keyboard.press("Shift+ArrowDown"); + note("F: moving a corner clears the marker message", (await status.innerText()).trim() === ""); + const h2 = await dialog.getByTestId("calibration-handle-2").boundingBox(); + await editor.mouse.move(h2.x + h2.width / 2, h2.y + h2.height / 2); + await editor.mouse.down(); + for (let i = 1; i <= 8; i++) { + await editor.mouse.move(h2.x + h2.width / 2 - 5 * i, h2.y + h2.height / 2 - 3 * i); + await sleep(30); + } + await editor.mouse.up(); + await sleep(500); + await dialog.screenshot({ path: path.join(OUT, "F1-calibration-dialog.png") }); + shots.push(path.join(OUT, "F1-calibration-dialog.png")); + await dialog.getByRole("button", { name: "Apply", exact: true }).click(); + r = await waitState((s) => { + const p = s.settings?.[1]?.perspective; + return Boolean(p) && p.corners.length === 4 && Math.abs(p.aspect - 297 / 210) < 1e-6; + }); + const corners = r.state.settings?.[1]?.perspective?.corners; + note( + "F: Apply stores camera 2's perspective (A4 landscape)", + r.ok && corners[0].x > 0.15 && corners[2].x < 0.85, + JSON.stringify(r.state.settings?.[1]?.perspective), + ); + await d.seekTo(midA); + await sleep(1000); + shots.push(await d.shot("F2-perspective-in-section")); + + // G. Save, relaunch, compare. + await editor.keyboard.press("Control+s"); + await sleep(1500); + const saved = cameraState(); + await close(session); + session = null; + + session = await launch(); + const d2 = driver(session.editor, session.preview); + const reopened = cameraState(); + note( + "G: the reopened project keeps sections and camera settings", + JSON.stringify(reopened) === JSON.stringify(saved), + `${reopened.layouts.length} layout(s), ${reopened.fulls.length} full camera(s)`, + ); + note("G: the timeline shows all three camera sections", (await d2.cameraPillCount()) === 3); + await d2.seekTo(midA); + shots.push(await d2.shot("G-reopened-section-A")); + await d2.seekTo(midC); + shots.push(await d2.shot("G-reopened-full-camera")); + fs.writeFileSync(path.join(OUT, "final-state.json"), JSON.stringify(reopened, null, 2)); +} catch (error) { + note("run completed without an exception", false, String(error?.stack ?? error)); +} finally { + note( + "no console errors in the editor", + consoleErrors.length === 0, + consoleErrors.slice(0, 5).join(" | "), + ); + if (session) await close(session); + log.end(); + fs.rmSync(COPY_FILE, { force: true }); + console.log(`removed copy: ${!fs.existsSync(COPY_FILE)}`); + fs.writeFileSync(path.join(OUT, "results.json"), JSON.stringify({ results, shots }, null, 2)); + exitCode = results.every((x) => x.ok) ? 0 : 1; +} +process.exit(exitCode); diff --git a/scripts/generate-original-cursor-themes.mjs b/scripts/generate-original-cursor-themes.mjs index 678cae5ff..f9d925c35 100644 --- a/scripts/generate-original-cursor-themes.mjs +++ b/scripts/generate-original-cursor-themes.mjs @@ -1,6 +1,7 @@ // Prepare transparent cursor PNGs from the original raster artwork in design/cursors. // Run with `node scripts/generate-original-cursor-themes.mjs`. -// Each source PNG contains the arrow on the left and the hand on the right. +// Each source PNG contains the arrow on the left and the hand on the right. Pixel Candy is not +// here: its pixel art is a grid, and scripts/generate-pixel-candy-voxels.mjs writes its PNGs. import { mkdir, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; @@ -27,10 +28,6 @@ const themes = [ id: "pop-coral", hotspots: { arrow: [265, 58], pointer: [1245, 65] }, }, - { - id: "pixel-candy", - hotspots: { arrow: [273, 76], pointer: [1218, 78] }, - }, { id: "star-sprout", hotspots: { arrow: [210, 65], pointer: [1218, 65] }, diff --git a/scripts/generate-pixel-candy-voxels.mjs b/scripts/generate-pixel-candy-voxels.mjs new file mode 100644 index 000000000..a6470c9c8 --- /dev/null +++ b/scripts/generate-pixel-candy-voxels.mjs @@ -0,0 +1,300 @@ +// Pixel Candy, drawn once as pixel art: this grid is both the 2D cursor (its PNGs) and the 3D +// model (one cube per pixel, mode 15, theme 3). +// Run with `node scripts/generate-pixel-candy-voxels.mjs` after changing a grid below: it writes +// public/cursors/pixel-candy/{arrow,pointer}.png, design/cursors/pixel-candy/source.png, and the +// PIX_* tables of the three compositor shaders between their `` markers, then +// prints the hotspots for src/lib/cursor/cursorThemes.ts. +// +// `O` is the plum outline, `P` the pink body, `H` its pale highlight down the lit edge, `.` +// nothing. The shade is not drawn: a body pixel with the outline right of it or under it turns +// darker pink by itself. The hotspot is the top-left corner of the arrow's first pixel and the top +// middle of the index; in the 3D prototype frame (crates/compositor/src/sculpt.rs) it is the +// origin. + +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { deflateSync } from "node:zlib"; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const SHADERS = path.join(ROOT, "crates", "compositor", "src"); +// Mirrors SCULPT_VOX in the shaders and VOX in sculpt.rs: sixteen rows make a cursor one unit tall. +const VOX = 0.0625; +// Sprite pixels per art pixel, in the 128px sprite: sixteen rows fill the 112px the other themes +// use. +const CELL = 7; +const SPRITE = 128; + +const SHAPES = [ + { + name: "arrow", + // Hotspot column: the left edge of the first pixel. + hotspotColumn: 0, + grid: [ + "O..........", + "OO.........", + "OHO........", + "OHPO.......", + "OHPPO......", + "OHPPPO.....", + "OHPPPPO....", + "OHPPPPPO...", + "OHPPPPPPO..", + "OHPPPPPPPO.", + "OHPPPPOOOOO", + "OHPOHPO....", + "OHO.OHPO...", + "OO..OHPO...", + ".....OHPO..", + "......OO...", + ], + }, + { + name: "pointer", + // Between the index's two columns. + hotspotColumn: 5, + grid: [ + "....OO..........", + "...OHPO.........", + "...OHPO.........", + "...OHPO.........", + "...OHPOOO.......", + "...OHPOPPOOO....", + "...OHPOPPOPPOOO.", + ".OOOHPOPPOPPOPPO", + "OHHOHPPPPPPPPPPO", + "OHPPHPPPPPPPPPPO", + ".OHPPPPPPPPPPPPO", + ".OHPPPPPPPPPPPO.", + "..OHPPPPPPPPPPO.", + "..OHPPPPPPPPPO..", + "...OHPPPPPPPPO..", + "...OOOOOOOOOOO..", + ], + }, +]; + +// sRGB, in the order of the shader's colour classes. +const COLOURS = { O: [74, 31, 92], P: [255, 111, 168], H: [255, 200, 222], S: [222, 70, 130] }; + +// The grid with its shade pixels. +function shaded(grid) { + const at = (c, r) => grid[r]?.[c] ?? "."; + return grid.map((row, r) => + [...row] + .map((ch, c) => (ch === "P" && (at(c + 1, r) === "O" || at(c, r + 1) === "O") ? "S" : ch)) + .join(""), + ); +} + +const shapes = SHAPES.map((shape) => { + const grid = shaded(shape.grid); + const cols = Math.max(...grid.map((row) => row.length)); + const rows = grid.length; + const mask = (test) => + grid.map((row) => [...row].reduce((m, ch, c) => (test(ch) ? m | (1 << c) : m), 0)); + return { + ...shape, + grid, + cols, + rows, + origin: [-shape.hotspotColumn * VOX, 0], + body: mask((ch) => ch !== "."), + line: mask((ch) => ch === "O"), + hi: mask((ch) => ch === "H"), + shade: mask((ch) => ch === "S"), + }; +}); + +// The body as rectangles (centre, half size, prototype units): each row's runs, merged with the +// rows below while the run stays the same. The shadow uses their exact outline. +function rects(shape) { + const open = []; + const done = []; + shape.body.forEach((m, r) => { + const runs = []; + for (let c = 0; c < shape.cols; c++) { + if (!(m & (1 << c))) continue; + const start = c; + while (c + 1 < shape.cols && m & (1 << (c + 1))) c++; + runs.push([start, c]); + } + for (const box of open.splice(0)) { + const k = runs.findIndex(([a, b]) => a === box.a && b === box.b); + if (k >= 0) { + box.r1 = r; + runs.splice(k, 1); + open.push(box); + } else done.push(box); + } + for (const [a, b] of runs) open.push({ a, b, r0: r, r1: r }); + }); + done.push(...open); + return done.map(({ a, b, r0, r1 }) => { + const [ox, oy] = shape.origin; + const [x0, x1] = [ox + a * VOX, ox + (b + 1) * VOX]; + const [y0, y1] = [oy - (r1 + 1) * VOX, oy - r0 * VOX]; + return [(x0 + x1) / 2, (y0 + y1) / 2, (x1 - x0) / 2, (y1 - y0) / 2]; + }); +} + +const boxes = shapes.map(rects); +const rectStarts = boxes.reduce((s, b) => [...s, s[s.length - 1] + b.length], [0]); +const grids = []; +let at = 0; +for (const s of shapes) { + grids.push([s.cols, s.rows, at, 0]); + at += s.rows; +} +const bounds = shapes.map(({ origin: [ox, oy], cols, rows }) => [ + ox + (cols * VOX) / 2, + oy - (rows * VOX) / 2, + (cols * VOX) / 2, + (rows * VOX) / 2, +]); + +// ---- PNG ---- + +const CRC = new Uint32Array(256).map((_, n) => { + let c = n; + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + return c >>> 0; +}); +function crc32(bytes) { + let c = 0xffffffff; + for (const b of bytes) c = CRC[(c ^ b) & 0xff] ^ (c >>> 8); + return (c ^ 0xffffffff) >>> 0; +} +function chunk(type, data) { + const out = Buffer.alloc(12 + data.length); + out.writeUInt32BE(data.length, 0); + out.write(type, 4, "ascii"); + data.copy(out, 8); + out.writeUInt32BE(crc32(out.subarray(4, 8 + data.length)), 8 + data.length); + return out; +} +function png(width, height, rgba) { + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header.set([8, 6, 0, 0, 0], 8); + const raw = Buffer.alloc((width * 4 + 1) * height); + for (let y = 0; y < height; y++) + rgba.copy(raw, y * (width * 4 + 1) + 1, y * width * 4, (y + 1) * width * 4); + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + chunk("IHDR", header), + chunk("IDAT", deflateSync(raw, { level: 9 })), + chunk("IEND", Buffer.alloc(0)), + ]); +} +// The grid at `cell` pixels per art pixel, its top-left at (x0, y0) of a `w` × `h` canvas. +function paint(rgba, w, shape, cell, x0, y0) { + shape.grid.forEach((row, r) => { + [...row].forEach((ch, c) => { + if (ch === ".") return; + for (let y = 0; y < cell; y++) { + for (let x = 0; x < cell; x++) { + const i = ((y0 + r * cell + y) * w + x0 + c * cell + x) * 4; + rgba.set([...COLOURS[ch], 255], i); + } + } + }); + }); +} + +const cursors = path.join(ROOT, "public", "cursors", "pixel-candy"); +await mkdir(cursors, { recursive: true }); +const hotspots = []; +for (const shape of shapes) { + const x0 = Math.round((SPRITE - shape.cols * CELL) / 2); + const y0 = Math.round((SPRITE - shape.rows * CELL) / 2); + const rgba = Buffer.alloc(SPRITE * SPRITE * 4); + paint(rgba, SPRITE, shape, CELL, x0, y0); + await writeFile(path.join(cursors, `${shape.name}.png`), png(SPRITE, SPRITE, rgba)); + // On the 32-logical reference of cursorThemes.ts. + hotspots.push([ + shape.name, + ((x0 + shape.hotspotColumn * CELL) * 32) / SPRITE, + (y0 * 32) / SPRITE, + ]); +} + +// The design master, arrow on the left and hand on the right, like the other themes' sources. +{ + const [w, h, cell] = [1774, 887, 40]; + const rgba = Buffer.alloc(w * h * 4); + shapes.forEach((shape, i) => { + const x0 = Math.round(i * (w / 2) + (w / 2 - shape.cols * cell) / 2); + paint(rgba, w, shape, cell, x0, Math.round((h - shape.rows * cell) / 2)); + }); + await writeFile( + path.join(ROOT, "design", "cursors", "pixel-candy", "source.png"), + png(w, h, rgba), + ); +} + +// ---- Shader tables ---- + +const num = (v) => { + const s = v.toFixed(4); + return Number(s) === 0 ? "0.0" : s.replace(/0+$/, "").replace(/\.$/, ".0"); +}; + +function block(lang) { + const t = { + wgsl: { i4: "vec4", f2: "vec2", f4: "vec4" }, + hlsl: { i4: "int4", f2: "float2", f4: "float4" }, + metal: { i4: "int4", f2: "float2", f4: "float4" }, + }[lang]; + const vec = (type, v) => + `${type}(${v.map((x) => (type === t.i4 ? String(x) : num(x))).join(", ")})`; + const tables = [ + ["PIX_BODY", shapes.flatMap((s) => s.body)], + ["PIX_LINE", shapes.flatMap((s) => s.line)], + ["PIX_HI", shapes.flatMap((s) => s.hi)], + ["PIX_SHADE", shapes.flatMap((s) => s.shade)], + ["PIX_RECT_N", rectStarts], + ]; + const vectors = [ + ["PIX_GRID", t.i4, grids], + ["PIX_ORIGIN", t.f2, shapes.map((s) => s.origin)], + ["PIX_BOX", t.f4, bounds], + ["PIX_RECT", t.f4, boxes.flat()], + ]; + if (lang === "wgsl") { + return [ + ...tables.map(([n, v]) => `const ${n} = array(${v.join(", ")});`), + ...vectors.map( + ([n, ty, v]) => + `const ${n} = array<${ty}, ${v.length}>(\n ${v.map((x) => vec(ty, x)).join(",\n ")}\n);`, + ), + ].join("\n"); + } + const decl = lang === "hlsl" ? "static const" : "constant"; + return [ + ...tables.map(([n, v]) => `${decl} int ${n}[${v.length}] = { ${v.join(", ")} };`), + ...vectors.map( + ([n, ty, v]) => + `${decl} ${ty} ${n}[${v.length}] = {\n ${v.map((x) => vec(ty, x)).join(",\n ")}\n};`, + ), + ].join("\n"); +} + +const targets = [ + ["vk_shaders/layer.wgsl", "wgsl"], + ["shaders.hlsl", "hlsl"], + ["shaders.metal", "metal"], +]; +const marked = /(\/\/ \n)[\s\S]*?(\/\/ <\/pixel-candy-voxels>)/; +const sources = await Promise.all( + targets.map(([file]) => readFile(path.join(SHADERS, file), "utf8")), +); +targets.forEach(([file], i) => { + if (!marked.test(sources[i])) throw new Error(`${file}: no markers`); +}); +for (const [i, [file, lang]] of targets.entries()) { + const src = sources[i].replace(marked, (_, open, close) => `${open}${block(lang)}\n${close}`); + await writeFile(path.join(SHADERS, file), src); +} +for (const [name, x, y] of hotspots) console.log(`${name}: hotspotX ${x}, hotspotY ${y}`); diff --git a/scripts/multicam-fixture.mjs b/scripts/multicam-fixture.mjs new file mode 100644 index 000000000..ac5b460fe --- /dev/null +++ b/scripts/multicam-fixture.mjs @@ -0,0 +1,933 @@ +// Several cameras in the picture, measured end to end (dev tool, not part of the build). +// +// Generates synthetic sources and two projects, runs the real headless export, and checks the +// exported pixels against the camera layout templates and the glide the compositor plans. +// +// node scripts/multicam-fixture.mjs generate +// screen.mp4 (1920x1080 dark grey, 8 s), cam0.mp4 (solid red), cam1.mp4 (solid green), +// cam2.mp4 (a 16x9 checkerboard warped "in perspective" onto known corners), all +// 1280x720, plus multicam.openscreen (three cameras, four layout regions, a perspective +// on camera 2) and onecam.openscreen (the same take with camera 1 only, no regions). +// Also board43.mp4 (a 12x9 board, 4:3, warped onto the same corners) and two more +// projects: persp.openscreen (camera 1 IS that board with a 4:3 perspective, in its +// default PiP and then in a Full Camera section) and seam.openscreen (a camera-1 Full +// Camera section directly followed by a camera-full-pip [2, 1] layout region). +// node scripts/multicam-fixture.mjs export +// Runs `electron . export` and prints the wall-clock time. Run from a built tree +// (`npm run build-vite`) with OPENSCREEN_COMPOSITOR_VIEW_NODE pointing at the addon and +// its ffmpeg DLLs first on PATH. +// node scripts/multicam-fixture.mjs check [multicam|persp|seam] +// Extracts frames by index and prints a pass/fail table; exits 1 on any failure. +// The project defaults to multicam. +// +// The expected rects (`pipRect`, `regionLayers`, `plannedRects`) MIRROR the templates and +// the planner; they are not an independent derivation. The template PiPs anchor on camera +// 1's default PiP, which the check measures (red box at 0.5 s) instead of recomputing it. +// +// ffmpeg: OPENSCREEN_FFMPEG, else the vendored build under crates/thirdparty. + +import { spawn, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), ".."); +const FFMPEG = + process.env.OPENSCREEN_FFMPEG ?? + path.join(ROOT, "crates", "thirdparty", "ffmpeg-n8.1.2-win64-lgpl-shared", "bin", "ffmpeg.exe"); + +const DURATION_S = 8; +const CAM_W = 1280; +const CAM_H = 720; +const SQUARE_PX = 80; // 16 x 9 squares on the 1280x720 board +// Where the board's corners land in camera 2's picture (pixels): TL, TR, BR, BL. +const BOARD_CORNERS_PX = [ + [160, 90], + [1050, 40], + [1180, 680], + [70, 600], +]; +const BOARD_ASPECT = CAM_W / CAM_H; +// Camera 1's board in persp.openscreen: 12 x 9 squares, so its corrected picture is 4:3 while +// the camera's own frame is 16:9 — a box laid out from the raw camera would stretch it. +const BOARD43 = { cols: 12, rows: 9, aspect: 4 / 3 }; + +// Mirrors src/lib/cameraLayoutTemplates.ts. +const PIP_WIDTH_FRAC = 0.22; +const PIP_MARGIN_FRAC = 0.025; +const PIP_GAP_FRAC = 0.02; +// Mirrors crates/compositor/src/regions.rs. +const TRANSITION_WINDOW_S = 1.01505; +const FULLSCREEN_LEAD_OUT_WINDOW_S = TRANSITION_WINDOW_S * 1.5; +const EXPORT_FPS = 60; +const TOLERANCE_PX = 2; + +const PERSP_REGIONS = [ + { id: "lay_persp_full", startMs: 3000, endMs: 6000, template: "camera-full", cameras: [0] }, +]; +const SEAM_REGIONS = [ + { id: "lay_seam_full", startMs: 1000, endMs: 3000, template: "camera-full", cameras: [0] }, + { id: "lay_seam_pip", startMs: 3000, endMs: 5000, template: "camera-full-pip", cameras: [1, 0] }, +]; + +const REGIONS = [ + { id: "lay_screen_pip", startMs: 1000, endMs: 3000, template: "screen-pip", cameras: [0, 1] }, + { id: "lay_full_pip", startMs: 3000, endMs: 5000, template: "camera-full-pip", cameras: [1, 0] }, + { id: "lay_side", startMs: 5500, endMs: 7000, template: "side-by-side", cameras: [0, 1] }, + { id: "lay_board", startMs: 7000, endMs: 8000, template: "camera-full", cameras: [2] }, +]; + +function run(cmd, args) { + const result = spawnSync(cmd, args, { encoding: "utf8", maxBuffer: 1 << 28 }); + if (result.status !== 0) { + throw new Error( + `${path.basename(cmd)} failed (${result.status}): ${result.stderr?.slice(-2000)}`, + ); + } + return result; +} + +function encode(input, filter, out) { + run(FFMPEG, [ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + input, + ...(filter ? ["-vf", filter] : []), + "-t", + String(DURATION_S), + "-pix_fmt", + "yuv420p", + "-c:v", + "libopenh264", + "-b:v", + "12M", + out, + ]); +} + +/** Unit square (0,0),(1,0),(1,1),(0,1) onto four points (Heckbert), as `cameraPerspective.ts`. */ +function homographyFromUnitSquare([p0, p1, p2, p3]) { + const dx1 = p1[0] - p2[0]; + const dx2 = p3[0] - p2[0]; + const dx3 = p0[0] - p1[0] + p2[0] - p3[0]; + const dy1 = p1[1] - p2[1]; + const dy2 = p3[1] - p2[1]; + const dy3 = p0[1] - p1[1] + p2[1] - p3[1]; + const den = dx1 * dy2 - dx2 * dy1; + const g = (dx3 * dy2 - dx2 * dy3) / den; + const h = (dx1 * dy3 - dx3 * dy1) / den; + return [ + p1[0] - p0[0] + g * p1[0], + p3[0] - p0[0] + h * p3[0], + p0[0], + p1[1] - p0[1] + g * p1[1], + p3[1] - p0[1] + h * p3[1], + p0[1], + g, + h, + 1, + ]; +} + +function invert3([a, b, c, d, e, f, g, h, i]) { + const det = a * (e * i - f * h) - b * (d * i - f * g) + c * (d * h - e * g); + return [ + (e * i - f * h) / det, + (c * h - b * i) / det, + (b * f - c * e) / det, + (f * g - d * i) / det, + (a * i - c * g) / det, + (c * d - a * f) / det, + (d * h - e * g) / det, + (b * g - a * h) / det, + (a * e - b * d) / det, + ]; +} + +function generate(dir) { + fs.mkdirSync(dir, { recursive: true }); + const media = { + screen: path.join(dir, "screen.mp4"), + cam0: path.join(dir, "cam0.mp4"), + cam1: path.join(dir, "cam1.mp4"), + cam2: path.join(dir, "cam2.mp4"), + }; + encode(`color=c=0x303030:s=1920x1080:r=30:d=${DURATION_S}`, null, media.screen); + encode(`color=c=red:s=${CAM_W}x${CAM_H}:r=30:d=${DURATION_S}`, null, media.cam0); + encode(`color=c=0x00ff00:s=${CAM_W}x${CAM_H}:r=30:d=${DURATION_S}`, null, media.cam1); + // ffmpeg's `perspective` filter is GPL-only, and the vendored build is LGPL: the board is + // drawn straight in perspective instead. Each output pixel centre is mapped back onto the + // board's unit square by the inverse homography, so the true corners are exact. + const inv = invert3(homographyFromUnitSquare(BOARD_CORNERS_PX)); + const [s, u, w] = [0, 3, 6].map((r) => `(${inv[r]}*(X+0.5)+${inv[r + 1]}*(Y+0.5)+${inv[r + 2]})`); + const board = (cols, rows) => + `geq=lum='st(0,${s}/${w});st(1,${u}/${w});` + + `if(lt(ld(0),0)+gte(ld(0),1)+lt(ld(1),0)+gte(ld(1),1),128,` + + `if(mod(floor(ld(0)*${cols})+floor(ld(1)*${rows}),2),235,16))':cb=128:cr=128`; + const black = `color=c=black:s=${CAM_W}x${CAM_H}:r=30:d=${DURATION_S},format=yuv420p`; + encode(black, board(CAM_W / SQUARE_PX, CAM_H / SQUARE_PX), media.cam2); + media.board43 = path.join(dir, "board43.mp4"); + encode(black, board(BOARD43.cols, BOARD43.rows), media.board43); + + const track = (sourcePath, label) => ({ + sourcePath, + startMs: 0, + offsetMs: 0, + visible: true, + width: CAM_W, + height: CAM_H, + ...(label === undefined ? {} : { label }), + }); + const corners = BOARD_CORNERS_PX.map(([x, y]) => ({ x: x / CAM_W, y: y / CAM_H })); + const slotted = (regions) => + regions.map(({ cameras, ...region }) => ({ + ...region, + slots: cameras.map((camera) => ({ camera })), + })); + // kind: "multicam" | "onecam" | "persp" | "seam". + const project = (kind) => ({ + schemaVersion: 8, + project: { + id: `proj_${kind}_fixture`, + title: `${kind} fixture`, + createdAt: "2026-10-05T00:00:00.000Z", + updatedAt: "2026-10-05T00:00:00.000Z", + primaryAssetId: "asset_screen", + }, + assets: [ + { + id: "asset_screen", + kind: "video", + label: "screen.mp4", + originalPath: media.screen, + durationSec: DURATION_S, + sizeBytes: fs.statSync(media.screen).size, + video: { codec: "unknown", width: 1920, height: 1080, fps: 30 }, + cameraTrack: track(kind === "persp" ? media.board43 : media.cam0), + ...(kind === "multicam" + ? { + additionalCameraTracks: [track(media.cam1, "Green"), track(media.cam2, "Board")], + } + : {}), + ...(kind === "seam" ? { additionalCameraTracks: [track(media.cam1, "Green")] } : {}), + }, + ], + transcript: null, + transcripts: [], + timeline: { + clips: [ + { + id: "clip_main", + assetId: "asset_screen", + sourceStartSec: 0, + sourceEndSec: DURATION_S, + timelineStartSec: 0, + timelineEndSec: DURATION_S, + wordRefs: [], + origin: "user", + reason: "Fixture", + }, + ], + gaps: [], + trimRanges: [], + muteRanges: [], + speedRanges: [], + captionRanges: [], + }, + annotations: [], + zoomRanges: [], + audioTracks: [], + legacyEditor: { + wallpaper: "#101010", + wallpaperMotion: "none", + frame: "none", + shadowIntensity: 0, + backgroundBlur: 0, + motionBlurAmount: 0, + webcamLayoutPreset: "picture-in-picture", + webcamMaskShape: "rectangle", + webcamRoundness: 0.1, + webcamBackgroundMode: "none", + ...(kind === "multicam" + ? { + cameraLayoutRegions: slotted(REGIONS), + cameraSettings: [null, null, { perspective: { corners, aspect: BOARD_ASPECT } }], + } + : {}), + ...(kind === "persp" + ? { + cameraLayoutRegions: slotted(PERSP_REGIONS), + cameraSettings: [{ perspective: { corners, aspect: BOARD43.aspect } }], + } + : {}), + ...(kind === "seam" ? { cameraLayoutRegions: slotted(SEAM_REGIONS) } : {}), + }, + }); + for (const kind of ["multicam", "onecam", "persp", "seam"]) { + const file = path.join(dir, `${kind}.openscreen`); + fs.writeFileSync(file, `${JSON.stringify(project(kind), null, 2)}\n`); + console.log(`wrote ${file}`); + } +} + +/** + * Runs the headless export and prints its wall-clock time, and the render time alone: from + * the first progress line to "Exported", which leaves Electron's start-up and the project + * load out. + */ +function exportProject(projectPath, outPath) { + const electron = createRequire(import.meta.url)("electron"); + const started = performance.now(); + let firstProgress = null; + let exported = null; + const child = spawn(electron, [ROOT, "export", projectPath, "-o", outPath], { + cwd: ROOT, + stdio: ["ignore", "pipe", "pipe"], + }); + const watch = (chunk, sink) => { + const text = chunk.toString(); + if (firstProgress === null && text.includes("Exporting")) firstProgress = performance.now(); + if (exported === null && text.includes("Exported")) exported = performance.now(); + sink.write(text); + }; + child.stdout.on("data", (chunk) => watch(chunk, process.stdout)); + child.stderr.on("data", (chunk) => watch(chunk, process.stderr)); + child.once("exit", (code) => { + const seconds = (performance.now() - started) / 1000; + const render = + firstProgress !== null && exported !== null + ? `${((exported - firstProgress) / 1000).toFixed(2)} s` + : "unknown"; + console.log(`export exit=${code} wall=${seconds.toFixed(2)} s render=${render}`); + process.exitCode = code ?? 1; + }); +} + +// --- checking --------------------------------------------------------------------------- + +/** Minimal P6 reader: {width, height, data} with 3 bytes per pixel. */ +function readPpm(file) { + const buf = fs.readFileSync(file); + const fields = []; + let i = 0; + while (fields.length < 4) { + while (/\s/.test(String.fromCharCode(buf[i]))) i++; + if (buf[i] === 0x23) { + while (buf[i] !== 0x0a) i++; + continue; + } + let token = ""; + while (!/\s/.test(String.fromCharCode(buf[i]))) token += String.fromCharCode(buf[i++]); + fields.push(token); + } + if (fields[0] !== "P6" || fields[3] !== "255") throw new Error(`${file}: not an 8-bit P6`); + return { width: Number(fields[1]), height: Number(fields[2]), data: buf.subarray(i + 1) }; +} + +/** One ffmpeg pass that writes the given frame indices as PPMs, in order. */ +function extractFrames(video, indices, dir) { + fs.rmSync(dir, { recursive: true, force: true }); + fs.mkdirSync(dir, { recursive: true }); + const select = indices.map((n) => `eq(n\\,${n})`).join("+"); + run(FFMPEG, [ + "-hide_banner", + "-loglevel", + "error", + "-i", + video, + "-vf", + `select='${select}'`, + "-fps_mode", + "passthrough", + "-c:v", + "ppm", + path.join(dir, "f%04d.ppm"), + ]); + return new Map( + indices.map((n, k) => [n, readPpm(path.join(dir, `f${String(k + 1).padStart(4, "0")}.ppm`))]), + ); +} + +const isRed = (r, g, b) => r > 150 && g < 100 && b < 100; +const isGreen = (r, g, b) => g > 150 && r < 100 && b < 100; + +/** Bounding box [x0, y0, x1, y1) of the pixels `test` accepts, and how many there are. */ +function bbox(frame, test) { + let x0 = Infinity; + let y0 = Infinity; + let x1 = -1; + let y1 = -1; + let count = 0; + const { width, height, data } = frame; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const o = (y * width + x) * 3; + if (!test(data[o], data[o + 1], data[o + 2])) continue; + count++; + if (x < x0) x0 = x; + if (x > x1) x1 = x; + if (y < y0) y0 = y; + if (y > y1) y1 = y; + } + } + return count === 0 ? null : { box: [x0, y0, x1 + 1, y1 + 1], count }; +} + +/** + * A template PiP of a 16:9 camera, like `pipLayer`: anchored on camera 1's default PiP + * (`anchor`, [x, y, w, h] fractions: its width, right and bottom edges), stacking leftwards; + * the corner constants without one. + */ +function pipRect(index, frame, anchor) { + const w = anchor ? anchor[2] : PIP_WIDTH_FRAC; + const h = (w * frame.width) / (16 / 9) / frame.height; + const firstRight = anchor ? anchor[0] + anchor[2] : 1 - PIP_MARGIN_FRAC; + const right = firstRight - index * (w + PIP_GAP_FRAC); + const bottom = anchor + ? anchor[1] + anchor[3] + : 1 - (PIP_MARGIN_FRAC * frame.width) / frame.height; + return [right - w, bottom - h, w, h]; +} + +/** Each region's layers as {camera: [x, y, w, h]} (fractions), like `resolveCameraLayout`. */ +function regionLayers(region, frame, anchor) { + const out = {}; + region.cameras.forEach((camera, i) => { + if (region.template === "camera-full" || (region.template === "camera-full-pip" && i === 0)) { + out[camera] = [0, 0, 1, 1]; + } else if (region.template === "side-by-side") { + out[camera] = i === 0 ? [0, 0, 0.5, 1] : [0.5, 0, 0.5, 1]; + } else { + out[camera] = pipRect(region.template === "camera-full-pip" ? i - 1 : i, frame, anchor); + } + }); + return out; +} + +function cubicBezier(x1, y1, x2, y2, x) { + const sample = (a, b, t) => ((1 - 3 * b + 3 * a) * t + (3 * b - 6 * a)) * t * t + 3 * a * t; + let lo = 0; + let hi = 1; + for (let i = 0; i < 60; i++) { + const mid = (lo + hi) / 2; + if (sample(x1, x2, mid) < x) lo = mid; + else hi = mid; + } + return sample(y1, y2, (lo + hi) / 2); +} +const ease = (t) => cubicBezier(0.16, 1, 0.3, 1, Math.min(1, Math.max(0, t))); + +/** + * The rect each camera is planned at, at source time t: `camera_layers_at` in + * crates/compositor/src/camera_layers.rs for these regions (no speed regions, so the screen + * clock is the source clock). `fallback` is camera 0's default rect outside every region. + * Opacity is ignored: the check only looks at where a camera is. + */ +function plannedRects(t, frame, fallback, regions = REGIONS) { + const sec = (ms) => ms / 1000; + // A camera-1 camera-full region is a Full Camera section, not a layout region; it is a + // layout region's neighbour only at a seam, with camera 1 on the whole frame. + const isFullCamera = (r) => r.template === "camera-full" && r.cameras[0] === 0; + const layouts = regions.filter((r) => !isFullCamera(r)); + const index = layouts.findIndex((r) => sec(r.startMs) <= t && t <= sec(r.endMs)); + const fallbackLayers = { 0: fallback }; + if (index < 0) return fallbackLayers; + const region = layouts[index]; + const [start, end] = [sec(region.startMs), sec(region.endMs)]; + const prev = regions.find((r) => r !== region && Math.abs(sec(r.endMs) - start) <= 0.001); + const hasNext = regions.some((r) => r !== region && Math.abs(sec(r.startMs) - end) <= 0.001); + const current = regionLayers(region, frame, fallback); + const half = (end - start) / 2; + const winIn = Math.min(TRANSITION_WINDOW_S, half); + const winOut = Math.min(FULLSCREEN_LEAD_OUT_WINDOW_S, half); + const blend = (from, to, k) => { + const out = {}; + for (const [camera, rect] of Object.entries(to)) { + const a = from[camera]; + out[camera] = a ? rect.map((v, i) => a[i] + (v - a[i]) * k) : rect; + } + for (const [camera, rect] of Object.entries(from)) out[camera] ??= rect; + return out; + }; + if (t - start < winIn) { + return blend( + prev ? regionLayers(prev, frame, fallback) : fallbackLayers, + current, + ease((t - start) / winIn), + ); + } + if (!hasNext && end - t < winOut) { + return blend(current, fallbackLayers, 1 - ease((end - t) / winOut)); + } + return current; +} + +const toPx = ([x, y, w, h], frame) => [ + x * frame.width, + y * frame.height, + (x + w) * frame.width, + (y + h) * frame.height, +]; +/** Largest edge distance (px) between a measured box and a planned rect. */ +function boxError(measured, rect, frame) { + if (!measured) return Infinity; + const want = toPx(rect, frame); + return Math.max(...measured.box.map((v, i) => Math.abs(v - want[i]))); +} +const fmtBox = (m) => (m ? `[${m.box.join(", ")}]` : "none"); +const fmtRect = (rect, frame) => + `[${toPx(rect, frame) + .map((v) => v.toFixed(1)) + .join(", ")}]`; + +/** Sub-pixel positions where the luminance along a line crosses mid-grey. */ +function edgesAlong(frame, horizontal, at) { + const { width, height, data } = frame; + const n = horizontal ? width : height; + const lum = (i) => { + const o = horizontal ? (at * width + i) * 3 : (i * width + at) * 3; + return (data[o] + data[o + 1] + data[o + 2]) / 3; + }; + const edges = []; + for (let i = 1; i < n; i++) { + const a = lum(i - 1) - 128; + const b = lum(i) - 128; + if (a < 0 !== b < 0) edges.push(i - 1 + a / (a - b) + 0.5); + } + return edges; +} + +/** + * The rectified board, cover-fitted into the frame like any other camera: squares of one size + * `s = max(W / cols, H / rows)`, centred, so its edges sit at `centre + k * s` on both axes. + * Checks three rows and three columns: edges where the square grid puts them (right angles and + * equal square sizes), and equally spaced within each line. + */ +function boardCheck(frame, cols = CAM_W / SQUARE_PX, rows = CAM_H / SQUARE_PX) { + const { width, height } = frame; + const size = Math.max(width / cols, height / rows); + const along = (centre, count, extent) => { + const out = []; + for (let k = -count; k <= count; k++) { + const e = centre + (k - count / 2) * size; + if (e > 3 && e < extent - 3) out.push(e); + } + return out; + }; + const lines = [ + ...[1, 4, 7].map((r) => ({ + horizontal: true, + at: Math.round(height / 2 + (r - rows / 2 + 0.5) * size), + want: along(width / 2, cols, width), + })), + ...[2, Math.floor(cols / 2), cols - 3].map((c) => ({ + horizontal: false, + at: Math.round(width / 2 + (c - cols / 2 + 0.5) * size), + want: along(height / 2, rows, height), + })), + ]; + let spacingError = 0; + let gridError = 0; + const found = []; + const steps = { horizontal: [], vertical: [] }; + for (const line of lines) { + const edges = edgesAlong(frame, line.horizontal, line.at); + found.push(`${edges.length}/${line.want.length}`); + if (edges.length < 2) { + spacingError = Infinity; + gridError = Infinity; + continue; + } + const step = (edges[edges.length - 1] - edges[0]) / (edges.length - 1); + steps[line.horizontal ? "horizontal" : "vertical"].push(step); + edges.forEach((e, k) => { + spacingError = Math.max(spacingError, Math.abs(e - (edges[0] + k * step))); + }); + if (edges.length !== line.want.length) { + gridError = Infinity; + continue; + } + edges.forEach((e, k) => { + gridError = Math.max(gridError, Math.abs(e - line.want[k])); + }); + } + const mean = (v) => v.reduce((x, y) => x + y, 0) / v.length; + return { + lines: lines.length, + found, + size, + spacingError, + gridError, + stepH: mean(steps.horizontal), + stepV: mean(steps.vertical), + }; +} + +function report(rows) { + for (const r of rows) { + const verdict = r.pass === null ? "INFO" : r.pass ? "PASS" : "FAIL"; + console.log(`${verdict} | ${r.name} | ${r.measured} | expected ${r.expected}`); + } + const checked = rows.filter((r) => r.pass !== null); + const failed = checked.filter((r) => !r.pass).length; + console.log(`${checked.length - failed}/${checked.length} passed`); + process.exitCode = failed === 0 ? 0 : 1; +} + +const fractionsOf = (found, frame) => + found + ? [ + found.box[0] / frame.width, + found.box[1] / frame.height, + (found.box[2] - found.box[0]) / frame.width, + (found.box[3] - found.box[1]) / frame.height, + ] + : null; + +const isWhite = (r, g, b) => r > 200 && g > 200 && b > 200; + +/** + * The board inside a box (`box` = [x0, y0, x1, y1) px): the mean square step across (from + * three rows) and down (from three columns), from the luminance edges strictly inside it. + */ +function boardInBox(frame, box, cols, rows) { + const [x0, y0, x1, y1] = box; + const { width, data } = frame; + const lum = (x, y) => { + const o = (y * width + x) * 3; + return (data[o] + data[o + 1] + data[o + 2]) / 3 - 128; + }; + const edges = (horizontal, at) => { + const out = []; + const [lo, hi] = horizontal ? [x0 + 3, x1 - 3] : [y0 + 3, y1 - 3]; + for (let i = lo + 1; i < hi; i++) { + const a = horizontal ? lum(i - 1, at) : lum(at, i - 1); + const b = horizontal ? lum(i, at) : lum(at, i); + if (a < 0 !== b < 0) out.push(i - 1 + a / (a - b) + 0.5); + } + return out; + }; + const step = (e) => (e.length < 2 ? Number.NaN : (e[e.length - 1] - e[0]) / (e.length - 1)); + const [bw, bh] = [x1 - x0, y1 - y0]; + const across = [1, Math.floor(rows / 2), rows - 2].map((r) => + step(edges(true, Math.round(y0 + ((r + 0.5) * bh) / rows))), + ); + const down = [2, Math.floor(cols / 2), cols - 3].map((c) => + step(edges(false, Math.round(x0 + ((c + 0.5) * bw) / cols))), + ); + const mean = (v) => v.reduce((x, y) => x + y, 0) / v.length; + return { stepH: mean(across), stepV: mean(down) }; +} + +function checkPersp(video) { + const frameAt = (sec) => Math.round(sec * EXPORT_FPS); + const frames = extractFrames( + video, + [frameAt(1), frameAt(4.25)], + path.join(os.tmpdir(), "openscreen-multicam-frames"), + ); + const rows = []; + const add = (name, pass, measured, expected) => rows.push({ name, pass, measured, expected }); + const { cols, rows: boardRows, aspect } = BOARD43; + + // Default PiP: the box takes the perspective's 4:3, and its squares are square. + const pipFrame = frames.get(frameAt(1)); + const pip = bbox(pipFrame, isWhite); + const pipAspect = pip ? (pip.box[2] - pip.box[0]) / (pip.box[3] - pip.box[1]) : Number.NaN; + const pipHeight = pip ? pip.box[3] - pip.box[1] : 0; + add( + "1.0 s camera 1 PiP box has the perspective's 4:3", + Math.abs(pipAspect * pipHeight - aspect * pipHeight) <= TOLERANCE_PX, + `${fmtBox(pip)} ratio ${pipAspect.toFixed(4)}`, + `${aspect.toFixed(4)} within ${TOLERANCE_PX} px of width`, + ); + const inPip = pip ? boardInBox(pipFrame, pip.box, cols, boardRows) : { stepH: NaN, stepV: NaN }; + add( + "1.0 s PiP checkerboard squares square (step across vs down)", + Math.abs(inPip.stepH - inPip.stepV) <= TOLERANCE_PX, + `${inPip.stepH.toFixed(2)} x ${inPip.stepV.toFixed(2)} px`, + `equal within ${TOLERANCE_PX} px`, + ); + + // Full Camera: the 4:3 picture cover-fitted into the frame, squares square and on the grid. + const board = boardCheck(frames.get(frameAt(4.25)), cols, boardRows); + add( + "4.25 s Full Camera checkerboard edges equally spaced", + board.spacingError <= TOLERANCE_PX, + `${board.lines} lines, edges ${board.found.join(" ")}, max err ${board.spacingError.toFixed(2)} px`, + `<= ${TOLERANCE_PX} px`, + ); + add( + "4.25 s Full Camera checkerboard squares square (step across vs down)", + Math.abs(board.stepH - board.stepV) <= TOLERANCE_PX, + `${board.stepH.toFixed(2)} x ${board.stepV.toFixed(2)} px`, + `equal within ${TOLERANCE_PX} px`, + ); + add( + "4.25 s Full Camera checkerboard on the cover-fitted square grid", + board.gridError <= TOLERANCE_PX, + `max err ${board.gridError.toFixed(2)} px`, + `squares of ${board.size.toFixed(1)} px, centred`, + ); + report(rows); +} + +function checkSeam(video) { + const t = (n) => n / EXPORT_FPS; + const frameAt = (sec) => Math.round(sec * EXPORT_FPS); + const sweep = []; + // From after the Full Camera section's own lead-in (1.0 s + 1.015 s) across the seam. + for (let n = frameAt(2.05); n <= frameAt(4.2); n += EXPORT_FPS / 30) sweep.push(n); + const indices = [...new Set([frameAt(0.5), frameAt(4.6), ...sweep])].sort((a, b) => a - b); + const frames = extractFrames( + video, + indices, + path.join(os.tmpdir(), "openscreen-multicam-frames"), + ); + const rows = []; + const add = (name, pass, measured, expected) => rows.push({ name, pass, measured, expected }); + const f05 = frames.get(frameAt(0.5)); + const fallback = fractionsOf(bbox(f05, isRed), f05); + add("0.5 s camera 1 default PiP present", !!fallback, fmtBox(bbox(f05, isRed)), "red box"); + + // Up to the seam camera 1 fills the frame: no lead-out of the Full Camera section. + let notFull = 0; + let worstFull = 0; + // From the seam on, camera 1 is on the planned glide from the frame to its PiP slot, and + // its box only ever shrinks: it never passes through the default PiP and back. (The slot + // IS the default PiP's rect — template PiPs anchor on it — so where it ends proves + // nothing; where the glide starts does.) + let worstGlide = 0; + let grows = 0; + let lastArea = Infinity; + let firstAfterSeam = null; + const trace = []; + for (const n of sweep) { + const frame = frames.get(n); + const got = bbox(frame, isRed); + const area = got ? (got.box[2] - got.box[0]) * (got.box[3] - got.box[1]) : 0; + trace.push(`${t(n).toFixed(2)}:${got ? got.box[0] : "-"}`); + if (t(n) < 3) { + const err = boxError(got, [0, 0, 1, 1], frame); + worstFull = Math.max(worstFull, err); + if (err > TOLERANCE_PX) notFull++; + } else { + const want = plannedRects(t(n), frame, fallback, SEAM_REGIONS)[0]; + worstGlide = Math.max(worstGlide, boxError(got, want, frame)); + if (area > lastArea + 4 * frame.width) grows++; + if (t(n) > 3 && firstAfterSeam === null) firstAfterSeam = { n, got, width: frame.width }; + } + lastArea = t(n) >= 3 ? area : lastArea; + } + add( + "2.05-3.0 s camera 1 fills the frame up to the seam", + notFull === 0, + `${notFull} frames off, max err ${worstFull.toFixed(2)} px`, + `0 (<= ${TOLERANCE_PX} px)`, + ); + add( + "3.0-4.2 s camera 1 on the planned glide from the frame", + worstGlide <= TOLERANCE_PX, + `max err ${worstGlide.toFixed(2)} px; left edge ${trace.filter((_, k) => k % 6 === 0).join(" ")}`, + `<= ${TOLERANCE_PX} px`, + ); + add("3.0-4.2 s camera 1 box never grows back", grows === 0, `${grows} growing steps`, "0"); + { + // The first frame after the seam is barely into the glide: still nearly the whole frame. + // Gliding from the default PiP instead, it would already be PiP-sized. + const first = firstAfterSeam; + const w = first?.got ? first.got.box[2] - first.got.box[0] : 0; + add( + `${first ? t(first.n).toFixed(3) : "?"} s glide starts from the frame, not the default PiP`, + !!first && w >= 0.75 * first.width, + first ? `${fmtBox(first.got)} width ${w} px` : "no frame", + `>= ${first ? (0.75 * first.width).toFixed(0) : "?"} px wide`, + ); + } + const f46 = frames.get(frameAt(4.6)); + const slot = plannedRects(t(frameAt(4.6)), f46, fallback, SEAM_REGIONS); + const red = bbox(f46, isRed); + const redErr = boxError(red, slot[0], f46); + add( + "4.6 s camera 1 in its PiP slot", + redErr <= TOLERANCE_PX, + `${fmtBox(red)} err ${redErr.toFixed(2)} px`, + fmtRect(slot[0], f46), + ); + const green = bbox(f46, isGreen); + add( + "4.6 s camera 2 fills the frame under it", + boxError(green, [0, 0, 1, 1], f46) <= TOLERANCE_PX, + fmtBox(green), + fmtRect([0, 0, 1, 1], f46), + ); + report(rows); +} + +function check(video) { + const t = (n) => n / EXPORT_FPS; + const frameAt = (sec) => Math.round(sec * EXPORT_FPS); + const glide = []; + for (let n = frameAt(2.9); n <= frameAt(3.6); n += EXPORT_FPS / 30) glide.push(n); + const indices = [ + ...new Set([ + frameAt(0.5), + frameAt(2), + frameAt(4), + frameAt(6), + frameAt(6.5), + frameAt(7.5), + ...glide, + ]), + ].sort((a, b) => a - b); + const frames = extractFrames( + video, + indices, + path.join(os.tmpdir(), "openscreen-multicam-frames"), + ); + const rows = []; + const add = (name, pass, measured, expected) => rows.push({ name, pass, measured, expected }); + + const f05 = frames.get(frameAt(0.5)); + const fallbackBox = bbox(f05, isRed); + const fallback = fallbackBox + ? [ + fallbackBox.box[0] / f05.width, + fallbackBox.box[1] / f05.height, + (fallbackBox.box[2] - fallbackBox.box[0]) / f05.width, + (fallbackBox.box[3] - fallbackBox.box[1]) / f05.height, + ] + : null; + add( + "0.5 s camera 1 default PiP present (reference for 6.0 s)", + !!fallbackBox, + fmtBox(fallbackBox), + "red box", + ); + add("0.5 s no other camera", !bbox(f05, isGreen), fmtBox(bbox(f05, isGreen)), "no green"); + + const atRect = (sec, camera, test, label) => { + const frame = frames.get(frameAt(sec)); + const want = plannedRects(t(frameAt(sec)), frame, fallback)[camera]; + const got = bbox(frame, test); + const err = boxError(got, want, frame); + add( + `${sec.toFixed(1)} s ${label}`, + err <= TOLERANCE_PX, + `${fmtBox(got)} err ${err.toFixed(2)} px`, + fmtRect(want, frame), + ); + }; + atRect(2, 0, isRed, "red PiP (screen-pip slot 0)"); + atRect(2, 1, isGreen, "green PiP (screen-pip slot 1)"); + atRect(4, 1, isGreen, "green fills the frame (camera-full-pip)"); + atRect(4, 0, isRed, "red PiP over it"); + + // The glide across 3.0 s: green present in every frame, on the planned rect, and growing. + let worst = 0; + let missing = 0; + let shrinks = 0; + let lastArea = 0; + const trace = []; + for (const n of glide) { + const frame = frames.get(n); + const got = bbox(frame, isGreen); + if (!got || got.count < 1000) missing++; + const want = plannedRects(t(n), frame, fallback)[1]; + worst = Math.max(worst, boxError(got, want, frame)); + const area = got ? (got.box[2] - got.box[0]) * (got.box[3] - got.box[1]) : 0; + if (area + 4 * frame.width < lastArea) shrinks++; + lastArea = area; + trace.push(`${t(n).toFixed(3)}:${got ? got.box[0] : "-"}`); + } + add( + `2.9-3.6 s green present in all ${glide.length} frames`, + missing === 0, + `${missing} without green`, + "0", + ); + add( + "2.9-3.6 s green on the planned glide rect", + worst <= TOLERANCE_PX, + `max err ${worst.toFixed(2)} px`, + `<= ${TOLERANCE_PX} px`, + ); + add("2.9-3.6 s green box never shrinks", shrinks === 0, `${shrinks} shrinking steps`, "0"); + const firstMoving = glide.find((n) => t(n) > 3); + const lastGlide = glide[glide.length - 1]; + const k0 = plannedRects(t(firstMoving), frames.get(firstMoving), fallback)[1]; + add( + "2.9-3.6 s glide starts at the 3.0 s boundary", + boxError( + bbox(frames.get(frameAt(2.9)), isGreen), + pipRect(1, frames.get(frameAt(2.9)), fallback), + frames.get(frameAt(2.9)), + ) <= TOLERANCE_PX && k0[0] < pipRect(1, frames.get(firstMoving), fallback)[0], + `left edge ${trace.slice(0, 6).join(" ")} ... ${trace[trace.length - 1]}`, + `PiP until 3.0, then x -> 0 (by ${t(lastGlide).toFixed(2)} s planned x ${(plannedRects(t(lastGlide), frames.get(lastGlide), fallback)[1][0] * frames.get(lastGlide).width).toFixed(1)})`, + ); + + { + // Green (slot 1) is drawn after red, so it hides whatever of red's still-gliding rect + // overhangs the right half: red's visible right edge is green's left edge. + const frame = frames.get(frameAt(6)); + const planned = plannedRects(t(frameAt(6)), frame, fallback); + const [x, y, w, h] = planned[0]; + const visible = [x, y, Math.min(x + w, planned[1][0]) - x, h]; + const got = bbox(frame, isRed); + const err = boxError(got, visible, frame); + add( + "6.0 s red, left half (lead-in from the default, k = ease(2/3)), under green", + err <= TOLERANCE_PX, + `${fmtBox(got)} err ${err.toFixed(2)} px`, + fmtRect(visible, frame), + ); + const settled = boxError(got, [0, 0, 0.5, 1], frame); + add("6.0 s red vs the settled left half", null, `err ${settled.toFixed(2)} px`, "information"); + } + atRect(6, 1, isGreen, "green, right half (fading in)"); + atRect(6.5, 0, isRed, "red, settled left half"); + atRect(6.5, 1, isGreen, "green, settled right half"); + + const board = boardCheck(frames.get(frameAt(7.5))); + add( + "7.5 s checkerboard edges equally spaced", + board.spacingError <= TOLERANCE_PX, + `${board.lines} lines, edges ${board.found.join(" ")}, max err ${board.spacingError.toFixed(2)} px`, + `<= ${TOLERANCE_PX} px`, + ); + add( + "7.5 s checkerboard squares square (step across vs down)", + Math.abs(board.stepH - board.stepV) <= TOLERANCE_PX, + `${board.stepH.toFixed(2)} x ${board.stepV.toFixed(2)} px`, + `equal within ${TOLERANCE_PX} px`, + ); + add( + "7.5 s checkerboard on the cover-fitted square grid", + board.gridError <= TOLERANCE_PX, + `max err ${board.gridError.toFixed(2)} px`, + `squares of ${board.size.toFixed(1)} px, centred`, + ); + + report(rows); +} + +const [command, a, b] = process.argv.slice(2); +if (command === "generate" && a) generate(path.resolve(a)); +else if (command === "export" && a && b) exportProject(path.resolve(a), path.resolve(b)); +else if (command === "check" && a && (b ?? "multicam") === "multicam") check(path.resolve(a)); +else if (command === "check" && a && b === "persp") checkPersp(path.resolve(a)); +else if (command === "check" && a && b === "seam") checkSeam(path.resolve(a)); +else { + console.error( + "usage: multicam-fixture.mjs generate | export | check [multicam|persp|seam]", + ); + process.exitCode = 2; +} diff --git a/scripts/stage-whisper-stt.sh b/scripts/stage-whisper-stt.sh index cb09ca555..a789b4828 100755 --- a/scripts/stage-whisper-stt.sh +++ b/scripts/stage-whisper-stt.sh @@ -79,20 +79,103 @@ fi TMP="$(mktemp -d)" trap 'rm -rf "${TMP}"' EXIT -echo "Fetching ${ARTIFACT} from the latest successful build-whisper-stt run..." -# No run id: gh resolves the most recent run that published this artifact. -# Artifacts expire (retention-days in build-whisper-stt.yml), so a stale branch +# Which run: the most recent successful build of THIS commit's helper sources. +# Not simply the most recent artifact, which is whatever branch last pushed a +# helper change: on 2026-09-30 that was a PR branch built from main without the +# fix 2.0.0-rc.2 was cut for, and rc.1 had shipped whatever main last built. +# Sources are compared by git object id, so the release branch's cherry-pick of +# a change matches the run built from main. +SOURCES=(electron/native/whisper-stt scripts/build-whisper-stt.sh .github/workflows/build-whisper-stt.yml) +# Prints the path's git object id at that commit, from the API, or nothing when +# the commit is gone (the run of a branch rewritten since). Any other answer is +# retried, then fatal: on 2026-10-01 two rc.5 jobs out of five rejected the very +# run the other three matched, in silence, and an API hiccup must never read as +# "no matching build". +object_at() { # + local attempt out + for attempt in 1 2 3; do + if out="$(gh api "repos/${REPO}/contents/$(dirname "$2")?ref=$1" \ + --jq ".[] | select(.path == \"$2\") | .sha" 2>"${TMP}/api-err")"; then + if [ -n "${out}" ]; then echo "${out}"; return 0; fi + elif grep -q "HTTP 404" "${TMP}/api-err"; then + return 0 + fi + sleep $((attempt * 2)) + done + echo "FATAL: could not read $2 at $1 from the API: $(cat "${TMP}/api-err")" >&2 + return 1 +} +same_sources() { # : were the helper's sources there the ones checked out here? + local path there here + for path in "${SOURCES[@]}"; do + there="$(object_at "$2" "${path}")" || exit 1 + here="$(git rev-parse "HEAD:${path}")" + if [ "${there}" != "${here}" ]; then + echo " run $1 (${2:0:8}): ${path} is ${there:-gone} there, ${here} here" + return 1 + fi + done +} +# Every successful run, newest first by our own sort. The API's order is not to +# be trusted: on 2026-10-01 the same `gh run list --limit 50` returned this +# morning's runs first in one job and August's in another, so whether the +# matching run made the cut depended on which job asked. Read into a file, so a +# failed or partial listing stops here instead of passing for a complete one, +# and with no `status=` filter: GitHub caps filtered queries at 1,000 results. +list_successful_runs() { + local attempt + for attempt in 1 2 3; do + if gh api --paginate \ + "repos/${REPO}/actions/workflows/build-whisper-stt.yml/runs?per_page=100" \ + --jq '.workflow_runs[] | select(.conclusion == "success") | "\(.created_at) \(.id) \(.head_sha)"' \ + >"${TMP}/runs" 2>"${TMP}/api-err"; then + sort -ru "${TMP}/runs" | cut -d' ' -f2- >"${TMP}/candidates" + return 0 + fi + sleep $((attempt * 2)) + done + echo "FATAL: could not list build-whisper-stt runs: $(cat "${TMP}/api-err")" >&2 + exit 1 +} +RUN_ID="" +for attempt in 1 2; do + list_successful_runs + while read -r id sha; do + if same_sources "${id}" "${sha}"; then RUN_ID="${id}"; break; fi + done <"${TMP}/candidates" + if [ -n "${RUN_ID}" ] || [ "${attempt}" = 2 ]; then break; fi + echo "No match in the run list; reading it once more..." + sleep 10 +done +if [ -z "${RUN_ID}" ]; then + cat >&2 < + +Refusing to package a helper built from other sources than the ones this +release ships. +EOF + exit 1 +fi + +echo "Fetching ${ARTIFACT} from build-whisper-stt run ${RUN_ID} (same helper sources)..." +# Artifacts expire (retention-days in build-whisper-stt.yml), so an old commit # can legitimately find nothing — say so in terms someone can act on. -if ! gh run download --repo "${REPO}" --name "${ARTIFACT}" --dir "${TMP}" 2>"${TMP}/err"; then +if ! gh run download "${RUN_ID}" --repo "${REPO}" --name "${ARTIFACT}" --dir "${TMP}" 2>"${TMP}/err"; then cat "${TMP}/err" >&2 cat >&2 < Refusing to package: the installer would ship with speech-to-text silently dead (no transcription, no captions). diff --git a/scripts/test-whisper-stt.mjs b/scripts/test-whisper-stt.mjs index 7e52bac2b..8ea9f2a34 100644 --- a/scripts/test-whisper-stt.mjs +++ b/scripts/test-whisper-stt.mjs @@ -19,12 +19,22 @@ // Env overrides: // OPENSCREEN_WHISPER_SERVER_EXE helper binary (default: the staged one for this host) // OPENSCREEN_WHISPER_MODEL GGML model (default: the userData cache location) +// OPENSCREEN_VAD_MODEL Silero VAD model (default: next to the GGML model; +// the speech checks run only when it is there, as in the app) +// OPENSCREEN_ALIGNER_MODEL CTC word aligner (default: the cached one for the detected +// language; the aligner checks run only when it is there) import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { + alignWordsOnEmissions, + emissionRegions, + parseEmissions, +} from "../electron/stt/ctcAlign.ts"; +import { CTC_ALIGNERS } from "../electron/stt/modelManager.ts"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.join(__dirname, ".."); @@ -47,6 +57,9 @@ const BIN = ); const MODEL = process.env.OPENSCREEN_WHISPER_MODEL ?? defaultModelPath(); +const VAD_MODEL = + process.env.OPENSCREEN_VAD_MODEL ?? path.join(path.dirname(MODEL), "ggml-silero-v6.2.0.bin"); +const WITH_VAD = fs.existsSync(VAD_MODEL); const LANGUAGE = argOf("--language") ?? "auto"; const WAV_ARG = argOf("--wav"); const REF_ARG = argOf("--ref"); @@ -208,7 +221,8 @@ async function main() { `${meta.durationSec.toFixed(2)}s`, ); console.log(`helper : ${BIN}`); - console.log(`model : ${MODEL}\n`); + console.log(`model : ${MODEL}`); + console.log(`vad : ${WITH_VAD ? VAD_MODEL : "(none, speech checks skipped)"}\n`); const port = 20100 + Math.floor(process.pid % 400); const child = spawn( @@ -222,6 +236,7 @@ async function main() { "127.0.0.1", "--threads", String(Math.max(1, os.cpus().length)), + ...(WITH_VAD ? ["--vad-model", VAD_MODEL] : []), ], { stdio: ["ignore", "pipe", "pipe"] }, ); @@ -237,6 +252,9 @@ async function main() { }); let json; + let emissions = null; + let alignerModel = null; + let alignerSkipped = ""; try { await waitForReady(`http://127.0.0.1:${port}/`); const form = new FormData(); @@ -249,6 +267,32 @@ async function main() { }); if (!res.ok) throw new Error(`/inference returned ${res.status}: ${await res.text()}`); json = await res.json(); + // The word aligner's acoustic pass, on the same helper, as SttManager runs it. + const cached = CTC_ALIGNERS[json.detected_language]; + alignerModel = + process.env.OPENSCREEN_ALIGNER_MODEL ?? + (cached ? path.join(path.dirname(path.dirname(MODEL)), "ctc-aligner", cached.name) : null); + if (WITH_VAD && alignerModel && fs.existsSync(alignerModel)) { + const speech = (json.speech ?? []).map((x) => ({ startSec: x.start, endSec: x.end })); + const aform = new FormData(); + aform.append("file", new Blob([fs.readFileSync(wavPath)]), path.basename(wavPath)); + aform.append("model", alignerModel); + aform.append("regions", JSON.stringify(emissionRegions(speech, meta.durationSec))); + const ares = await fetch(`http://127.0.0.1:${port}/emissions`, { + method: "POST", + body: aform, + }); + if (!ares.ok) throw new Error(`/emissions returned ${ares.status}: ${await ares.text()}`); + emissions = await ares.json(); + } else { + // Say which file was looked for: a custom OPENSCREEN_WHISPER_MODEL moves + // the cache this path is derived from, and would read as "no aligner". + alignerSkipped = !WITH_VAD + ? "no VAD model, so no speech to align" + : alignerModel + ? `no aligner at ${alignerModel} (set OPENSCREEN_ALIGNER_MODEL)` + : `no aligner for "${json.detected_language}"`; + } } finally { child.kill(); cleanup(); @@ -320,29 +364,110 @@ async function main() { "word starts are monotonic non-decreasing", ); - // NOT a failure. whisper.cpp gives the last word of a segment the segment's - // own t1 as its end, while the word's DTW start runs 80–150 ms late — so a - // token emitted near the boundary (typically standalone punctuation) can land - // after it and invert. The contract's [startSec, endSec) guarantee is enforced - // one layer up, deliberately: whisperServer.ts clamps to - // `Math.max(startSec + 0.02, endSec)` and snapWordBoundaries.ts keeps - // "degenerate words (whisper sometimes reports end <= start) non-empty". This - // harness speaks to the raw helper, below that clamp, so it reports the count - // as information instead of asserting on it. - const inverted = words.filter((w) => w.end < w.start); - if (inverted.length > 0) { - console.log( - ` info ${inverted.length}/${words.length} raw word(s) have end < start ` + - `(${inverted.map((w) => JSON.stringify(w.word)).join(", ")}) — expected at segment ` + - "boundaries; whisperServer.ts clamps these before they reach the document.", + // With the VAD model, whisper decodes only the speech and every time has to + // be mapped back onto the clip. whisper.cpp's own VAD mapped segment times + // but not word times, which put words seconds early (13 s into a 25 s clip) + // while every check above still passed. A word starting in what the VAD + // called silence is that bug; 0.15 s covers the 0.1 s tail kept past each stretch. + if (WITH_VAD) { + const speech = json.speech ?? []; + check(Array.isArray(json.speech), "response carries the speech intervals", `${speech.length}`); + const stray = words.filter( + (w) => !speech.some((s) => w.start >= s.start - 0.15 && w.start <= s.end + 0.15), + ); + check( + stray.length === 0, + "every word starts inside a stretch of speech", + stray.map((w) => `${JSON.stringify(w.word)}@${w.start}`).join(", "), ); } + // A word runs from the end of the token before it to the end of its own last + // token, and the helper rejects a non-monotonic DTW path, so it cannot end + // before it starts. It can be empty (end == start): whisperServer.ts widens + // those to 20 ms before they reach the document. + const inverted = words.filter((w) => w.end < w.start); + check( + inverted.length === 0, + "no word ends before it starts", + inverted.map((w) => JSON.stringify(w.word)).join(", "), + ); + if (refText) { const rate = wer(refText, text); check(rate <= 0.15, "WER within tolerance", `${rate.toFixed(4)}`); } + // The aligner (electron/stt/ctcAlign.ts) only gets letter scores from the + // helper; a forward pass gone wrong on a backend (a missing op, a wrong + // layout) still answers 200 with well-formed garbage. Reading its best letter + // per frame back as text is what tells the two apart. + if (emissions) { + const parsed = parseEmissions(emissions); + check(parsed !== null, "aligner answers well-formed letter scores", alignerModel); + if (parsed) { + const V = parsed.vocab.length; + let worst = 0; + let greedy = ""; + for (const r of parsed.regions) { + let prev = -1; + for (let t = 0; t < r.frames; t++) { + let sum = 0; + let best = 0; + for (let v = 0; v < V; v++) { + sum += Math.exp(r.logprobs[t * V + v]); + if (r.logprobs[t * V + v] > r.logprobs[t * V + best]) best = v; + } + worst = Math.max(worst, Math.abs(sum - 1)); + if (best !== prev && best !== parsed.blank) + greedy += parsed.vocab[best] === "|" ? " " : parsed.vocab[best]; + prev = best; + } + greedy += " "; + } + check( + worst < 1e-3, + "aligner frames are log-probabilities", + `max |sum - 1| = ${worst.toExponential(1)}`, + ); + if (gpuExpected) { + check( + emissions.device !== "CPU", + "aligner runs on the GPU on a GPU-capable host", + emissions.device, + ); + } + // Letters, not words: the aligner spells "OpenScreen" as it hears it. + // Measured: 0.31 on a real French take, 0.11 on English TTS; a forward + // pass gone wrong decodes to noise, near 1. + const letters = (x) => normalize(x).join("").split("").join(" "); + const heard = wer(letters(text), letters(greedy)); + console.log(`\naligner: ${greedy.trim().slice(0, 200)}`); + check( + heard <= 0.5, + "aligner hears what whisper heard", + `letter error rate ${heard.toFixed(2)} on ${emissions.device}`, + ); + const speech = (json.speech ?? []).map((x) => ({ startSec: x.start, endSec: x.end })); + const plain = words.map((w) => ({ + word: w.word.trim(), + startSec: w.start, + endSec: w.end, + anchorSec: w.anchor ?? w.start, + })); + const aligned = alignWordsOnEmissions(plain, speech, parsed); + check( + aligned.every( + (w, i) => w.endSec >= w.startSec && (i === 0 || w.startSec >= aligned[i - 1].startSec), + ), + "aligned words stay ordered and non-empty", + ); + console.log(`aligner: ${emissions.elapsed_s.toFixed(2)}s on ${emissions.device}`); + } + } else { + console.log(`\n(aligner checks skipped: ${alignerSkipped})`); + } + if (json.timing) { console.log( `\ntiming : ${json.timing.elapsed_s?.toFixed?.(2)}s for ` + diff --git a/scripts/test-windows-wgc-helper.mjs b/scripts/test-windows-wgc-helper.mjs index 785e48c1e..62e1d9fde 100644 --- a/scripts/test-windows-wgc-helper.mjs +++ b/scripts/test-windows-wgc-helper.mjs @@ -21,8 +21,33 @@ const WITH_MICROPHONE = process.argv.includes("--mic"); const WITH_WINDOW = process.env.OPENSCREEN_WGC_TEST_WINDOW === "true" || process.argv.includes("--window"); +/** + * Records a window that has a context menu open, next to two unrelated + * topmost popups from other processes, and checks the menu is in the video and + * the strangers are not (getopenscreen/openscreen#894). The second half is the + * point: DWM picks the windows IncludeSecondaryWindows draws by style and + * z-order, so what it lets in has to be measured, not assumed. + * + * It shows windows on the desktop. Run it with no ordinary window (neither a + * popup nor a tool window) over the test zone, about x 400-1110, y 150-685 at + * 100 percent DPI, for as long as it runs: with one above the target the menu is + * missing from the first frame to the last (#910). A run that prints + * `fixture invalid` found one and measures nothing. + */ +const WITH_WINDOW_POPUP = + process.env.OPENSCREEN_WGC_TEST_WINDOW_POPUP === "true" || + process.argv.includes("--window-popup"); const WITH_WEBCAM = process.env.OPENSCREEN_WGC_TEST_WEBCAM === "true" || process.argv.includes("--webcam"); +/** + * Adds a second camera that does not exist to a `--webcam` run, listed through + * the `webcams` config next to the real one: the helper must drop only that + * camera, say so with its index, and still record the first. + */ +const WITH_MISSING_SECOND_WEBCAM = + process.env.OPENSCREEN_WGC_TEST_MISSING_SECOND_WEBCAM === "true" || + process.argv.includes("--missing-second-webcam"); +const MISSING_WEBCAM_NAME = "OpenScreen Nonexistent Camera"; const CAPTURE_CURSOR = process.env.OPENSCREEN_WGC_TEST_CAPTURE_CURSOR === "true" || process.argv.includes("--capture-cursor"); @@ -91,6 +116,9 @@ const STOP_LATENCY_BUDGET_MS = 15_000; if (WITH_SOFTWARE_ENCODER && WITH_SOFTWARE_FALLBACK) { throw new Error("--software-encoder and --software-fallback are mutually exclusive"); } +if (WITH_MISSING_SECOND_WEBCAM && !WITH_WEBCAM) { + throw new Error("--missing-second-webcam needs --webcam"); +} function runHelper( config, @@ -99,10 +127,11 @@ function runHelper( stallReadbackMs = 0, stallFrameCallbackMs = 0, legacyFrameCallback = false, + extraEnv = {}, } = {}, ) { return new Promise((resolve, reject) => { - const env = { ...process.env }; + const env = { ...process.env, ...extraEnv }; delete env[INJECT_DEFAULT_SINK_WRITER_FAILURE_ENV]; delete env[STALL_READBACK_ENV]; delete env[STALL_FRAME_CALLBACK_ENV]; @@ -299,6 +328,390 @@ function startScreenActivity() { return () => child.kill(); } +/** + * One PowerShell file plays both roles of the --window-popup fixture, each in + * its own process: + * + * - `target`: an ordinary titled window that opens a flat-colour context menu + * (a ToolStripDropDown owned by that window, which is what a real app's menu is). + * - `intruder`: a borderless TopMost popup of another flat colour that belongs to + * nobody the target knows. `-Tool 1` adds WS_EX_TOOLWINDOW, the other + * qualifying style, so both are exercised. + * + * DPI-aware so every coordinate is a physical pixel, the unit WGC reports in. + */ +const POPUP_FIXTURE_SCRIPT = String.raw`param([string]$Role, [int]$X, [int]$Y, [int]$W, [int]$H, [string]$Color, [int]$Popup = 0, [int]$Tool = 0, [string]$Placement = 'overhang') +Add-Type -ReferencedAssemblies System.Windows.Forms, System.Drawing -TypeDefinition @" +using System; +using System.Drawing; +using System.Runtime.InteropServices; +using System.Text; +using System.Windows.Forms; +public class PopupForm : Form { + public bool IsPopup, ToolWindow; + protected override bool ShowWithoutActivation { get { return true; } } + protected override CreateParams CreateParams { + get { + CreateParams cp = base.CreateParams; + if (IsPopup) { cp.Style = unchecked((int)0x80000000) | 0x10000000 | 0x06000000; } + cp.ExStyle = (cp.ExStyle & ~0x80) | 0x8 | 0x08000000 | (ToolWindow ? 0x80 : 0); + return cp; + } + } +} +public static class Fx { + [StructLayout(LayoutKind.Sequential)] + public struct RECT { public int Left, Top, Right, Bottom; } + [DllImport("user32.dll")] public static extern bool SetProcessDPIAware(); + [DllImport("user32.dll")] public static extern int GetWindowLong(IntPtr hwnd, int index); + [DllImport("user32.dll")] static extern IntPtr GetWindow(IntPtr hwnd, uint cmd); + [DllImport("user32.dll")] static extern bool IsWindowVisible(IntPtr hwnd); + [DllImport("user32.dll")] static extern bool GetWindowRect(IntPtr hwnd, out RECT rect); + [DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern int GetClassName(IntPtr hwnd, StringBuilder name, int max); + [DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr hwnd, out uint pid); + [DllImport("dwmapi.dll")] public static extern int DwmGetWindowAttribute(IntPtr hwnd, int attr, out RECT rect, int size); + [DllImport("dwmapi.dll")] static extern int DwmGetWindowAttribute(IntPtr hwnd, int attr, out int value, int size); + // Every visible window above the target that overlaps it, top of the z-order + // first, as class/pid/style/exstyle/Q|N. Q marks WS_POPUP or WS_EX_TOOLWINDOW, + // the styles IncludeSecondaryWindows can draw; DWM only draws a window when + // every window between it and the target qualifies, so one N above the target + // would make "the stranger is absent" true for a reason that proves nothing. + public static string ChainAbove(IntPtr target, RECT t) { + StringBuilder sb = new StringBuilder(); + for (IntPtr h = GetWindow(target, 3); h != IntPtr.Zero; h = GetWindow(h, 3)) { + int cloaked; DwmGetWindowAttribute(h, 13, out cloaked, 4); + RECT r; GetWindowRect(h, out r); + if (!IsWindowVisible(h) || cloaked != 0 || r.Right <= t.Left || r.Left >= t.Right || r.Bottom <= t.Top || r.Top >= t.Bottom) continue; + int style = GetWindowLong(h, -16), ex = GetWindowLong(h, -20); + StringBuilder cn = new StringBuilder(64); GetClassName(h, cn, 64); + uint pid; GetWindowThreadProcessId(h, out pid); + bool q = (style & unchecked((int)0x80000000)) != 0 || (ex & 0x80) != 0; + if (sb.Length > 0) sb.Append(';'); + sb.Append(cn.ToString().Replace(' ', '_')).Append('/').Append(pid).Append("/0x").Append(style.ToString("x")).Append("/0x").Append(ex.ToString("x")).Append(q ? "/Q" : "/N"); + } + return sb.Length == 0 ? "-" : sb.ToString(); + } +} +"@ +[void][Fx]::SetProcessDPIAware() +[System.Windows.Forms.Application]::EnableVisualStyles() +$rgb = $Color.Split(',') | ForEach-Object { [int]$_ } +$fill = [System.Drawing.Color]::FromArgb($rgb[0], $rgb[1], $rgb[2]) +if ($Role -eq 'intruder') { + $form = New-Object PopupForm + $form.IsPopup = ($Popup -eq 1) + $form.ToolWindow = ($Tool -eq 1) + $form.FormBorderStyle = 'None' + $form.StartPosition = 'Manual' + $form.Bounds = New-Object System.Drawing.Rectangle($X, $Y, $W, $H) + $form.BackColor = $fill + $form.TopMost = $true + $form.Add_Shown({ + $style = '0x{0:x}' -f [Fx]::GetWindowLong($form.Handle, -16) + $exstyle = '0x{0:x}' -f [Fx]::GetWindowLong($form.Handle, -20) + [Console]::Out.WriteLine("READY hwnd=$($form.Handle.ToInt64()) style=$style exstyle=$exstyle") + [Console]::Out.Flush() + }) + [System.Windows.Forms.Application]::Run($form) + exit +} +$form = New-Object System.Windows.Forms.Form +$form.Text = 'OpenScreen popup fixture' +$form.FormBorderStyle = 'FixedSingle' +$form.MaximizeBox = $false +$form.StartPosition = 'Manual' +$form.Location = New-Object System.Drawing.Point($X, $Y) +$form.ClientSize = New-Object System.Drawing.Size($W, $H) +$form.BackColor = [System.Drawing.Color]::FromArgb(200, 200, 200) +$menu = New-Object System.Windows.Forms.ContextMenuStrip +$menu.AutoClose = $false +$menu.ShowImageMargin = $false +$menu.ShowCheckMargin = $false +$menu.Padding = [System.Windows.Forms.Padding]::Empty +$menu.BackColor = $fill +$label = New-Object System.Windows.Forms.ToolStripLabel +$label.AutoSize = $false +$label.Size = New-Object System.Drawing.Size(240, 160) +$label.BackColor = $fill +[void]$menu.Items.Add($label) +$form.Add_Shown({ + $r = New-Object Fx+RECT + [void][Fx]::DwmGetWindowAttribute($form.Handle, 9, [ref]$r, 16) + $menuX = if ($Placement -eq 'inside') { $r.Left + 60 } else { $r.Right - 140 } + $menu.Show($form, $form.PointToClient((New-Object System.Drawing.Point($menuX, ($r.Top + 80))))) + $b = $menu.Bounds + $chain = [Fx]::ChainAbove($form.Handle, $r) + [Console]::Out.WriteLine("READY hwnd=$($form.Handle.ToInt64()) left=$($r.Left) top=$($r.Top) right=$($r.Right) bottom=$($r.Bottom) menu=$($b.X),$($b.Y),$($b.Width),$($b.Height) chain=$chain") + [Console]::Out.Flush() +}) +[System.Windows.Forms.Application]::Run($form) +`; + +function startPopupFixture(scriptPath, args) { + return new Promise((resolve, reject) => { + const child = spawn( + "powershell", + ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", scriptPath, ...args.map(String)], + { stdio: ["ignore", "pipe", "pipe"], windowsHide: false }, + ); + let stdout = ""; + let stderr = ""; + const timer = setTimeout(() => { + child.kill(); + reject(new Error(`Popup fixture never became ready.\n${stdout}\n${stderr}`)); + }, 30_000); + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + const ready = stdout.match(/^READY (.*)$/m); + if (ready) { + clearTimeout(timer); + resolve({ + child, + info: Object.fromEntries([...ready[1].matchAll(/(\w+)=(\S+)/g)].map((m) => [m[1], m[2]])), + }); + } + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.once("exit", (code) => { + clearTimeout(timer); + reject( + new Error(`Popup fixture exited (${code}) before it was ready.\n${stdout}\n${stderr}`), + ); + }); + }); +} + +/** Flat fixture colours, matched loosely: H.264 shifts a saturated colour a few levels. */ +const POPUP_COLOR = { + menu: { rgb: "255,0,255", match: (r, g, b) => r > 200 && g < 70 && b > 200 }, + stranger: { rgb: "0,255,0", match: (r, g, b) => g > 200 && r < 80 && b < 80 }, + strangerTool: { rgb: "255,255,0", match: (r, g, b) => r > 200 && g > 200 && b < 80 }, + strangerToolOnly: { rgb: "0,255,255", match: (r, g, b) => g > 200 && b > 200 && r < 80 }, +}; +/** + * The three ways a stranger can meet IncludeSecondaryWindows' style rule, one + * process each: WS_POPUP alone, WS_POPUP + WS_EX_TOOLWINDOW (what a real menu or + * tooltip is), and WS_EX_TOOLWINDOW alone. All TopMost, all over the target. + */ +const POPUP_STRANGERS = { + stranger: { x: 300, y: 480, popup: 1, tool: 0 }, + strangerTool: { x: 700, y: 480, popup: 1, tool: 1 }, + strangerToolOnly: { x: 1000, y: 560, popup: 0, tool: 1 }, +}; + +/** The last frame of a recording as raw RGB, so it is measured and not eyeballed. */ +function readLastRgbFrame(videoPath) { + const video = probeStreams(videoPath).find((stream) => stream.codec_type === "video"); + const { width, height } = video; + const ffmpeg = spawnSync( + "ffmpeg", + [ + "-v", + "error", + "-sseof", + "-0.5", + "-i", + videoPath, + "-frames:v", + "1", + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + "pipe:1", + ], + { windowsHide: true, maxBuffer: width * height * 3 + 1024 * 1024 }, + ); + if (ffmpeg.status !== 0 || ffmpeg.stdout.length !== width * height * 3) { + throw new Error(`ffmpeg frame extraction failed: ${ffmpeg.stderr?.toString() ?? ""}`); + } + return { width, height, data: ffmpeg.stdout }; +} + +function measureColor({ width, height, data }, matches) { + const box = { count: 0, minX: Infinity, maxX: -1, minY: Infinity, maxY: -1 }; + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const i = (y * width + x) * 3; + if (matches(data[i], data[i + 1], data[i + 2])) { + box.count += 1; + box.minX = Math.min(box.minX, x); + box.maxX = Math.max(box.maxX, x); + box.minY = Math.min(box.minY, y); + box.maxY = Math.max(box.maxY, y); + } + } + } + return box; +} + +function measurePopupFrame(videoPath) { + const frame = readLastRgbFrame(videoPath); + return { + width: frame.width, + height: frame.height, + menu: measureColor(frame, POPUP_COLOR.menu.match), + stranger: measureColor(frame, POPUP_COLOR.stranger.match), + strangerTool: measureColor(frame, POPUP_COLOR.strangerTool.match), + strangerToolOnly: measureColor(frame, POPUP_COLOR.strangerToolOnly.match), + }; +} + +/** + * One take: `target` shows its menu (`inside`, or `overhang` past the right + * edge), the three strangers sit over the lower half of it, and the helper is + * pointed at the target window three times: with the kill switch (the bug), as + * shipped (the fix), and as a monitor capture (the control that proves the + * strangers really are on screen and really are detectable). + */ +async function runWindowPopupScenario(placement, scriptPath, baseConfig) { + const fixtures = []; + const measured = {}; + try { + // Strangers first, target last: a window created later sits higher in the + // z-order, and the target must have nothing but them above it. + for (const [name, stranger] of Object.entries(POPUP_STRANGERS)) { + fixtures.push( + await startPopupFixture(scriptPath, [ + ...["-Role", "intruder", "-X", stranger.x, "-Y", stranger.y, "-W", "250", "-H", "120"], + ...["-Color", POPUP_COLOR[name].rgb, "-Popup", stranger.popup, "-Tool", stranger.tool], + ]), + ); + } + const target = await startPopupFixture(scriptPath, [ + ...["-Role", "target", "-X", "400", "-Y", "150", "-W", "700", "-H", "500"], + ...["-Color", POPUP_COLOR.menu.rgb, "-Placement", placement], + ]); + fixtures.push(target); + measured.fixture = { + target: target.info, + ...Object.fromEntries( + Object.keys(POPUP_STRANGERS).map((name, i) => [name, fixtures[i].info]), + ), + }; + // Let DWM finish drawing all three before the first frame is asked for. + await new Promise((resolve) => setTimeout(resolve, 1000)); + + const takes = { + killSwitch: { + sourceType: "window", + extraEnv: { OPENSCREEN_WGC_DISABLE_SECONDARY_WINDOWS: "1" }, + }, + shipped: { sourceType: "window", extraEnv: {} }, + monitor: { sourceType: "display", extraEnv: {} }, + }; + for (const [name, take] of Object.entries(takes)) { + const videoPath = path.join( + os.tmpdir(), + `openscreen-wgc-popup-${placement}-${name}-${process.pid}-${Date.now()}.mp4`, + ); + const result = await runHelper( + { + ...baseConfig, + outputPath: videoPath, + outputs: { screenPath: videoPath }, + sourceType: take.sourceType, + sourceId: take.sourceType === "window" ? `window:${target.info.hwnd}:0` : "screen:0:0", + }, + { extraEnv: take.extraEnv }, + ); + assertStopWasClean(result); + if (result.code !== 0) { + throw new Error( + `WGC helper exited with ${result.code}\n${result.stdout}\n${result.stderr}`, + ); + } + const secondary = result.stdout + .split(/\r?\n/) + .find((line) => line.includes('"event":"secondary-windows"')); + measured[name] = { + secondaryWindows: secondary ? JSON.parse(secondary).applied : null, + ...measurePopupFrame(videoPath), + }; + fs.rmSync(videoPath, { force: true }); + } + } finally { + for (const fixture of fixtures) { + fixture.child.kill(); + } + } + return measured; +} + +function assertWindowPopupScenario(placement, m) { + const failures = []; + const target = m.fixture.target; + const ext = { width: Number(target.right) - Number(target.left) }; + const [menuX, , menuW] = target.menu.split(",").map(Number); + const menuLeftInFrame = menuX - Number(target.left); + // Everything above the target must qualify, or a missing stranger proves nothing. + const blockers = target.chain.split(";").filter((entry) => entry.endsWith("/N")); + if (blockers.length > 0) { + failures.push( + `fixture invalid, non-qualifying windows sit above the target: ${blockers.join(" ")}`, + ); + } + // The baseline is the bug itself; if the menu shows up with the option off, this + // fixture is not testing what it claims to. + if (m.killSwitch.secondaryWindows !== null || m.killSwitch.menu.count > 20) { + failures.push(`kill switch did not remove the menu: ${JSON.stringify(m.killSwitch)}`); + } + // The control: the strangers must be visible to a monitor capture, or "absent" below + // means nothing. + for (const name of ["menu", ...Object.keys(POPUP_STRANGERS)]) { + if (m.monitor[name].count < 1000) { + failures.push( + `control monitor capture did not see the ${name} colour: ${m.monitor[name].count}px`, + ); + } + } + if (m.shipped.secondaryWindows === false) { + console.log( + "Windows runtime cannot include secondary windows (needs 11 24H2, build 26100): skipping the menu check.", + ); + } else { + if (m.shipped.secondaryWindows !== true) { + failures.push( + `helper did not report secondary-windows applied: ${m.shipped.secondaryWindows}`, + ); + } + if (m.shipped.menu.count < 5000) { + failures.push(`menu missing from the window recording: ${m.shipped.menu.count}px`); + } + const visibleWidth = m.shipped.menu.maxX - m.shipped.menu.minX + 1; + if (placement === "overhang") { + // Clipped at the window edge: touches the last column, never widens the frame. + if (m.shipped.menu.maxX < m.shipped.width - 2) { + failures.push( + `overhanging menu does not reach the frame edge: maxX=${m.shipped.menu.maxX}`, + ); + } + const expected = m.shipped.width - menuLeftInFrame; + if (Math.abs(visibleWidth - expected) > 8 || visibleWidth >= menuW) { + failures.push( + `overhanging menu is not clipped: ${visibleWidth}px visible, expected about ${expected}`, + ); + } + } else if (Math.abs(visibleWidth - menuW) > 8) { + failures.push(`menu inside the window is ${visibleWidth}px wide, expected about ${menuW}`); + } + if (Math.abs(m.shipped.width - ext.width) > 2) { + failures.push(`frame width ${m.shipped.width} differs from the window's ${ext.width}`); + } + } + // The privacy half: another process's topmost popups must not leak in. + for (const name of Object.keys(POPUP_STRANGERS)) { + if (m.shipped[name].count >= 20) { + failures.push( + `LEAK: ${name} popup found in the window recording: ${m.shipped[name].count}px`, + ); + } + } + return failures; +} + function normalizeDeviceName(value) { return value .toLowerCase() @@ -459,6 +872,9 @@ const outputPath = path.join( `openscreen-wgc-helper-${WITH_WEBCAM ? "webcam" : WITH_WINDOW ? "window" : WITH_SYSTEM_AUDIO || WITH_MICROPHONE ? "audio" : "video"}-${process.pid}-${Date.now()}-${randomUUID()}.mp4`, ); const webcamOutputPath = WITH_WEBCAM ? outputPath.replace(/\.mp4$/i, "-webcam.mp4") : null; +const missingWebcamOutputPath = WITH_MISSING_SECOND_WEBCAM + ? outputPath.replace(/\.mp4$/i, "-webcam-2.mp4") + : null; const fixtureWindow = WITH_WINDOW ? await startFixtureWindow() : null; @@ -490,15 +906,67 @@ const config = { webcamDirectShowClsid: resolveDirectShowWebcamClsid( process.env.OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_NAME ?? "", ), - webcamWidth: 640, - webcamHeight: 360, - webcamFps: 30, + // DEFAULT_WEBCAM_QUALITY from src/hooks/webcamCaptureTarget.ts -- the target the + // app actually sends, so this exercises the format negotiation rather than a + // size no shipped pipeline ever asks for. A camera that cannot reach it is + // driven at its own best format, which is the case worth covering anyway. + webcamWidth: Number(process.env.OPENSCREEN_WGC_TEST_WEBCAM_WIDTH ?? 3840), + webcamHeight: Number(process.env.OPENSCREEN_WGC_TEST_WEBCAM_HEIGHT ?? 2160), + webcamFps: Number(process.env.OPENSCREEN_WGC_TEST_WEBCAM_FPS ?? 30), outputs: { screenPath: outputPath, ...(webcamOutputPath ? { webcamPath: webcamOutputPath } : {}), }, }; +if (WITH_MISSING_SECOND_WEBCAM) { + config.webcams = [ + { + camDeviceId: config.webcamDeviceId, + camDeviceName: config.webcamDeviceName, + camClsid: config.webcamDirectShowClsid, + camWidth: config.webcamWidth, + camHeight: config.webcamHeight, + camFps: config.webcamFps, + camPath: webcamOutputPath, + }, + { + camDeviceId: "", + camDeviceName: MISSING_WEBCAM_NAME, + camClsid: "", + camWidth: 1280, + camHeight: 720, + camFps: 30, + camPath: missingWebcamOutputPath, + }, + ]; +} + +if (WITH_WINDOW_POPUP) { + const scriptPath = path.join(os.tmpdir(), `openscreen-popup-fixture-${process.pid}.ps1`); + fs.writeFileSync(scriptPath, POPUP_FIXTURE_SCRIPT); + const scenarios = {}; + const failures = []; + try { + for (const placement of ["inside", "overhang"]) { + scenarios[placement] = await runWindowPopupScenario(placement, scriptPath, config); + failures.push( + ...assertWindowPopupScenario(placement, scenarios[placement]).map( + (failure) => `${placement}: ${failure}`, + ), + ); + } + } finally { + fs.rmSync(scriptPath, { force: true }); + } + console.log(JSON.stringify(scenarios, null, 2)); + if (failures.length > 0) { + throw new Error(`WGC window popup check failed:\n${failures.join("\n")}`); + } + console.log("WGC window popup check passed"); + process.exit(0); +} + const stopScreenActivity = WITH_STALLED_FRAME_CALLBACK ? startScreenActivity() : null; let result; try { @@ -631,9 +1099,23 @@ const encoderSelectionLine = result.stdout .split(/\r?\n/) .find((line) => line.includes('"event":"encoder-selection"')); const encoderSelection = encoderSelectionLine ? JSON.parse(encoderSelectionLine) : null; -const nativeWebcamDiagnostics = result.stderr - .split(/\r?\n/) - .filter((line) => line.includes("Native webcam candidate")); +const nativeWebcamDiagnostics = result.stderr.split(/\r?\n/).filter( + (line) => + line.includes("Native webcam candidate") || + // Which capture format the camera was actually driven at. Without this + // the smoke test could pass on a 640x480 take from a 4K camera and say + // nothing about it. + line.includes("Native webcam format") || + line.includes("DirectShow webcam format") || + line.includes("DirectShow webcam connected") || + line.includes("falling back to the device default") || + // How long the camera took to produce its first frame, and the tally of what + // the capture loop did with everything it read. Without these, a camera that + // delivers nothing is only visible as a failed Finalize, which cannot say why. + line.includes("First webcam frame") || + line.includes("Webcam capture loop ended") || + line.includes("Webcam frame is"), +); const nativeMicrophoneDiagnostics = result.stderr .split(/\r?\n/) .filter( @@ -667,6 +1149,35 @@ if ( if (WITH_WEBCAM && !webcamStreams.some((stream) => stream.codec_type === "video")) { throw new Error(`WGC helper webcam output has no video stream: ${webcamOutputPath}`); } +if (WITH_MISSING_SECOND_WEBCAM) { + const unavailable = result.stdout + .split(/\r?\n/) + .filter((line) => line.includes('"code":"webcam-unavailable"')) + .map((line) => JSON.parse(line.slice(line.indexOf('{"event"')))); + if (!unavailable.some((event) => event.index === 1 && event.deviceName === MISSING_WEBCAM_NAME)) { + throw new Error( + `WGC helper did not report camera 1 (${MISSING_WEBCAM_NAME}) as unavailable: ${result.stdout}`, + ); + } + const stoppedLine = result.stdout + .split(/\r?\n/) + .find((line) => line.includes('"event":"recording-stopped"')); + const stopped = stoppedLine + ? JSON.parse(stoppedLine.slice(stoppedLine.indexOf('{"event"'))) + : null; + if (JSON.stringify(stopped?.webcamPaths) !== JSON.stringify([webcamOutputPath])) { + throw new Error( + `recording-stopped.webcamPaths should list only the real camera: ${stoppedLine ?? "missing"}`, + ); + } + if (fs.existsSync(missingWebcamOutputPath) && fs.statSync(missingWebcamOutputPath).size > 0) { + throw new Error(`WGC helper wrote a file for the missing camera: ${missingWebcamOutputPath}`); + } + console.log("WGC helper missing-second-webcam check passed", { + unavailable, + webcamPaths: stopped.webcamPaths, + }); +} if ( (CAPTURE_CURSOR && !cursorCapture) || (cursorCapture && diff --git a/src/cli/CliExportRunner.clipList.test.ts b/src/cli/CliExportRunner.clipList.test.ts new file mode 100644 index 000000000..39696c400 --- /dev/null +++ b/src/cli/CliExportRunner.clipList.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { type AxcutAsset, type AxcutDocument, axcutSchemaVersion } from "@/lib/ai-edition/schema"; +import { buildNativeClipList } from "./CliExportRunner"; + +function doc(asset: AxcutAsset): AxcutDocument { + return { + schemaVersion: axcutSchemaVersion, + project: { + id: "proj_1", + title: "Cli", + createdAt: "2026-10-05T10:00:00Z", + updatedAt: "2026-10-05T10:00:00Z", + primaryAssetId: asset.id, + }, + assets: [asset], + transcript: null, + transcripts: [], + timeline: { + clips: [ + { + id: "c1", + assetId: asset.id, + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + wordRefs: [], + origin: "user", + reason: "", + }, + ], + gaps: [], + trimRanges: [], + muteRanges: [], + speedRanges: [], + captionRanges: [], + }, + annotations: [], + zoomRanges: [], + audioTracks: [], + legacyEditor: null, + }; +} + +const ASSET: AxcutAsset = { + id: "a1", + kind: "video", + label: "asset", + originalPath: "/tmp/a.mp4", + cameraTrack: null, +}; + +describe("CLI export clip list", () => { + it("carries the asset's extra cameras", () => { + const clips = buildNativeClipList( + doc({ + ...ASSET, + additionalCameraTracks: [ + { sourcePath: "/tmp/cam2.mp4", startMs: 1000, offsetMs: -250, visible: true, label: "" }, + ], + }), + ); + expect(clips).toHaveLength(1); + expect(clips[0].additionalCameras).toEqual([{ path: "/tmp/cam2.mp4", offsetSec: 0.75 }]); + }); + + it("sends no extra cameras for a one-camera asset", () => { + const clips = buildNativeClipList(doc(ASSET)); + expect(clips[0]).not.toHaveProperty("additionalCameras"); + }); +}); diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx index de080359e..52f28ff45 100644 --- a/src/cli/CliExportRunner.tsx +++ b/src/cli/CliExportRunner.tsx @@ -26,7 +26,7 @@ import { appendAutoZoomSuggestions, collectAutoZoomSuggestionsForDocument, } from "@/lib/ai-edition/timeline/apply-auto-zooms"; -import { assetCameraSource } from "@/lib/ai-edition/timeline/camera"; +import { assetAdditionalCameraSources, assetCameraSource } from "@/lib/ai-edition/timeline/camera"; import { resolveClipSourceEndSec } from "@/lib/ai-edition/timeline/clipDuration"; import type { CliDoneResult, CliExportRequest } from "@/lib/cliContracts"; import { GIF_SIZE_PRESETS, type GifSizePreset } from "@/lib/exporter"; @@ -79,7 +79,7 @@ function replaceExtension(filePath: string, newExtension: string): string { /** Mirrors ExportDialog.buildNativeClipList: trim-narrowed visible clips mapped * onto the native multiclip contract. Kept in lock-step with * buildSceneDescription so export and scene agree on the clip stream. */ -function buildNativeClipList(axcutDocument: AxcutDocument): CompositorClipInput[] { +export function buildNativeClipList(axcutDocument: AxcutDocument): CompositorClipInput[] { const assetById = new Map(axcutDocument.assets.map((asset) => [asset.id, asset])); return resolveVisibleClips(axcutDocument).flatMap((clip) => { const asset = assetById.get(clip.assetId); @@ -87,6 +87,8 @@ function buildNativeClipList(axcutDocument: AxcutDocument): CompositorClipInput[ return []; } const camera = assetCameraSource(asset); + // Cameras 2-4, only sent when the asset has any (same rule as `buildSceneDescription`). + const additionalCameras = assetAdditionalCameraSources(asset); const sourceEndSec = resolveClipSourceEndSec(clip, asset); return [ { @@ -96,6 +98,7 @@ function buildNativeClipList(axcutDocument: AxcutDocument): CompositorClipInput[ sourceEndSec, webcamOffsetSec: camera.offsetSec, hasAudio: true, + ...(additionalCameras.length > 0 ? { additionalCameras } : {}), }, ]; }); diff --git a/src/components/ai-edition/AnnotationLayer.test.tsx b/src/components/ai-edition/AnnotationLayer.test.tsx index 9f2ea890c..aff91ba33 100644 --- a/src/components/ai-edition/AnnotationLayer.test.tsx +++ b/src/components/ai-edition/AnnotationLayer.test.tsx @@ -1,7 +1,8 @@ // @vitest-environment jsdom -import { render } from "@testing-library/react"; -import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { act, render } from "@testing-library/react"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import type { AxcutAnnotationRegion } from "@/lib/ai-edition/schema"; +import { publishFootageQuad } from "@/native/footageQuadStore"; import { AnnotationLayer } from "./AnnotationLayer"; // react-rnd reads its offset in the parent off `getBoundingClientRect` on mount, and jsdom lays @@ -18,6 +19,7 @@ beforeAll(() => { }); }); afterAll(() => vi.restoreAllMocks()); +afterEach(() => publishFootageQuad(null)); function region(overrides: Partial): AxcutAnnotationRegion { return { @@ -46,11 +48,11 @@ function region(overrides: Partial): AxcutAnnotationRegio // A 1000×500 frame, the footage padded in to 800×400 at (100, 50). const FOOTAGE = { x: 100, y: 50, width: 800, height: 400 }; -function layer(annotations: AxcutAnnotationRegion[]) { +function layer(annotations: AxcutAnnotationRegion[], selectedAnnotationId: string | null = null) { const view = render( (".react-draggable")]; - return { frame, boxes }; + return { frame, boxes, view }; +} + +/** The corners of the selected blur's gimbal, frame px. */ +function maskCorners(container: HTMLElement): number[][] { + return [...container.querySelectorAll("[data-testid=mask-gimbal] circle")].map((c) => [ + Math.round(Number(c.getAttribute("cx")) * 1000) / 1000, + Math.round(Number(c.getAttribute("cy")) * 1000) / 1000, + ]); } describe("AnnotationLayer", () => { @@ -82,12 +92,47 @@ describe("AnnotationLayer", () => { expect(boxes[0].style.width).toBe("240px"); }); - it("keeps a blur inside the footage it hides", () => { - const { frame, boxes } = layer([region({ type: "blur", content: "" })]); - const container = boxes[0].parentElement as HTMLElement; - expect(container).not.toBe(frame); - expect(container.style.left).toBe("100px"); - expect(container.style.width).toBe("800px"); - expect(boxes[0].style.transform).toBe("translate(80px,80px)"); + it("puts a blur's gimbal on the footage it hides, at rest", () => { + const { view, boxes } = layer([region({ type: "blur", content: "" })], "a"); + expect(boxes).toHaveLength(0); + // 100 + 10 % of 800 = 180, 50 + 20 % of 400 = 130; 30 % × 40 % of the footage. + expect(maskCorners(view.container)).toEqual([ + [180, 130], + [420, 130], + [420, 290], + [180, 290], + ]); + }); + + it("moves a blur's gimbal with the footage the compositor draws: zoomed, then leaning", () => { + const { view } = layer([region({ type: "blur", content: "" })], "a"); + // The frame on screen shows the footage zoomed ×2 around its middle. + act(() => publishFootageQuad([-0.3, -0.3, 1.3, -0.3, 1.3, 1.3, -0.3, 1.3], false)); + expect(maskCorners(view.container)).toEqual([ + [-140, 10], + [340, 10], + [340, 330], + [-140, 330], + ]); + // Leaning under the real camera: the gimbal is a quad, not a rect. + act(() => publishFootageQuad([0.1, 0.1, 0.9, 0.2, 0.9, 0.8, 0.1, 0.9], true)); + const [tl, tr] = maskCorners(view.container); + expect(tr[1]).toBeGreaterThan(tl[1]); + }); + + it("frames an arrow, not the empty box around it", () => { + const arrow = region({ + type: "figure", + space: "frame", + position: { x: 10, y: 20 }, + size: { width: 60, height: 60 }, + figureData: { arrowDirection: "down-right", color: "#fff", strokeWidth: 8 }, + }); + const { boxes } = layer([arrow], "a"); + // Its box is 600 × 300 px: the arrow takes the height (scale 3), centred 150 px in, and + // its strokes span 21..79 of 100. + expect(boxes[0].style.transform).toBe(`translate(${100 + 150 + 63}px,${100 + 63}px)`); + expect(boxes[0].style.width).toBe(`${58 * 3}px`); + expect(boxes[0].style.height).toBe(`${58 * 3}px`); }); }); diff --git a/src/components/ai-edition/AnnotationLayer.tsx b/src/components/ai-edition/AnnotationLayer.tsx index 7020d7967..67ce481cd 100644 --- a/src/components/ai-edition/AnnotationLayer.tsx +++ b/src/components/ai-edition/AnnotationLayer.tsx @@ -6,13 +6,17 @@ // list and a single selection id instead. // // The layer covers the whole preview frame: text, images and arrows are placed on the frame -// and move anywhere in it, padding included. A privacy blur stays on the footage, so its boxes -// live in a child sized to the footage rect and cannot leave it (see -// `lib/ai-edition/annotations/placement.ts`). +// and move anywhere in it, padding included. A privacy blur stays on the footage and follows +// it: the compositor paints the mask on the zoomed footage, leaning under 3D, so its gimbal goes +// through the footage of the frame on screen (`MaskGimbal`, `FootageQuad`), and cannot leave it +// (see `lib/ai-edition/annotations/placement.ts`). +import { rectQuad } from "@/lib/ai-edition/annotations/footageQuad"; import { belongsInFrame, fitTextBox, toFrameSpace } from "@/lib/ai-edition/annotations/placement"; import type { AxcutAnnotationRegion } from "@/lib/ai-edition/schema"; +import { useFootageQuad } from "@/native/footageQuadStore"; import { AnnotationOverlay, type PxRect } from "./AnnotationOverlay"; +import { MaskGimbal } from "./MaskGimbal"; interface AnnotationLayerProps { annotations: AxcutAnnotationRegion[]; @@ -40,6 +44,8 @@ export function AnnotationLayer({ onCommit, }: AnnotationLayerProps) { const currentTimeMs = Math.round(currentTimeSec * 1000); + // Where the footage lies in the frame on screen; at rest (no frame yet), its layout rect. + const liveFootage = useFootageQuad(); const visible = annotations .filter((annotation) => { @@ -66,7 +72,6 @@ export function AnnotationLayer({ } const frameBox: PxRect = { x: 0, y: 0, width: frameWidth, height: frameHeight }; - const footageBox: PxRect = { x: 0, y: 0, width: footage.width, height: footage.height }; // The footage as fractions of the frame: the form the conversions in `placement.ts` take. const footageInFrame = { x: footage.x / frameWidth, @@ -76,48 +81,51 @@ export function AnnotationLayer({ }; const fitText = (region: AxcutAnnotationRegion) => fitTextBox(region, frameWidth / frameHeight); - const overlay = (annotation: AxcutAnnotationRegion, inFrame: boolean) => { - const width = inFrame ? frameWidth : footage.width; - const height = inFrame ? frameHeight : footage.height; - return ( - - ); - }; + const overlay = (annotation: AxcutAnnotationRegion) => ( + + ); + const quad = + liveFootage ?? + rectQuad(footageInFrame.x, footageInFrame.y, footageInFrame.width, footageInFrame.height); return (
- {visible.filter((a) => belongsInFrame(a.type)).map((a) => overlay(a, true))} -
- {visible.filter((a) => !belongsInFrame(a.type)).map((a) => overlay(a, false))} -
+ {visible.filter((a) => belongsInFrame(a.type)).map(overlay)} + {/* Only the selected blur has a gimbal: the mask itself is painted by the compositor. */} + {visible + .filter((a) => !belongsInFrame(a.type) && a.id === selectedAnnotationId) + .map((a) => ( + + ))}
); } diff --git a/src/components/ai-edition/AnnotationOverlay.tsx b/src/components/ai-edition/AnnotationOverlay.tsx index 356a740fe..f260847dc 100644 --- a/src/components/ai-edition/AnnotationOverlay.tsx +++ b/src/components/ai-edition/AnnotationOverlay.tsx @@ -1,6 +1,8 @@ -// Chrome d'édition d'une annotation : cadre de sélection, glissement, poignées de -// redimensionnement. Les PIXELS de l'annotation — texte, image, flèche, flou — sont peints par le -// compositeur natif, aperçu compris. +// Chrome d'édition d'une annotation posée sur le cadre (texte, image, flèche) : cadre de +// sélection, glissement, poignées de redimensionnement. Les PIXELS de l'annotation sont peints par +// le compositeur natif, aperçu compris. Un flou suit le métrage : son gimbal est `MaskGimbal`. +// Une flèche se dessine au milieu de sa boîte, à l'échelle de son plus petit côté : son cadre +// encadre le dessin (`arrowRect`), et ses gestes écrivent la boîte qui le redonne (`arrowBox`). // // Ce fichier était le port du `AnnotationOverlay` de l'éditeur v2 : il rendait les quatre types en // DOM et portait la saisie du tracé libre, soit ~400 lignes qui ne s'exécutaient plus depuis que @@ -11,6 +13,7 @@ import { useEffect, useRef, useState } from "react"; import { Rnd } from "react-rnd"; +import { arrowBox, arrowRect } from "@/lib/ai-edition/annotations/arrowBounds"; import type { AxcutAnnotationRegion } from "@/lib/ai-edition/schema"; import { clampToBound } from "@/lib/projectDefaults"; import { cn } from "@/lib/utils"; @@ -27,15 +30,15 @@ export interface PxRect { interface AnnotationOverlayProps { annotation: Region; isSelected: boolean; - /** Taille, en px, de l'élément où la boîte se déplace : le cadre, ou le footage pour un flou. - * Chaque écriture est un pourcentage de cet élément. */ + /** Taille, en px, de l'élément où la boîte se déplace : le cadre. Chaque écriture est un + * pourcentage de cet élément. */ containerWidth: number; containerHeight: number; /** D'où se lit le rect enregistré, en px dans cet élément : l'élément lui-même, ou le rect du * footage pour une annotation encore rangée sur lui mais qui se déplace dans le cadre. */ box: PxRect; /** Le patch qui range l'annotation dans le cadre, appliqué avant son premier geste ; `null` - * quand elle y est déjà (ou qu'elle n'en sort jamais, comme un flou). */ + * quand elle y est déjà. */ toFrame: Partial | null; /** Texte seulement : la boîte taillée sur le texte, centre conservé. */ fitText?: (region: Region) => Pick; @@ -72,12 +75,27 @@ export function AnnotationOverlay({ zIndex, isSelectedBoost, }: AnnotationOverlayProps) { - const committedX = box.x + (annotation.position.x / 100) * box.width; - const committedY = box.y + (annotation.position.y / 100) * box.height; - const committedWidth = (annotation.size.width / 100) * box.width; - const committedHeight = (annotation.size.height / 100) * box.height; - const blurShape = annotation.type === "blur" ? (annotation.blurData?.shape ?? "rectangle") : null; const isText = annotation.type === "text"; + const arrow = + annotation.type === "figure" + ? { + direction: annotation.figureData?.arrowDirection ?? "right", + stroke: annotation.figureData?.strokeWidth ?? 4, + } + : null; + // Le cadre du geste : la boîte rangée, ou, pour une flèche, la flèche dessinée dedans. + const stored: PxRect = { + x: box.x + (annotation.position.x / 100) * box.width, + y: box.y + (annotation.position.y / 100) * box.height, + width: (annotation.size.width / 100) * box.width, + height: (annotation.size.height / 100) * box.height, + }; + const { + x: committedX, + y: committedY, + width: committedWidth, + height: committedHeight, + } = arrow ? arrowRect(stored, arrow.direction, arrow.stroke) : stored; const isDraggingRef = useRef(false); // Le geste en cours a-t-il bougé quelque chose ? Un clic sur une annotation sélectionnée passe // aussi par le début et la fin d'un glissement : sans mouvement, il n'écrit rien. @@ -103,14 +121,22 @@ export function AnnotationOverlay({ const { x, y, width, height } = liveRect; - /** Un rect en px de l'élément, en pourcentages de lui : ce que le document range. */ - const toPct = (rect: PxRect): Pick => ({ - position: { x: (rect.x / containerWidth) * 100, y: (rect.y / containerHeight) * 100 }, - size: { - width: (rect.width / containerWidth) * 100, - height: (rect.height / containerHeight) * 100, - }, - }); + /** Le cadre du geste en px de l'élément, en pourcentages de lui : ce que le document range. + * Pour une flèche, la boîte qui la dessine là (carrée : la flèche ne s'étire pas). */ + const toPct = (frame: PxRect): Pick => { + const rect = arrow ? arrowBox(frame, arrow.direction, arrow.stroke) : frame; + return { + position: { x: (rect.x / containerWidth) * 100, y: (rect.y / containerHeight) * 100 }, + size: { + width: (rect.width / containerWidth) * 100, + height: (rect.height / containerHeight) * 100, + }, + }; + }; + /** Ce qu'un glissement écrit : la position ; pour une flèche aussi la taille, parce que sa + * boîte devient carrée au premier geste. */ + const moved = (frame: PxRect): Partial => + arrow ? toPct(frame) : { position: toPct(frame).position }; // Une annotation encore sur le footage passe dans le cadre à son premier mouvement, au pixel // près : les écritures qui suivent sont des pourcentages du cadre. @@ -152,14 +178,12 @@ export function AnnotationOverlay({ // Pousse la position PENDANT le geste : c'est le natif qui peint, il doit donc suivre // le curseur. `onChange` ne met à jour qu'en mémoire ; l'écriture disque se fait une // seule fois, au relâchement (`onCommit`). - onChange(annotation.id, { - position: toPct({ x: d.x, y: d.y, width, height }).position, - }); + onChange(annotation.id, moved({ x: d.x, y: d.y, width, height })); }} onDragStop={(_e, d) => { if (movedRef.current) { setLiveRect((prev) => ({ ...prev, x: d.x, y: d.y })); - settle({ position: toPct({ x: d.x, y: d.y, width, height }).position }); + settle(moved({ x: d.x, y: d.y, width, height })); } movedRef.current = false; setTimeout(() => { @@ -202,28 +226,22 @@ export function AnnotationOverlay({ onClick(annotation.id); }} bounds="parent" - // Un texte garde sa forme : sa boîte est celle de ses mots, seule la taille change. - lockAspectRatio={isText} + // Un texte garde sa forme : sa boîte est celle de ses mots, seule la taille change. Une + // flèche aussi : elle ne s'étire pas, elle grandit. + lockAspectRatio={isText || arrow !== null} className={cn( "cursor-move", - isSelected && - annotation.type !== "blur" && - "ring-2 ring-[#34B27B] ring-offset-2 ring-offset-transparent", + isSelected && "ring-2 ring-[#34B27B] ring-offset-2 ring-offset-transparent", )} style={{ zIndex: isSelectedBoost ? zIndex + 1000 : zIndex, pointerEvents: isSelected ? "auto" : "none", - border: - isSelected && annotation.type !== "blur" ? "2px solid rgba(52, 178, 123, 0.8)" : "none", - backgroundColor: - isSelected && annotation.type !== "blur" ? "rgba(52, 178, 123, 0.1)" : "transparent", - boxShadow: - isSelected && annotation.type !== "blur" ? "0 0 0 1px rgba(52, 178, 123, 0.35)" : "none", + border: isSelected ? "2px solid rgba(52, 178, 123, 0.8)" : "none", + backgroundColor: isSelected ? "rgba(52, 178, 123, 0.1)" : "transparent", + boxShadow: isSelected ? "0 0 0 1px rgba(52, 178, 123, 0.35)" : "none", }} - // Un flou en tracé libre se déplace et se redimensionne comme les autres : ce qui le - // bloquait, c'était la zone de saisie du tracé qui capturait le pointeur — et elle est - // partie avec l'outil. Un texte ne se tire que par ses coins, qui agrandissent ses lettres. - enableResizing={isSelected ? (isText ? CORNERS_ONLY : true) : false} + // Un texte et une flèche ne se tirent que par leurs coins, qui les agrandissent. + enableResizing={isSelected ? (isText || arrow ? CORNERS_ONLY : true) : false} disableDragging={!isSelected} resizeHandleStyles={{ topLeft: { @@ -268,23 +286,7 @@ export function AnnotationOverlay({ }, }} > -
- {/* Le cadre d'un flou sélectionné, à la forme du masque. Les autres types portent le - leur sur le `Rnd` lui-même ; un flou n'en a pas, pour ne pas encadrer la zone qu'il - est censé cacher — sans ce liseré il n'aurait AUCUN retour de sélection. */} - {isSelected && annotation.type === "blur" ? ( -
- ) : null} -
+
); } diff --git a/src/components/ai-edition/CameraCalibrationModal.test.tsx b/src/components/ai-edition/CameraCalibrationModal.test.tsx new file mode 100644 index 000000000..0e68013dc --- /dev/null +++ b/src/components/ai-edition/CameraCalibrationModal.test.tsx @@ -0,0 +1,330 @@ +// @vitest-environment jsdom +import "@testing-library/jest-dom"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { CameraSettings } from "@/components/video-editor/types"; + +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: (scope: string) => (key: string) => `${scope}.${key}`, +})); + +// A small still: 50 x 40 RGBA. jsdom has neither `ImageData` nor a video decoder. +const IMAGE = { width: 50, height: 40, data: new Uint8ClampedArray(50 * 40 * 4) } as ImageData; +vi.mock("@/lib/ai-edition/timeline/grabFrame", () => ({ + grabFrame: vi.fn(async () => IMAGE), +})); + +const detectCornerMarkers = vi.fn(); +vi.mock("@/lib/arucoMarkers", () => ({ + detectCornerMarkers: (...args: unknown[]) => detectCornerMarkers(...args), +})); +const printMarkerSheet = vi.fn(); +vi.mock("@/lib/markerSheet", () => ({ + printMarkerSheet: (...args: unknown[]) => printMarkerSheet(...args), +})); + +import { CameraCalibrationModal } from "./CameraCalibrationModal"; + +const CAMERA = { index: 1, label: "Desk", src: "file:///cam.mp4", timeSec: 2 }; + +function renderModal( + mode: "perspective" | "crop", + initial: CameraSettings | null, + onApply = vi.fn(), + onClose = vi.fn(), +) { + render( + , + ); + return { onApply, onClose }; +} + +const apply = () => screen.getByRole("button", { name: "dialogs.cameraCalibration.apply" }); +const stillLoaded = () => + waitFor(() => expect(screen.queryByText("dialogs.cameraCalibration.loading")).toBeNull()); + +describe("CameraCalibrationModal", () => { + beforeEach(() => { + // jsdom has no 2D canvas: `getContext` would log "not implemented" and return null. The + // dialog draws nothing without a context, which is what these tests assume. + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); + }); + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("apply is disabled for a crossed quad", async () => { + renderModal("perspective", { + perspective: { + corners: [ + { x: 0.1, y: 0.1 }, + { x: 0.9, y: 0.9 }, + { x: 0.9, y: 0.1 }, + { x: 0.1, y: 0.9 }, + ], + aspect: 1, + }, + }); + await stillLoaded(); + expect(apply()).toBeDisabled(); + expect(screen.getByRole("alert")).toHaveTextContent("dialogs.cameraCalibration.invalidQuad"); + }); + + it("apply stores corners, aspect and margin in one call", async () => { + const { onApply, onClose } = renderModal("perspective", null); + await stillLoaded(); + fireEvent.click(screen.getByRole("button", { name: "1:1" })); + fireEvent.change(screen.getByRole("slider"), { target: { value: "10" } }); + fireEvent.click(apply()); + expect(onApply).toHaveBeenCalledTimes(1); + expect(onApply).toHaveBeenCalledWith({ + perspective: { + corners: [ + { x: 0.15, y: 0.15 }, + { x: 0.85, y: 0.15 }, + { x: 0.85, y: 0.85 }, + { x: 0.15, y: 0.85 }, + ], + aspect: 1, + margin: 0.1, + }, + }); + expect(onClose).toHaveBeenCalled(); + }); + + it("arrow keys move the focused handle", async () => { + const { onApply } = renderModal("perspective", null); + await stillLoaded(); + const handle = screen.getByTestId("calibration-handle-0"); + handle.focus(); + // One image pixel right, then ten down. + fireEvent.keyDown(handle, { key: "ArrowRight" }); + fireEvent.keyDown(handle, { key: "ArrowDown", shiftKey: true }); + fireEvent.click(apply()); + const corners = onApply.mock.calls[0][0].perspective.corners; + expect(corners[0].x).toBeCloseTo(0.15 + 1 / 50, 9); + expect(corners[0].y).toBeCloseTo(0.15 + 10 / 40, 9); + expect(corners[1]).toEqual({ x: 0.85, y: 0.15 }); + // The A4 landscape default. + expect(onApply.mock.calls[0][0].perspective.aspect).toBeCloseTo(297 / 210, 9); + }); + + it("crop mode stores a crop", async () => { + const { onApply } = renderModal("crop", null); + await stillLoaded(); + const crop = screen.getByTestId("calibration-crop"); + // Shrink from the right edge, then move right. + for (let i = 0; i < 20; i++) fireEvent.keyDown(crop, { key: "ArrowLeft", shiftKey: true }); + for (let i = 0; i < 5; i++) fireEvent.keyDown(crop, { key: "ArrowRight" }); + fireEvent.click(apply()); + expect(onApply).toHaveBeenCalledTimes(1); + const stored = onApply.mock.calls[0][0].crop; + expect(stored.x).toBeCloseTo(0.05, 9); + expect(stored.y).toBe(0); + expect(stored.width).toBeCloseTo(0.8, 9); + expect(stored.height).toBe(1); + }); + + it("cancel stores nothing", async () => { + const { onApply, onClose } = renderModal("perspective", null); + await stillLoaded(); + fireEvent.keyDown(screen.getByTestId("calibration-handle-2"), { key: "ArrowLeft" }); + fireEvent.click(screen.getByRole("button", { name: "common.actions.cancel" })); + expect(onClose).toHaveBeenCalled(); + expect(onApply).not.toHaveBeenCalled(); + }); + + it("an invalid free ratio says why and marks the field", async () => { + renderModal("perspective", null); + await stillLoaded(); + fireEvent.click(screen.getByRole("button", { name: "dialogs.cameraCalibration.formats.free" })); + const input = screen.getByRole("spinbutton"); + expect(input).toHaveAttribute("aria-invalid", "false"); + fireEvent.change(input, { target: { value: "50" } }); + expect(input).toHaveAttribute("aria-invalid", "true"); + expect(screen.getByRole("alert")).toHaveTextContent("dialogs.cameraCalibration.invalidRatio"); + expect(apply()).toBeDisabled(); + }); + + it("a drag ends on pointercancel and its listeners go with the dialog", async () => { + const add = vi.spyOn(window, "addEventListener"); + const remove = vi.spyOn(window, "removeEventListener"); + const { unmount } = render( + , + ); + await stillLoaded(); + const frame = screen.getByTestId("calibration-frame"); + vi.spyOn(frame, "getBoundingClientRect").mockReturnValue(new DOMRect(0, 0, 100, 80)); + const crop = screen.getByTestId("calibration-crop"); + const dragTypes = ["pointermove", "pointerup", "pointercancel"]; + // The drag's own listeners: added after `from`, still registered unless removed since. + const dragListeners = (from: number) => + add.mock.calls.slice(from).filter(([type]) => dragTypes.includes(type)); + const stillAttached = (from: number) => + dragListeners(from).filter( + ([type, fn]) => !remove.mock.calls.some(([t, f]) => t === type && f === fn), + ).length; + + let from = add.mock.calls.length; + fireEvent.pointerDown(crop, { clientX: 10, clientY: 10 }); + expect(dragListeners(from)).toHaveLength(3); + fireEvent(window, new Event("pointercancel")); + expect(stillAttached(from)).toBe(0); + + // A drag still running when the dialog goes away is ended with it. + from = add.mock.calls.length; + fireEvent.pointerDown(crop, { clientX: 10, clientY: 10 }); + expect(stillAttached(from)).toBe(3); + unmount(); + expect(stillAttached(from)).toBe(0); + }); + + it("detect fills the handles", async () => { + const found = [ + { x: 0.2, y: 0.25 }, + { x: 0.8, y: 0.2 }, + { x: 0.85, y: 0.9 }, + { x: 0.1, y: 0.8 }, + ]; + detectCornerMarkers.mockReturnValueOnce(found); + const { onApply } = renderModal("perspective", null); + await stillLoaded(); + fireEvent.click( + screen.getByRole("button", { name: "dialogs.cameraCalibration.detectMarkers" }), + ); + expect(detectCornerMarkers).toHaveBeenCalledWith(IMAGE); + expect(screen.getByRole("status")).toHaveTextContent("dialogs.cameraCalibration.markersFound"); + fireEvent.click(apply()); + expect(onApply.mock.calls[0][0].perspective.corners).toEqual(found); + }); + + it("detect without four markers shows the message and keeps the handles", async () => { + detectCornerMarkers.mockReturnValueOnce(null); + const { onApply } = renderModal("perspective", null); + await stillLoaded(); + fireEvent.click( + screen.getByRole("button", { name: "dialogs.cameraCalibration.detectMarkers" }), + ); + expect(screen.getByRole("status")).toHaveTextContent( + "dialogs.cameraCalibration.markersNotFound", + ); + fireEvent.click(apply()); + expect(onApply.mock.calls[0][0].perspective.corners).toEqual([ + { x: 0.15, y: 0.15 }, + { x: 0.85, y: 0.15 }, + { x: 0.85, y: 0.85 }, + { x: 0.15, y: 0.85 }, + ]); + }); + + it("the marker message goes away once a handle moves", async () => { + detectCornerMarkers.mockReturnValueOnce(null); + renderModal("perspective", null); + await stillLoaded(); + const detect = () => + fireEvent.click( + screen.getByRole("button", { name: "dialogs.cameraCalibration.detectMarkers" }), + ); + detect(); + expect(screen.getByRole("status")).toHaveTextContent( + "dialogs.cameraCalibration.markersNotFound", + ); + fireEvent.keyDown(screen.getByTestId("calibration-handle-1"), { key: "ArrowLeft" }); + expect(screen.getByRole("status")).toBeEmptyDOMElement(); + + // A drag that starts clears it too, before the pointer even moves. + detectCornerMarkers.mockReturnValueOnce(null); + detect(); + expect(screen.getByRole("status")).toHaveTextContent( + "dialogs.cameraCalibration.markersNotFound", + ); + const frame = screen.getByTestId("calibration-frame"); + vi.spyOn(frame, "getBoundingClientRect").mockReturnValue(new DOMRect(0, 0, 100, 80)); + // Corner 0 sits at 15 % / 15 % of the frame. + fireEvent.pointerDown(frame, { clientX: 15, clientY: 12 }); + expect(screen.getByRole("status")).toBeEmptyDOMElement(); + fireEvent(window, new Event("pointerup")); + }); + + it("print hands the translated texts to the sheet", async () => { + renderModal("perspective", null); + await stillLoaded(); + fireEvent.click( + screen.getByRole("button", { name: "dialogs.cameraCalibration.printMarkerSheet" }), + ); + expect(printMarkerSheet).toHaveBeenCalledWith({ + instruction: "dialogs.cameraCalibration.markerSheetInstruction", + labels: [ + "0 – dialogs.cameraCalibration.corners.topLeft", + "1 – dialogs.cameraCalibration.corners.topRight", + "2 – dialogs.cameraCalibration.corners.bottomRight", + "3 – dialogs.cameraCalibration.corners.bottomLeft", + ], + }); + }); + + it("reset removes the stored perspective", async () => { + const { onApply } = renderModal("perspective", { + perspective: { + corners: [ + { x: 0.1, y: 0.1 }, + { x: 0.9, y: 0.1 }, + { x: 0.9, y: 0.9 }, + { x: 0.1, y: 0.9 }, + ], + aspect: 2, + }, + }); + await stillLoaded(); + fireEvent.click(screen.getByRole("button", { name: "dialogs.cameraCalibration.reset" })); + expect(onApply).toHaveBeenCalledWith({ perspective: undefined }); + }); + + it("perspective mode says a stored crop is ignored", async () => { + renderModal("perspective", { crop: { x: 0.1, y: 0.1, width: 0.5, height: 0.5 } }); + await stillLoaded(); + expect(screen.getByText("dialogs.cameraCalibration.cropIgnored")).toBeInTheDocument(); + }); + + it("says nothing about the crop when there is none", async () => { + renderModal("perspective", null); + await stillLoaded(); + expect(screen.queryByText("dialogs.cameraCalibration.cropIgnored")).toBeNull(); + }); + + it("crop mode does not show the perspective note", async () => { + renderModal("crop", { crop: { x: 0.1, y: 0.1, width: 0.5, height: 0.5 } }); + await stillLoaded(); + expect(screen.queryByText("dialogs.cameraCalibration.cropIgnored")).toBeNull(); + }); + + it("the caller can report camera 1's crop, which lives outside the settings", async () => { + render( + , + ); + await stillLoaded(); + expect(screen.getByText("dialogs.cameraCalibration.cropIgnored")).toBeInTheDocument(); + }); +}); diff --git a/src/components/ai-edition/CameraCalibrationModal.tsx b/src/components/ai-edition/CameraCalibrationModal.tsx new file mode 100644 index 000000000..d0e0438db --- /dev/null +++ b/src/components/ai-edition/CameraCalibrationModal.tsx @@ -0,0 +1,711 @@ +// The camera calibration dialog. Perspective mode: four corner handles on a still of the camera, +// dragged with the pointer or nudged with the arrow keys, a loupe for the corner being placed, +// the target format and margin, and a live preview of the corrected picture. "Detect markers" +// places the corners on the four printed markers (`arucoMarkers.ts`); "Print marker sheet" +// prints them (`markerSheet.ts`). Crop mode: a rectangle with corner and edge handles. Built on +// `ModalShell`, so the editor's shortcuts and undo stay blocked while it is open; Apply hands +// back one settings patch (one undo step). + +import { + type CSSProperties, + type KeyboardEvent as ReactKeyboardEvent, + type PointerEvent as ReactPointerEvent, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import type { + CameraPerspective, + CameraPoint, + CameraSettings, + CropRegion, +} from "@/components/video-editor/types"; +import { useScopedT } from "@/contexts/I18nContext"; +import { + type Corners, + type CropEdges, + clampCrop, + clampPoint, + insetCorners, + isFullCrop, + isValidQuad, + loupeSourceRect, + moveCrop, + nearestHandle, + previewSize, + renderRectified, + resizeCrop, +} from "@/lib/ai-edition/timeline/calibrationGeometry"; +import { grabFrame } from "@/lib/ai-edition/timeline/grabFrame"; +import { detectCornerMarkers } from "@/lib/arucoMarkers"; +import { printMarkerSheet } from "@/lib/markerSheet"; +import type { CalibrationMode } from "./CamerasSection"; +import { previewBoxStyle } from "./cropDraft"; +import { ModalShell } from "./Modals"; +import styles from "./NewEditorShell.module.css"; +import { ChoiceRow } from "./RightPanes"; + +/** The camera being calibrated and the still it is calibrated on. */ +export interface CalibrationCamera { + /** 0 = camera 1. */ + index: number; + label: string; + /** Video URL of the camera's file. */ + src: string; + /** Time of that file shown by the playhead. */ + timeSec: number; +} + +export interface CameraCalibrationModalProps { + open: boolean; + camera: CalibrationCamera; + mode: CalibrationMode; + /** The camera's stored settings; the dialog starts from them. */ + initial: CameraSettings | null; + /** The camera has a crop, which a perspective replaces. Camera 1 keeps its crop outside + * `initial`, so the caller says; defaults to `initial.crop`. */ + hasCrop?: boolean; + /** The change to store (a key set to `undefined` removes it); called at most once. */ + onApply: (patch: Partial) => void; + onClose: () => void; +} + +type FormatId = "a4Portrait" | "a4Landscape" | "wide" | "standard" | "square" | "free"; + +const FORMATS: ReadonlyArray<{ id: FormatId; aspect: number | null }> = [ + { id: "a4Portrait", aspect: 210 / 297 }, + { id: "a4Landscape", aspect: 297 / 210 }, + { id: "wide", aspect: 16 / 9 }, + { id: "standard", aspect: 4 / 3 }, + { id: "square", aspect: 1 }, + { id: "free", aspect: null }, +]; + +const MIN_ASPECT = 0.1; +const MAX_ASPECT = 10; +const MAX_MARGIN_PCT = 20; +/** How far from a corner (screen px) a press still grabs it. */ +const HANDLE_RADIUS_PX = 24; +const LOUPE_SIZE_PX = 120; +const LOUPE_ZOOM = 4; +const PREVIEW_LONG_SIDE_PX = 320; +/** One arrow press in crop mode, as a fraction of the image. */ +const CROP_STEP = 0.01; + +const CORNER_KEYS = ["topLeft", "topRight", "bottomRight", "bottomLeft"] as const; +const CROP_CORNERS = ["nw", "ne", "sw", "se"] as const; +const CROP_EDGES = ["n", "s", "w", "e"] as const; +const FULL_CROP: CropRegion = { x: 0, y: 0, width: 1, height: 1 }; + +function formatOf(aspect: number): FormatId { + const match = FORMATS.find((f) => f.aspect !== null && Math.abs(f.aspect - aspect) < 1e-3); + return match?.id ?? "free"; +} + +function parseAspect(text: string): number | null { + const value = Number(text.replace(",", ".")); + return Number.isFinite(value) && value >= MIN_ASPECT && value <= MAX_ASPECT ? value : null; +} + +function copyCorners(c: Corners): Corners { + return [{ ...c[0] }, { ...c[1] }, { ...c[2] }, { ...c[3] }]; +} + +/** + * A pointer drag on `window` until release or cancel; `onMove` gets the offset from the press. + * Returns a disposer that removes the listeners early (the dialog closing mid-drag). + */ +function trackDrag(e: ReactPointerEvent, onMove: (dxPx: number, dyPx: number) => void) { + const startX = e.clientX; + const startY = e.clientY; + const move = (ev: PointerEvent) => onMove(ev.clientX - startX, ev.clientY - startY); + const stop = () => { + window.removeEventListener("pointermove", move); + window.removeEventListener("pointerup", stop); + window.removeEventListener("pointercancel", stop); + }; + window.addEventListener("pointermove", move); + window.addEventListener("pointerup", stop); + window.addEventListener("pointercancel", stop); + return stop; +} + +export function CameraCalibrationModal({ + open, + camera, + mode, + initial, + hasCrop = initial?.crop != null, + onApply, + onClose, +}: CameraCalibrationModalProps) { + const t = useScopedT("dialogs"); + const tc = useScopedT("common"); + // The dialog is mounted per opening, so the stored settings seed the draft once. + const stored = initial?.perspective; + const [corners, setCorners] = useState(() => + stored ? copyCorners(stored.corners) : insetCorners(), + ); + const [format, setFormat] = useState(() => + stored ? formatOf(stored.aspect) : "a4Landscape", + ); + const [freeAspect, setFreeAspect] = useState(() => + stored ? String(Math.round(stored.aspect * 1000) / 1000) : "1.5", + ); + const [marginPct, setMarginPct] = useState(() => Math.round((stored?.margin ?? 0) * 100)); + const [crop, setCrop] = useState(() => initial?.crop ?? FULL_CROP); + const [image, setImage] = useState(null); + const [loadFailed, setLoadFailed] = useState(false); + const [activeHandle, setActiveHandle] = useState(null); + const [markerResult, setMarkerResult] = useState<"found" | "notFound" | null>(null); + + const frameRef = useRef(null); + const stillRef = useRef(null); + const loupeRef = useRef(null); + const previewRef = useRef(null); + const handleRefs = useRef>([]); + // The running drag's disposer: a new drag or unmounting ends the previous one. + const stopDragRef = useRef<(() => void) | null>(null); + + useEffect( + () => () => { + stopDragRef.current?.(); + stopDragRef.current = null; + }, + [], + ); + + // The marker message describes the corners as detected; any manual change makes it stale. + const startDrag = (e: ReactPointerEvent, onMove: (dxPx: number, dyPx: number) => void) => { + setMarkerResult(null); + stopDragRef.current?.(); + stopDragRef.current = trackDrag(e, onMove); + }; + + // The still, at full resolution. + useEffect(() => { + let cancelled = false; + setImage(null); + setLoadFailed(false); + grabFrame(camera.src, camera.timeSec) + .then((data) => { + if (!cancelled) setImage(data); + }) + .catch(() => { + if (!cancelled) setLoadFailed(true); + }); + return () => { + cancelled = true; + }; + }, [camera.src, camera.timeSec]); + + // Paint the still. No 2D context (jsdom) leaves the canvas blank; the handles still work. + useEffect(() => { + const canvas = stillRef.current; + if (!canvas || !image) return; + canvas.width = image.width; + canvas.height = image.height; + canvas.getContext("2d")?.putImageData(image, 0, 0); + }, [image]); + + const aspect = + format === "free" + ? parseAspect(freeAspect) + : (FORMATS.find((f) => f.id === format)?.aspect ?? null); + const margin = marginPct / 100; + const quadValid = isValidQuad(corners); + // Memoized: the preview resamples whenever this object changes, not on every render. + const perspective = useMemo( + () => + quadValid && aspect !== null ? { corners, aspect, ...(margin > 0 ? { margin } : {}) } : null, + [corners, aspect, margin, quadValid], + ); + const preview = previewSize(aspect ?? 1, PREVIEW_LONG_SIDE_PX); + + // The corrected picture, resampled on every change of the corners, format or margin. + useEffect(() => { + if (mode !== "perspective") return; + const canvas = previewRef.current; + const ctx = canvas?.getContext("2d"); + if (!canvas || !ctx) return; + ctx.clearRect(0, 0, canvas.width, canvas.height); + if (!image || !perspective) return; + const pixels = renderRectified(image, perspective, canvas.width, canvas.height); + if (pixels) ctx.putImageData(new ImageData(pixels, canvas.width, canvas.height), 0, 0); + }, [mode, image, perspective]); + + // The loupe: the area under the active corner, magnified, crisp pixels and a crosshair. + useEffect(() => { + const canvas = loupeRef.current; + const source = stillRef.current; + const ctx = canvas?.getContext("2d"); + if (!canvas || !source || !ctx || !image || activeHandle === null) return; + const c = corners[activeHandle]; + const center = { x: c.x * image.width, y: c.y * image.height }; + const r = loupeSourceRect(center, image.width, image.height, LOUPE_SIZE_PX, LOUPE_ZOOM); + ctx.imageSmoothingEnabled = false; + ctx.clearRect(0, 0, LOUPE_SIZE_PX, LOUPE_SIZE_PX); + ctx.drawImage(source, r.x, r.y, r.width, r.height, 0, 0, LOUPE_SIZE_PX, LOUPE_SIZE_PX); + const cx = ((center.x - r.x) / r.width) * LOUPE_SIZE_PX; + const cy = ((center.y - r.y) / r.height) * LOUPE_SIZE_PX; + ctx.strokeStyle = "rgb(255 80 80)"; + ctx.lineWidth = 1; + ctx.beginPath(); + ctx.moveTo(cx, 0); + ctx.lineTo(cx, LOUPE_SIZE_PX); + ctx.moveTo(0, cy); + ctx.lineTo(LOUPE_SIZE_PX, cy); + ctx.stroke(); + }, [image, corners, activeHandle]); + + const setCorner = (index: number, p: CameraPoint) => { + setMarkerResult(null); + setCorners((prev) => { + const next = copyCorners(prev); + next[index] = clampPoint(p); + return next; + }); + }; + + // A press anywhere near a corner grabs it, so a corner is found without aiming at its dot. + const onFramePointerDown = (e: ReactPointerEvent) => { + const frame = frameRef.current; + if (!frame) return; + const r = frame.getBoundingClientRect(); + if (r.width <= 0 || r.height <= 0) return; + const points = corners.map((c) => ({ x: c.x * r.width, y: c.y * r.height })); + const hit = nearestHandle( + points, + { x: e.clientX - r.left, y: e.clientY - r.top }, + HANDLE_RADIUS_PX, + ); + if (hit === null) return; + e.preventDefault(); + handleRefs.current[hit]?.focus(); + setActiveHandle(hit); + const start = corners[hit]; + startDrag(e, (dx, dy) => + setCorner(hit, { x: start.x + dx / r.width, y: start.y + dy / r.height }), + ); + }; + + // The arrows move the focused corner by one image pixel, ten with Shift. + const onHandleKeyDown = (index: number) => (e: ReactKeyboardEvent) => { + const dx = e.key === "ArrowLeft" ? -1 : e.key === "ArrowRight" ? 1 : 0; + const dy = e.key === "ArrowUp" ? -1 : e.key === "ArrowDown" ? 1 : 0; + if (dx === 0 && dy === 0) return; + // The editor shell seeks on the arrows, from window: keep them here. + e.preventDefault(); + e.nativeEvent.stopPropagation(); + const step = e.shiftKey ? 10 : 1; + const c = corners[index]; + setCorner(index, { + x: c.x + (dx * step) / (image?.width ?? 1000), + y: c.y + (dy * step) / (image?.height ?? 1000), + }); + }; + + const startCropMove = (e: ReactPointerEvent) => { + const frame = frameRef.current; + if (!frame) return; + e.preventDefault(); + e.stopPropagation(); + const r = frame.getBoundingClientRect(); + if (r.width <= 0 || r.height <= 0) return; + const start = crop; + startDrag(e, (dx, dy) => setCrop(moveCrop(start, dx / r.width, dy / r.height))); + }; + + const startCropResize = (edges: CropEdges) => (e: ReactPointerEvent) => { + const frame = frameRef.current; + if (!frame) return; + e.preventDefault(); + e.stopPropagation(); + const r = frame.getBoundingClientRect(); + if (r.width <= 0 || r.height <= 0) return; + const start = crop; + startDrag(e, (dx, dy) => setCrop(resizeCrop(start, edges, dx / r.width, dy / r.height))); + }; + + // The arrows move the crop; Shift + the arrows resize it from its bottom-right corner. + const onCropKeyDown = (e: ReactKeyboardEvent) => { + const dx = e.key === "ArrowLeft" ? -1 : e.key === "ArrowRight" ? 1 : 0; + const dy = e.key === "ArrowUp" ? -1 : e.key === "ArrowDown" ? 1 : 0; + if (dx === 0 && dy === 0) return; + e.preventDefault(); + e.nativeEvent.stopPropagation(); + if (e.shiftKey) { + setCrop( + resizeCrop(crop, { right: dx !== 0, bottom: dy !== 0 }, dx * CROP_STEP, dy * CROP_STEP), + ); + return; + } + setCrop(moveCrop(crop, dx * CROP_STEP, dy * CROP_STEP)); + }; + + // The four printed markers place the corners; without all four the corners stay put. + const detectMarkers = () => { + if (!image) return; + const found = detectCornerMarkers(image); + if (found) setCorners(copyCorners(found)); + setMarkerResult(found ? "found" : "notFound"); + }; + + const printSheet = () => { + // Marker ID n belongs on corner n, the handles' order. + const label = (id: number) => `${id} – ${t(`cameraCalibration.corners.${CORNER_KEYS[id]}`)}`; + printMarkerSheet({ + instruction: t("cameraCalibration.markerSheetInstruction"), + labels: [label(0), label(1), label(2), label(3)], + }); + }; + + const isPerspective = mode === "perspective"; + const canApply = isPerspective ? perspective !== null : true; + const hasStored = isPerspective ? Boolean(initial?.perspective) : Boolean(initial?.crop); + + const apply = () => { + if (isPerspective) { + if (!perspective) return; + onApply({ perspective: { ...perspective, corners: copyCorners(perspective.corners) } }); + } else { + const next = clampCrop(crop); + onApply({ crop: isFullCrop(next) ? undefined : next }); + } + onClose(); + }; + + // Reset removes the stored correction (or crop) altogether. + const reset = () => { + setMarkerResult(null); + onApply(isPerspective ? { perspective: undefined } : { crop: undefined }); + onClose(); + }; + + const imageAspect = image ? image.width / image.height : 16 / 9; + const title = isPerspective + ? t("cameraCalibration.titlePerspective", { camera: camera.label }) + : t("cameraCalibration.titleCrop", { camera: camera.label }); + // The loupe sits in the corner away from the corner being placed. + const active = activeHandle === null ? null : corners[activeHandle]; + const loupeOnRight = active !== null && active.x < 0.5 && active.y < 0.5; + + return ( + +

+ {isPerspective ? t("cameraCalibration.perspectiveHelp") : t("cameraCalibration.cropHelp")} +

+ {isPerspective && hasCrop ? ( +

+ {t("cameraCalibration.cropIgnored")} +

+ ) : null} +
+ + {image === null ? ( +

+ {loadFailed ? t("cameraCalibration.loadFailed") : t("cameraCalibration.loading")} +

+ ) : null} + {isPerspective ? ( + <> + + `${c.x},${c.y}`).join(" ")} + fill={quadValid ? "rgb(80 160 255 / 0.15)" : "rgb(255 80 80 / 0.2)"} + stroke={quadValid ? "rgb(255 255 255 / 0.9)" : "rgb(255 80 80)"} + strokeWidth={1.5} + vectorEffect="non-scaling-stroke" + /> + + {corners.map((c, i) => ( +
+ + {isPerspective ? ( +
+ + +

+ {markerResult === "found" + ? t("cameraCalibration.markersFound") + : markerResult === "notFound" + ? t("cameraCalibration.markersNotFound") + : null} +

+
+ ) : null} + + {isPerspective ? ( +
+
+ {t("cameraCalibration.format")} + + label={t("cameraCalibration.format")} + columns={3} + options={[ + { value: "a4Portrait", label: t("cameraCalibration.formats.a4Portrait") }, + { value: "a4Landscape", label: t("cameraCalibration.formats.a4Landscape") }, + { value: "wide", label: "16:9" }, + { value: "standard", label: "4:3" }, + { value: "square", label: "1:1" }, + { value: "free", label: t("cameraCalibration.formats.free") }, + ]} + value={format} + onChange={setFormat} + /> + {format === "free" ? ( + + ) : null} + {format === "free" && aspect === null ? ( + + ) : null} + +
+
+ {t("cameraCalibration.preview")} + +
+
+ ) : null} + + {isPerspective && !quadValid ? ( + + ) : null} + +
+ +
+ + +
+
+ + ); +} diff --git a/src/components/ai-edition/CamerasSection.test.tsx b/src/components/ai-edition/CamerasSection.test.tsx new file mode 100644 index 000000000..a382aaa5f --- /dev/null +++ b/src/components/ai-edition/CamerasSection.test.tsx @@ -0,0 +1,185 @@ +// @vitest-environment jsdom +import "@testing-library/jest-dom"; +import { fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import type { AxcutDocument } from "@/lib/ai-edition/schema"; + +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: (scope: string) => (key: string, vars?: Record) => + vars?.n !== undefined + ? `${scope}.${key}#${vars.n}${vars.label ? `|${vars.label}` : ""}` + : `${scope}.${key}`, +})); +vi.mock("@/lib/ai-edition/timeline/grabFrame", () => ({ + grabFrameDataUrl: vi.fn(() => Promise.resolve("data:image/png;base64,AAAA")), +})); + +import { readFileSync } from "node:fs"; +import { act } from "@testing-library/react"; +import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { grabFrameDataUrl } from "@/lib/ai-edition/timeline/grabFrame"; +import { CamerasSection, calibrationCameraAt, cameraHasCrop } from "./CamerasSection"; + +const track = (sourcePath: string, extra: Record = {}) => ({ + sourcePath, + visible: true, + startMs: 0, + offsetMs: 0, + ...extra, +}); + +function makeDoc(extraCameras: number): AxcutDocument { + return { + assets: [ + { + id: "a1", + cameraTrack: track("C:/cam1.mp4"), + additionalCameraTracks: Array.from({ length: extraCameras }, (_, i) => + track(`C:/cam${i + 2}.mp4`, { label: i === 0 ? "Desk" : "" }), + ), + }, + ], + timeline: { + clips: [ + { + id: "c1", + assetId: "a1", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + }, + ], + }, + } as unknown as AxcutDocument; +} + +const render2 = (extra: number, setCameraSettings = vi.fn(), onOpenCalibration = vi.fn()) => { + render( + , + ); + return { setCameraSettings, onOpenCalibration }; +}; + +describe("CamerasSection", () => { + it("lists every camera of the clip", () => { + render2(2); + expect(screen.getByTestId("camera-row-0")).toHaveTextContent("settings.cameras.cameraN#1"); + expect(screen.getByTestId("camera-row-1")).toHaveTextContent( + "settings.cameras.cameraNamed#2|Desk", + ); + expect(screen.getByTestId("camera-row-2")).toHaveTextContent("settings.cameras.cameraN#3"); + }); + + it("camera 1 offers only the perspective", () => { + const { onOpenCalibration } = render2(1); + const row = screen.getByTestId("camera-row-0"); + expect(row).toHaveTextContent("settings.cameras.camera1Hint"); + expect(row.querySelectorAll("button")).toHaveLength(1); + fireEvent.click(row.querySelector("button") as Element); + expect(onOpenCalibration).toHaveBeenCalledWith(0, "perspective"); + }); + + it("rotating camera 2 calls setCameraSettings", () => { + const { setCameraSettings, onOpenCalibration } = render2(1); + const row = screen.getByTestId("camera-row-1"); + fireEvent.click( + row.querySelector("button[aria-pressed]:not([aria-pressed='true'])") as Element, + ); + expect(setCameraSettings).toHaveBeenCalledWith(1, { rotation: 180 }); + const buttons = Array.from(row.querySelectorAll("button")); + fireEvent.click(buttons.find((b) => b.textContent === "settings.cameras.crop") as Element); + expect(onOpenCalibration).toHaveBeenCalledWith(1, "crop"); + }); + + it("follows the playhead from the store without a prop", async () => { + vi.useFakeTimers(); + useProjectStore.setState({ currentTimeSec: 1 }); + render( + , + ); + act(() => { + useProjectStore.setState({ currentTimeSec: 4 }); + }); + act(() => { + vi.advanceTimersByTime(300); + }); + expect(vi.mocked(grabFrameDataUrl).mock.calls.at(-1)?.[1]).toBe(4); + vi.useRealTimers(); + }); + + it("keeps the playhead subscription out of LayoutPane", () => { + const source = readFileSync("src/components/ai-edition/RightPanes.tsx", "utf8"); + const start = source.indexOf("export function LayoutPane("); + const end = source.indexOf("/** The tightest the frame gets"); + expect(source.slice(start, end)).not.toContain("currentTimeSec"); + }); + + it("finds the calibration still of a camera at the playhead", () => { + const t = (key: string, vars?: Record) => + vars?.n !== undefined ? `${key}#${vars.n}${vars.label ? `|${vars.label}` : ""}` : key; + const doc = makeDoc(1); + const desk = calibrationCameraAt(doc, 2, 1, t); + expect(desk).toMatchObject({ index: 1, label: "cameras.cameraNamed#2|Desk", timeSec: 2 }); + expect(desk?.src).toMatch(/^file:\/\/.*cam2\.mp4$/); + expect(calibrationCameraAt(doc, 2, 0, t)?.label).toBe("cameras.cameraN#1"); + // No such camera, or no document: nothing to calibrate. + expect(calibrationCameraAt(doc, 2, 3, t)).toBeNull(); + expect(calibrationCameraAt(null, 2, 0, t)).toBeNull(); + }); + + it("a perspective disables the crop with a hint", () => { + const onOpenCalibration = vi.fn(); + render( + , + ); + const row = screen.getByTestId("camera-row-1"); + const crop = screen.getByRole("button", { name: "settings.cameras.crop" }); + expect(crop).toBeDisabled(); + expect(row).toHaveTextContent("settings.cameras.cropOffWithPerspective"); + fireEvent.click(crop); + expect(onOpenCalibration).not.toHaveBeenCalled(); + }); + + it("without a perspective the crop stays available and no hint shows", () => { + render2(1); + expect(screen.getByRole("button", { name: "settings.cameras.crop" })).toBeEnabled(); + expect(screen.queryByText("settings.cameras.cropOffWithPerspective")).toBeNull(); + }); + + it("knows whether a camera has a crop", () => { + const crop = { x: 0.1, y: 0.1, width: 0.5, height: 0.5 }; + expect(cameraHasCrop(null, [null, { crop }], 1)).toBe(true); + expect(cameraHasCrop(null, [null, { mirror: true }], 1)).toBe(false); + const withCamera1Crop = (webcamCropRegion: unknown) => + ({ legacyEditor: { webcamCropRegion } }) as unknown as AxcutDocument; + expect(cameraHasCrop(withCamera1Crop(crop), [], 0)).toBe(true); + expect(cameraHasCrop(withCamera1Crop({ x: 0, y: 0, width: 1, height: 1 }), [], 0)).toBe(false); + expect(cameraHasCrop(makeDoc(0), [], 0)).toBe(false); + }); +}); diff --git a/src/components/ai-edition/CamerasSection.tsx b/src/components/ai-edition/CamerasSection.tsx new file mode 100644 index 000000000..92585ae43 --- /dev/null +++ b/src/components/ai-edition/CamerasSection.tsx @@ -0,0 +1,245 @@ +// The "Cameras" section of the layout pane: every camera of the clip under the playhead, with +// a thumbnail and the per-camera settings. Camera 1 keeps its rotation, mirror and crop in +// the controls above (older fields), so its row only offers the perspective correction. + +import { useEffect, useMemo, useState } from "react"; +import { toFileUrl } from "@/components/video-editor/projectPersistence"; +import type { CameraSettings, CropRegion } from "@/components/video-editor/types"; +import { useScopedT } from "@/contexts/I18nContext"; +import type { AxcutDocument } from "@/lib/ai-edition/schema"; +import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { assetAdditionalCameraSources, assetCameraSource } from "@/lib/ai-edition/timeline/camera"; +import { camerasForClipAt, type ProjectCamera } from "@/lib/ai-edition/timeline/cameraList"; +import { grabFrameDataUrl } from "@/lib/ai-edition/timeline/grabFrame"; +import { locateVirtualPosition } from "@/lib/ai-edition/timeline/virtual-preview"; +import type { CameraRotation } from "@/lib/cameraOrientation"; +import styles from "./NewEditorShell.module.css"; +import { ChoiceRow, Toggle } from "./RightPanes"; + +/** What the calibration dialog edits: the picture's crop, or its perspective. */ +export type CalibrationMode = "crop" | "perspective"; + +const THUMBNAIL_SIDE = 96; +/** The playhead moves every frame during playback; a still is only worth grabbing once it rests. */ +const THUMBNAIL_DEBOUNCE_MS = 250; + +export interface CamerasSectionProps { + document: AxcutDocument | null; + /** Overrides the playhead on the ruler; by default it is read from the project store here, so + * only this section re-renders while the playhead moves. */ + playheadSec?: number; + /** `legacyEditor.cameraSettings`, normalized: index 0 = camera 1. */ + cameraSettings: (CameraSettings | null)[]; + setCameraSettings: (index: number, patch: Partial | null) => void | Promise; + /** Opens the calibration dialog for a camera. */ + onOpenCalibration?: (cameraIndex: number, mode: CalibrationMode) => void; +} + +export type CameraStill = { src: string; timeSec: number }; + +/** Where each camera's file is, and which time of it the playhead shows. */ +export function stillsAt( + document: AxcutDocument | null, + playheadSec: number, +): Map { + const stills = new Map(); + if (!document) return stills; + const position = locateVirtualPosition(document.timeline.clips, playheadSec); + if (!position) return stills; + const asset = document.assets.find((a) => a.id === position.clip.assetId); + const sources = [assetCameraSource(asset), ...assetAdditionalCameraSources(asset)]; + sources.forEach((source, index) => { + if (!source.path) return; + const src = /^(https?|blob|data):/.test(source.path) ? source.path : toFileUrl(source.path); + stills.set(index, { src, timeSec: Math.max(0, position.sourceTimeSec - source.offsetSec) }); + }); + return stills; +} + +/** + * Whether a camera has a crop stored. Camera 1 keeps its crop in `webcamCropRegion` (a + * full-frame rect means none); the others in `cameraSettings[k].crop`. + */ +export function cameraHasCrop( + document: AxcutDocument | null, + cameraSettings: (CameraSettings | null)[], + index: number, +): boolean { + if (index !== 0) return cameraSettings[index]?.crop != null; + const legacy = document?.legacyEditor as Record | null | undefined; + const crop = legacy?.webcamCropRegion as Partial | undefined; + if (!crop) return false; + const coversFrame = (v: unknown) => typeof v !== "number" || v >= 1 - 1e-6; + return !(coversFrame(crop.width) && coversFrame(crop.height)); +} + +/** The camera the calibration dialog opens on: its label and its still at the playhead. */ +export function calibrationCameraAt( + document: AxcutDocument | null, + playheadSec: number, + index: number, + t: (key: string, vars?: Record) => string, +): { index: number; label: string; src: string; timeSec: number } | null { + if (!document) return null; + const camera = camerasForClipAt(document, playheadSec, t).find((c) => c.index === index); + const still = stillsAt(document, playheadSec).get(index); + if (!camera?.available || !still) return null; + return { index, label: camera.label, ...still }; +} + +function CameraThumbnail({ still, label }: { still: CameraStill | undefined; label: string }) { + const [url, setUrl] = useState(null); + const src = still?.src; + const timeSec = still?.timeSec; + useEffect(() => { + if (src === undefined || timeSec === undefined) { + setUrl(null); + return; + } + let cancelled = false; + const timer = setTimeout(() => { + grabFrameDataUrl(src, timeSec, THUMBNAIL_SIDE) + .then((dataUrl) => { + if (!cancelled) setUrl(dataUrl); + }) + .catch(() => { + if (!cancelled) setUrl(null); + }); + }, THUMBNAIL_DEBOUNCE_MS); + return () => { + cancelled = true; + clearTimeout(timer); + }; + }, [src, timeSec]); + return ( +
+ {url ? ( + {label} + ) : null} +
+ ); +} + +const BUTTON = `${styles.btn} ${styles.btnSecondary}`; + +export function CamerasSection({ + document, + playheadSec: playheadOverride, + cameraSettings, + setCameraSettings, + onOpenCalibration, +}: CamerasSectionProps) { + const ts = useScopedT("settings"); + const storePlayheadSec = useProjectStore((s) => s.currentTimeSec); + const playheadSec = playheadOverride ?? storePlayheadSec; + const cameras: ProjectCamera[] = useMemo( + () => (document ? camerasForClipAt(document, playheadSec, ts) : []), + [document, playheadSec, ts], + ); + const stills = useMemo(() => stillsAt(document, playheadSec), [document, playheadSec]); + if (cameras.length === 0) return null; + return ( + <> +
{ts("cameras.title")}
+ {cameras.map((camera) => { + const settings = cameraSettings[camera.index] ?? null; + const isFirst = camera.index === 0; + // A perspective replaces the crop at render, so the crop is not offered then. + const hasPerspective = settings?.perspective != null; + return ( +
+
+ +
+
{camera.label}
+ {camera.available ? null : ( +

{ts("cameras.unavailable")}

+ )} +
+
+ {isFirst ?

{ts("cameras.camera1Hint")}

: null} + {isFirst ? null : ( + <> + + label={ts("cameras.rotation")} + options={[ + { value: 0, label: "0°" }, + { value: 180, label: "180°" }, + ]} + value={settings?.rotation ?? 0} + onChange={(rotation) => void setCameraSettings(camera.index, { rotation })} + /> +
+ {ts("cameras.mirror")} + void setCameraSettings(camera.index, { mirror })} + /> +
+ + )} +
+ {isFirst ? null : ( + + )} + + {!isFirst && settings ? ( + + ) : null} +
+ {!isFirst && hasPerspective ? ( +

{ts("cameras.cropOffWithPerspective")}

+ ) : null} +
+ ); + })} + + ); +} diff --git a/src/components/ai-edition/CaptionsPane.tsx b/src/components/ai-edition/CaptionsPane.tsx index 258fa2dc9..772a4ca43 100644 --- a/src/components/ai-edition/CaptionsPane.tsx +++ b/src/components/ai-edition/CaptionsPane.tsx @@ -500,6 +500,8 @@ export function CaptionsPane({ onClose }: { onClose?: () => void } = {}) { value: font, label: font, icon: {font}, + // The specimen already spells the name. + title: null, }))} value={settings.fontFamily} disabled={disabled} diff --git a/src/components/ai-edition/ChoiceRow.test.tsx b/src/components/ai-edition/ChoiceRow.test.tsx index f5aa7e969..bce82d900 100644 --- a/src/components/ai-edition/ChoiceRow.test.tsx +++ b/src/components/ai-edition/ChoiceRow.test.tsx @@ -43,3 +43,86 @@ describe("ChoiceRow keyboard", () => { expect(screen.getByRole("button", { name: "one" })).toHaveFocus(); }); }); + +// A tooltip that says what the button already says is noise (tooltips.md, rule 3). A button with +// no visible label is the one that needs it: it is named for a screen reader and for the mouse. +describe("ChoiceRow tooltips", () => { + const icon = ; + + it("draws no tooltip on an option whose label is visible", () => { + render( + + label="row" + value="a" + onChange={vi.fn()} + options={[ + { value: "a", label: "Alpha" }, + { value: "b", label: "Beta" }, + ]} + />, + ); + for (const name of ["Alpha", "Beta"]) { + expect(screen.getByRole("button", { name })).not.toHaveAttribute("title"); + } + }); + + it("names an icon-only option in its tooltip", () => { + render( + + label="row" + value="a" + onChange={vi.fn()} + options={[ + { value: "a", label: "Alpha", icon }, + { value: "b", label: "Beta", icon }, + ]} + />, + ); + expect(screen.getByRole("button", { name: "Alpha" })).toHaveAttribute("title", "Alpha"); + expect(screen.getByRole("button", { name: "Beta" })).toHaveAttribute("title", "Beta"); + }); + + it("draws no tooltip on an option that shows its icon and its label", () => { + render( + + label="row" + display="both" + value="a" + onChange={vi.fn()} + options={[{ value: "a", label: "Alpha", icon }]} + />, + ); + expect(screen.getByRole("button", { name: "Alpha" })).not.toHaveAttribute("title"); + }); + + it("draws none on an icon that already spells the label, when the caller says so", () => { + render( + + label="row" + value="a" + onChange={vi.fn()} + options={[{ value: "a", label: "Lora", icon: Lora, title: null }]} + />, + ); + expect(screen.getByRole("button", { name: "Lora" })).not.toHaveAttribute("title"); + }); + + it("keeps the tooltip a caller passes, even beside a visible label", () => { + render( + + label="row" + value="a" + onChange={vi.fn()} + options={[ + { value: "a", label: "Alpha", title: "Alpha · 2 clips" }, + { value: "b", label: "Beta" }, + ]} + />, + ); + expect(screen.getByRole("button", { name: "Alpha" })).toHaveAttribute( + "title", + "Alpha · 2 clips", + ); + expect(screen.getByRole("button", { name: "Beta" })).not.toHaveAttribute("title"); + }); +}); diff --git a/src/components/ai-edition/ExportDialog.cancel.test.tsx b/src/components/ai-edition/ExportDialog.cancel.test.tsx index fe52beb90..8d2403b9c 100644 --- a/src/components/ai-edition/ExportDialog.cancel.test.tsx +++ b/src/components/ai-edition/ExportDialog.cancel.test.tsx @@ -98,7 +98,7 @@ async function start() { , ); - fireEvent.click(screen.getByRole("button", { name: /README GIF/ })); + fireEvent.click(screen.getByRole("button", { name: "GIF" })); fireEvent.click(screen.getByRole("button", { name: "Export GIF" })); await waitFor(() => expect(exportGifNative).toHaveBeenCalledOnce()); const id = vi.mocked(exportGifNative).mock.calls[0][4]; @@ -240,7 +240,7 @@ describe("GIF export cancellation", () => { , ); - fireEvent.click(screen.getByRole("button", { name: /README GIF/ })); + fireEvent.click(screen.getByRole("button", { name: "GIF" })); fireEvent.click(screen.getByRole("button", { name: "Export GIF" })); view.unmount(); await act(async () => choose({ success: true, path: "/tmp/late.gif" })); diff --git a/src/components/ai-edition/ExportDialog.params.test.tsx b/src/components/ai-edition/ExportDialog.params.test.tsx index 14760de02..76cc992c2 100644 --- a/src/components/ai-edition/ExportDialog.params.test.tsx +++ b/src/components/ai-edition/ExportDialog.params.test.tsx @@ -137,7 +137,6 @@ describe("ExportDialog MP4 params", () => { const at60 = await exportMp4(); expect(at60).toMatchObject({ width: 1920, height: 1080, fps: 60, bitrate: expected(60) }); - fireEvent.click(screen.getByRole("button", { name: "Advanced" })); fireEvent.click(screen.getByRole("button", { name: "30" })); const at30 = await exportMp4(); expect(at30).toMatchObject({ fps: 30, bitrate: expected(30) }); @@ -145,7 +144,7 @@ describe("ExportDialog MP4 params", () => { }); }); -describe("ExportDialog destinations", () => { +describe("ExportDialog format settings", () => { beforeEach(() => { window.electronAPI = { pickExportSavePath: vi.fn(async () => ({ path: "/tmp/out.mp4" })), @@ -158,47 +157,42 @@ describe("ExportDialog destinations", () => { vi.clearAllMocks(); }); - const destination = (name: RegExp) => screen.getByRole("button", { name }); - - it("opens on Web / YouTube, with the detailed settings folded under Advanced", () => { + it("opens on MP4, 1080p, 60 fps, H.264, with no codec choice to make", async () => { renderDialog(); - expect(destination(/Web \/ YouTube/)).toHaveAttribute("aria-pressed", "true"); - expect(destination(/Web \/ YouTube/)).toHaveTextContent("MP4 · 1920 × 1080 · 60 fps"); - expect(screen.getByRole("button", { name: "Advanced" })).toHaveAttribute( - "aria-expanded", - "false", - ); + // No picker, on purpose: H.265 is software-only on Linux, slower than software on + // the measured Macs, and unreadable in half the players. Every export is H.264. expect(screen.queryByRole("button", { name: "H.265" })).toBeNull(); + expect(screen.queryByRole("button", { name: "H.264" })).toBeNull(); + // And no idle hint plate: the format is the first control on screen and already picked. + expect(screen.queryByText(/pick a format/i)).toBeNull(); + expect(await exportMp4()).toMatchObject({ width: 1920, height: 1080, fps: 60, codec: "h264" }); }); - it("Social exports the same frame at half the frame rate, so half the bitrate", async () => { + it("half the frame rate is half the bitrate, same frame", async () => { renderDialog(); const web = await exportMp4(); - fireEvent.click(destination(/Social/)); + fireEvent.click(screen.getByRole("button", { name: "30" })); const social = await exportMp4(); - // Same size: a destination never touches the project's format. expect(social).toMatchObject({ width: web?.width, height: web?.height, fps: 30 }); expect(social?.bitrate).toBe((web?.bitrate ?? 0) / 2); }); - it("stops showing a destination as picked once an Advanced setting leaves it", () => { + it("gives the GIF loop toggle an accessible name", () => { + // The switch renders no text of its own; without a name a screen reader only + // announces "pressed" and never says which setting it is. renderDialog(); - fireEvent.click(destination(/Social/)); - expect(destination(/Social/)).toHaveAttribute("aria-pressed", "true"); - fireEvent.click(screen.getByRole("button", { name: "Advanced" })); - fireEvent.click(screen.getByRole("button", { name: "H.265" })); - for (const d of [/Web \/ YouTube/, /Social/, /Studio/, /README GIF/]) { - expect(destination(d)).toHaveAttribute("aria-pressed", "false"); - } + fireEvent.click(screen.getByRole("button", { name: "GIF" })); + expect(screen.getByRole("button", { name: "Loop GIF" })).toHaveAttribute( + "aria-pressed", + "true", + ); }); - it("README GIF asks for a README-sized GIF", async () => { + it("a GIF is 15 fps and README-sized at the small preset", async () => { renderDialog(); - fireEvent.click(destination(/README GIF/)); - expect(destination(/README GIF/)).toHaveTextContent("GIF · 852 × 480 · 15 fps"); - fireEvent.click(screen.getByRole("button", { name: /export gif/i })); - await waitFor(() => expect(exportGifNative).toHaveBeenCalled()); - expect(vi.mocked(exportGifNative).mock.calls[0][3]).toMatchObject({ + fireEvent.click(screen.getByRole("button", { name: "GIF" })); + fireEvent.click(screen.getByRole("button", { name: "Small (480p)" })); + expect(await exportGif()).toMatchObject({ width: 852, height: 480, fps: 15, @@ -206,24 +200,23 @@ describe("ExportDialog destinations", () => { }); }); - it("sizes README GIF from the source, whatever MP4 tier was picked before", async () => { - // The report on #814: a 640x360 clip got an 852x480 GIF straight away, sized from the hidden - // 1080p tier the dialog opens on, and a 640x360 one after Studio, whose tier is "Source". + it("sizes a GIF from the source, whatever MP4 tier was picked before", async () => { + // The report on #814: a 640x360 clip got an 852x480 GIF straight away, sized from the + // 1080p tier the dialog opens on. It must always come off the source size. renderDialog(SMALL_SOURCE_DOC); - fireEvent.click(destination(/README GIF/)); - expect(destination(/README GIF/)).toHaveTextContent("GIF · 640 × 360 · 15 fps"); + fireEvent.click(screen.getByRole("button", { name: "GIF" })); + fireEvent.click(screen.getByRole("button", { name: "Small (480p)" })); const direct = await exportGif(); expect(direct).toMatchObject({ width: 640, height: 360 }); - fireEvent.click(destination(/Studio/)); - fireEvent.click(destination(/README GIF/)); - expect(destination(/README GIF/)).toHaveTextContent("GIF · 640 × 360 · 15 fps"); + fireEvent.click(screen.getByRole("button", { name: "MP4" })); + fireEvent.click(screen.getByRole("button", { name: /^Source/ })); + fireEvent.click(screen.getByRole("button", { name: "GIF" })); expect(await exportGif()).toEqual(direct); }); it("never exports a GIF bigger than its source, whatever the tier and size preset", async () => { renderDialog(SMALL_SOURCE_DOC); - fireEvent.click(screen.getByRole("button", { name: "Advanced" })); for (const tier of [/^720p/, /^1080p/, /^Source/]) { fireEvent.click(screen.getByRole("button", { name: "MP4" })); fireEvent.click(screen.getByRole("button", { name: tier })); @@ -235,3 +228,47 @@ describe("ExportDialog destinations", () => { } }); }); + +describe("ExportDialog clip list", () => { + beforeEach(() => { + window.electronAPI = { + pickExportSavePath: vi.fn(async () => ({ path: "/tmp/out.mp4" })), + onNativeExportProgress: vi.fn(() => noop), + } as unknown as ElectronAPI; + }); + + afterEach(() => { + cleanup(); + vi.clearAllMocks(); + }); + + it("hands the native export the asset's extra cameras", async () => { + const withCameras: AxcutDocument = { + ...DOC, + assets: [ + { + ...DOC.assets[0], + additionalCameraTracks: [ + { sourcePath: "/tmp/cam2.mp4", startMs: 500, offsetMs: 250, visible: true, label: "" }, + { sourcePath: "/tmp/cam3.mp4", startMs: 0, offsetMs: 0, visible: false, label: "" }, + ], + }, + ], + }; + renderDialog(withCameras); + await exportMp4(); + const clips = vi.mocked(exportMultiNative).mock.calls.at(-1)?.[0]; + expect(clips?.[0]?.additionalCameras).toEqual([ + { path: "/tmp/cam2.mp4", offsetSec: 0.75 }, + // A hidden track keeps its slot, so the indices stay aligned with the tracks. + { path: "", offsetSec: 0 }, + ]); + }); + + it("sends no extra cameras for a one-camera asset", async () => { + renderDialog(); + await exportMp4(); + const clips = vi.mocked(exportMultiNative).mock.calls.at(-1)?.[0]; + expect(clips?.[0]).not.toHaveProperty("additionalCameras"); + }); +}); diff --git a/src/components/ai-edition/ExportDialog.tsx b/src/components/ai-edition/ExportDialog.tsx index a17f427a0..be340bb60 100644 --- a/src/components/ai-edition/ExportDialog.tsx +++ b/src/components/ai-edition/ExportDialog.tsx @@ -6,7 +6,7 @@ // Format/quality/GIF options live in the dialog's local state. The // dialog uses the new shell's modal style. -import { ChevronDown, Download, FileVideo, FolderOpen, Loader2, Star } from "lucide-react"; +import { Download, FileVideo, FolderOpen, Loader2, Star } from "lucide-react"; import { useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; import { useScopedT } from "@/contexts/I18nContext"; @@ -19,13 +19,12 @@ import { } from "@/lib/ai-edition/document/outputFormat"; import type { AxcutDocument } from "@/lib/ai-edition/schema"; import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings"; -import { assetCameraSource } from "@/lib/ai-edition/timeline/camera"; +import { assetAdditionalCameraSources, assetCameraSource } from "@/lib/ai-edition/timeline/camera"; import { resolveClipSourceEndSec } from "@/lib/ai-edition/timeline/clipDuration"; import { type ExportFormat, type ExportProgress, type ExportQuality, - type ExportVideoCodec, GIF_FRAME_RATES, GIF_SIZE_PRESETS, type GifFrameRate, @@ -44,6 +43,7 @@ import type { CompositorClipInput } from "@/native/contracts"; import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription"; import { ModalShell } from "./Modals"; import styles from "./NewEditorShell.module.css"; +import { Toggle } from "./RightPanes"; type Phase = "idle" | "configuring" | "rendering" | "writing" | "done" | "error"; @@ -117,6 +117,8 @@ function buildNativeClipList(document: AxcutDocument): CompositorClipInput[] { return []; } const camera = assetCameraSource(asset); + // Cameras 2-4, only sent when the asset has any (same rule as `buildSceneDescription`). + const additionalCameras = assetAdditionalCameraSources(asset); // sourceEndSec is optional in the schema (unknown until probed) — fall back through // the single canonical precedence used by every consumer (clip.probe → asset.duration // → timeline-length guess). See `resolveClipSourceEndSec` for the full order. @@ -132,6 +134,7 @@ function buildNativeClipList(document: AxcutDocument): CompositorClipInput[] { sourceEndSec, webcamOffsetSec: camera.offsetSec, hasAudio: true, + ...(additionalCameras.length > 0 ? { additionalCameras } : {}), }, ]; }); @@ -146,40 +149,6 @@ const QUALITY_OPTIONS: Array<{ { value: "source", labelKey: "exportQuality.high" }, ]; -interface ExportChoice { - format: ExportFormat; - quality: ExportQuality; - fps: 24 | 30 | 60; - codec: ExportVideoCodec; - gifFrameRate: GifFrameRate; - gifSize: GifSizePreset; - gifLoop: boolean; -} - -/** Named sets of the Advanced settings, nothing more: a destination sets controls the user can - * see and change there, and never the aspect ratio, which belongs to the project. It shows as - * picked while the settings still match it. */ -const DESTINATIONS: Array<{ labelKey: string; set: Partial }> = [ - { labelKey: "destinationWeb", set: { format: "mp4", quality: "good", fps: 60, codec: "h264" } }, - // Half the frames at the same bits per frame: half the bitrate. - { - labelKey: "destinationSocial", - set: { format: "mp4", quality: "good", fps: 30, codec: "h264" }, - }, - { - labelKey: "destinationStudio", - set: { format: "mp4", quality: "source", fps: 60, codec: "h264" }, - }, - { - labelKey: "destinationReadmeGif", - set: { format: "gif", gifFrameRate: 15, gifSize: "small", gifLoop: true }, - }, -]; - -function matchesDestination(choice: ExportChoice, set: Partial) { - return (Object.keys(set) as Array).every((key) => choice[key] === set[key]); -} - interface ExportDialogProps { open: boolean; onClose: () => void; @@ -195,11 +164,9 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { const [format, setFormat] = useState("mp4"); const [quality, setQuality] = useState("good"); const [fps, setFps] = useState<24 | 30 | 60>(60); - const [codec, setCodec] = useState("h264"); const [gifFrameRate, setGifFrameRate] = useState(15); const [gifSize, setGifSize] = useState("medium"); const [gifLoop, setGifLoop] = useState(true); - const [advancedOpen, setAdvancedOpen] = useState(false); const [phase, setPhase] = useState("idle"); const [progress, setProgress] = useState(null); const [error, setError] = useState(null); @@ -294,7 +261,7 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { // GIF is 8-bit indexed and grows fast with area, so the size preset caps the output // height. It starts from the "Source" size, never from the quality tier: that control is // MP4-only and hidden while GIF is picked, yet the tier an MP4 choice left behind used to - // size the GIF (a 640x360 clip gave 852x480 after Web / YouTube, 640x360 after Studio). + // size the GIF (a 640x360 clip gave 852x480 after the 1080p tier, 640x360 after Source). // So no preset upscales and `original` is the source size. The native side falls back // to its own defaults when undefined. const gifOutputDims = (preset: GifSizePreset): { width?: number; height?: number } => { @@ -468,7 +435,10 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { width: outDims?.width, height: outDims?.height, fps, - codec, + // H.264 only. The native pipeline still encodes H.265, but nothing + // offers it: it is software-only on Linux, slower than software on the + // measured Macs, and the files half the players cannot open. + codec: "h264", bitrate: outDims?.bitrate, }); if (activeExport.current !== job) return; @@ -523,27 +493,6 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { const isBusy = phase === "rendering" || phase === "writing" || phase === "configuring"; const pct = progress?.percentage ?? 0; const gifSizeLabel = GIF_SIZE_PRESETS[gifSize].label; - const choice: ExportChoice = { format, quality, fps, codec, gifFrameRate, gifSize, gifLoop }; - const applyDestination = (set: Partial) => { - if (set.format) setFormat(set.format); - if (set.quality) setQuality(set.quality); - if (set.fps) setFps(set.fps); - if (set.codec) setCodec(set.codec); - if (set.gifFrameRate) setGifFrameRate(set.gifFrameRate); - if (set.gifSize) setGifSize(set.gifSize); - if (set.gifLoop !== undefined) setGifLoop(set.gifLoop); - }; - /** What a destination produces, in the numbers the Advanced settings show. */ - const destinationSummary = (set: Partial) => { - if (set.format === "gif") { - const gif = gifOutputDims(set.gifSize ?? gifSize); - const size = gif.width ? `${gif.width} × ${gif.height} · ` : ""; - return `GIF · ${size}${set.gifFrameRate ?? gifFrameRate} fps`; - } - const dims = tierOutputDims(set.quality ?? quality); - const size = dims ? `${dims.width} × ${dims.height} · ` : ""; - return `MP4 · ${size}${set.fps ?? fps} fps`; - }; return (
-
-
{t("exportDialog.destination")}
-
- {DESTINATIONS.map((d) => ( - - ))} -
-
- - - - {advancedOpen ? ( - <> -
- } - onClick={() => setFormat("mp4")} - disabled={isBusy} - /> - } - onClick={() => setFormat("gif")} - disabled={isBusy} - /> -
+ } + onClick={() => setFormat("mp4")} + disabled={isBusy} + /> + } + onClick={() => setFormat("gif")} + disabled={isBusy} + /> +
- {format === "mp4" ? ( -
-
{t("exportDialog.quality")}
-
- {QUALITY_OPTIONS.map((q) => ( - - ))} -
-
+
{t("exportDialog.quality")}
+
+ {QUALITY_OPTIONS.map((q) => ( + - ))} -
-
-
-
{t("exportDialog.codec")}
-
- {( - [ - ["h264", "H.264"], - ["h265", "H.265"], - // VP9 has no AMF hardware encoder on this GPU — the native pipeline - // (the only MP4 export path now) rejects it outright (tested: a - // software libvpx-vp9 fallback worked but was too slow to ship). - // Hidden here rather than left selectable-then-erroring. - ] as Array<[ExportVideoCodec, string]> - ).map(([value, label]) => ( - - ))} -
-
-
- - ) : ( -
-
-
{t("exportDialog.frameRate")}
-
- {GIF_FRAME_RATES.map((r) => ( - - ))} -
-
-
-
{t("exportDialog.size")}
-
- {(Object.keys(GIF_SIZE_PRESETS) as GifSizePreset[]).map((s) => ( - - ))} -
-
-
- {t("exportDialog.loopGif")} + {dims.width} × {dims.height} + {isUpscale ? ` · ${t("exportDialog.qualityUpscaleWarning")}` : ""} + + ); + })()} + + ))} +
+
+
{t("exportDialog.frameRate")}
+
+ {([24, 30, 60] as const).map((r) => (
-
- {gifFrameRate} FPS · {gifSizeLabel} ·{" "} - {gifLoop ? t("exportDialog.loopOn") : t("exportDialog.loopOff")} -
-
- )} - - ) : null} + onClick={() => setFps(r)} + style={segStyle(fps === r)} + > + {r} + + ))} +
+ + + ) : ( +
+
+
{t("exportDialog.frameRate")}
+
+ {GIF_FRAME_RATES.map((r) => ( + + ))} +
+
+
+
{t("exportDialog.size")}
+
+ {(Object.keys(GIF_SIZE_PRESETS) as GifSizePreset[]).map((s) => ( + + ))} +
+
+
+ {t("exportDialog.loopGif")} + +
+
+ {gifFrameRate} FPS · {gifSizeLabel} ·{" "} + {gifLoop ? t("exportDialog.loopOn") : t("exportDialog.loopOff")} +
+
+ )} - {t("exportDialog.pickFormatAndExport")} - - ); - } + // Nothing to say before an export: the format is the first control on screen and + // already picked. The old "Pick a format and press Export to start" plate was written + // for the UI that hid the format toggle under Advanced, and stayed up through the save + // picker where it was simply false. + if (phase === "idle" || phase === "configuring") return null; if (phase === "done") { return (
- - , + + + + + , ); // Flush the session load so the assistant bubble (and its copy button) is on screen. await waitFor(() => { - expect(view.getAllByTitle("chat.copyMessage").length).toBeGreaterThan(0); + expect(view.getAllByRole("button", { name: "chat.copyMessage" }).length).toBeGreaterThan(0); }); return view; } +const copyButtonsOf = (view: ReturnType) => + view.getAllByRole("button", { name: "chat.copyMessage" }); + describe("ChatStripPanel Copy message (issue #738)", () => { it("copies the full message through the preload bridge and reports success, never touching the denied navigator clipboard", async () => { stubClipboard(copyToClipboard); const view = await renderPanelWithAssistantMessage(); // Two copy buttons (user + assistant); the assistant message is the last one rendered. - const copyButtons = view.getAllByTitle("chat.copyMessage"); + const copyButtons = copyButtonsOf(view); // user + assistant + the empty trailing assistant turn. expect(copyButtons).toHaveLength(3); await act(async () => { @@ -193,7 +199,7 @@ describe("ChatStripPanel Copy message (issue #738)", () => { stubClipboard(undefined); const view = await renderPanelWithAssistantMessage(); - const copyButtons = view.getAllByTitle("chat.copyMessage"); + const copyButtons = copyButtonsOf(view); // The fallback inherits Electron's denial in a bare context: the click // lands on the navigator path and reports the failure toast. await act(async () => { @@ -229,7 +235,7 @@ describe("ChatStripPanel Copy message (issue #738)", () => { // copy button sits last. The handler must not branch on content — an // empty string crosses to the bridge exactly like any other content and // still reports success. - const copyButtons = view.getAllByTitle("chat.copyMessage"); + const copyButtons = copyButtonsOf(view); await act(async () => { fireEvent.click(copyButtons[copyButtons.length - 1]); await Promise.resolve(); diff --git a/src/components/ai-edition/LeftPanel.providerRefresh.test.tsx b/src/components/ai-edition/LeftPanel.providerRefresh.test.tsx index f14a8d73c..0501e5093 100644 --- a/src/components/ai-edition/LeftPanel.providerRefresh.test.tsx +++ b/src/components/ai-edition/LeftPanel.providerRefresh.test.tsx @@ -45,6 +45,7 @@ vi.mock("@/contexts/I18nContext", () => ({ useScopedT: () => (key: string) => key, })); +import { TooltipProvider } from "@/components/ui/tooltip"; import { EditorDialogsProvider, useEditorDialogActions } from "@/contexts/EditorDialogsContext"; import { ChatStripPanel } from "./LeftPanel"; @@ -80,10 +81,12 @@ afterEach(() => { describe("ChatStripPanel, against the lifted provider dialog", () => { it("re-reads the LLM snapshot when the dialog closes, and not when it opens", async () => { render( - - - - , + + + + + + , ); // Mount: the dialog is closed, so the same effect that watches for a close seeds the // composer's view of the provider config. diff --git a/src/components/ai-edition/LeftPanel.tooltips.test.tsx b/src/components/ai-edition/LeftPanel.tooltips.test.tsx new file mode 100644 index 000000000..9796e9b64 --- /dev/null +++ b/src/components/ai-edition/LeftPanel.tooltips.test.tsx @@ -0,0 +1,220 @@ +// @vitest-environment jsdom +// The chat panel's icon-only controls name themselves in the shared tooltip, not in a native +// `title`. Rewind and Compact say what they do, Send shows Enter as a chip, and a button that is +// greyed for now (Send with no provider, Compact with no conversation) still opens its tooltip: +// that is where "Set up a provider" is said, and a natively disabled button takes no pointer +// events, so it uses `aria-disabled`. + +import "@testing-library/jest-dom"; +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const snapshot = vi.hoisted(() => ({ + value: { + config: null as { provider: string; model: string } | null, + connectedProviders: [] as string[], + availableProviders: [] as unknown[], + credentialSummary: [] as unknown[], + }, +})); +const sessions = vi.hoisted(() => ({ list: [] as unknown[] })); +const chatCompact = vi.hoisted(() => vi.fn()); + +vi.mock("@/native/client", () => ({ + nativeBridgeClient: { + aiEdition: { + llmGetSnapshot: () => Promise.resolve(snapshot.value), + chatListSessions: () => Promise.resolve(sessions.list), + chatSelectSession: () => + Promise.resolve({ + id: "session-1", + projectId: "project-1", + title: "Conversation 1", + createdAt: "2026-09-23T00:00:00Z", + updatedAt: "2026-09-23T00:00:00Z", + messages: [ + { + id: "m1", + role: "user", + content: "Cut the silences", + checkpointId: "cp1", + createdAt: "2026-09-23T00:00:01Z", + }, + { + id: "m2", + role: "assistant", + content: "Done", + createdAt: "2026-09-23T00:00:02Z", + }, + ], + }), + chatBudget: () => Promise.resolve(null), + chatCompact, + llmListProviderModels: () => Promise.resolve({ models: [] }), + }, + }, +})); + +vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn(), info: vi.fn() } })); + +// An echoing translator: the assertions read against keys, not prose that drifts with copy edits. +vi.mock("@/contexts/I18nContext", () => ({ + useI18n: () => ({ locale: "en", setLocale: vi.fn() }), + useScopedT: () => (key: string) => key, +})); + +import { TooltipProvider } from "@/components/ui/tooltip"; +import { EditorDialogsProvider } from "@/contexts/EditorDialogsContext"; +import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { ChatStripPanel } from "./LeftPanel"; + +class StubResizeObserver { + observe = vi.fn(); + unobserve = vi.fn(); + disconnect = vi.fn(); +} + +const oneSession = [ + { id: "session-1", title: "Conversation 1", messageCount: 2, createdAt: "2026-09-23T00:00:00Z" }, +]; + +beforeEach(() => { + vi.stubGlobal("ResizeObserver", StubResizeObserver); + // jsdom implements no scrolling, and the transcript pins itself to the bottom on every render. + Element.prototype.scrollTo = vi.fn(); + (window as unknown as { electronAPI?: unknown }).electronAPI = { + onAiEditionChatEvent: () => vi.fn(), + }; + useProjectStore.setState({ projectId: "project-1" }); + snapshot.value = { + config: null, + connectedProviders: [], + availableProviders: [], + credentialSummary: [], + }; + sessions.list = []; + chatCompact.mockClear(); +}); + +afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + (window as unknown as { electronAPI?: unknown }).electronAPI = undefined; +}); + +function renderPanel() { + return render( + + + + + , + ); +} + +/** Opens a control's tooltip the way the keyboard does (focus opens it at once) and returns what + * it says and whether it shows a shortcut chip. */ +async function tooltipOn(control: HTMLElement) { + act(() => control.focus()); + const text = (await screen.findByRole("tooltip")).textContent; + const chip = document.querySelector('[data-slot="tooltip-content"] kbd')?.textContent ?? null; + act(() => control.blur()); + await waitFor(() => expect(screen.queryByRole("tooltip")).toBeNull()); + return { text, chip }; +} + +describe("ChatStripPanel tooltips", () => { + it("names the header buttons in the shared tooltip, with no native title", async () => { + sessions.list = oneSession; + renderPanel(); + await screen.findByRole("button", { name: "chat.deleteConversation" }); + + for (const name of [ + "chat.compactContext", + "chat.aiSettings", + "chat.history", + "chat.newConversation", + "chat.renameConversation", + "chat.deleteConversation", + ]) { + const button = screen.getByRole("button", { name }); + expect(button).not.toHaveAttribute("title"); + expect((await tooltipOn(button)).text).toBe(name); + } + }); + + // Plain text has no interactive element to hold a tooltip, so its hint stays native. + it("keeps the native title only on plain text", async () => { + sessions.list = oneSession; + renderPanel(); + expect(await screen.findByTitle("chat.clickToRename")).toBeInTheDocument(); + }); + + it("says what Rewind does, not only what the button is called", async () => { + sessions.list = oneSession; + renderPanel(); + const rewind = await screen.findByRole("button", { name: "chat.rewindToMessage" }); + expect(rewind).not.toHaveAttribute("title"); + expect((await tooltipOn(rewind)).text).toBe("chat.rewindTip"); + }); + + it("names Copy in the shared tooltip too", async () => { + sessions.list = oneSession; + renderPanel(); + const [copy] = await screen.findAllByRole("button", { name: "chat.copyMessage" }); + expect(copy).not.toHaveAttribute("title"); + expect((await tooltipOn(copy)).text).toBe("chat.copyMessage"); + }); + + // Nothing to talk to yet: Send is greyed, but it is where the reason is said. + it("says why Send is off when no provider is set up, and shows no chip then", async () => { + renderPanel(); + const send = await screen.findByRole("button", { name: "chat.send" }); + await waitFor(() => expect(send).toHaveAttribute("aria-disabled", "true")); + expect(send).not.toBeDisabled(); + expect(send).not.toHaveAttribute("title"); + const tip = await tooltipOn(send); + expect(tip.text).toBe("chat.composerDisabledNoProvider"); + expect(tip.chip).toBeNull(); + }); + + it("shows Enter as a chip on Send once it can send, never inside the string", async () => { + snapshot.value = { + config: { provider: "openai", model: "gpt-x" }, + connectedProviders: ["openai"], + availableProviders: [], + credentialSummary: [], + }; + renderPanel(); + fireEvent.change(await screen.findByPlaceholderText("chat.composerPlaceholder"), { + target: { value: "Cut the silences" }, + }); + const send = screen.getByRole("button", { name: "chat.send" }); + await waitFor(() => expect(send).not.toHaveAttribute("aria-disabled")); + const tip = await tooltipOn(send); + expect(tip.text).toBe("chat.sendEnter"); + expect(tip.chip).toBe("Enter"); + }); + + // While the title is being edited the button is greyed, but it stays hoverable and does nothing. + it("keeps Rename tooltip-able while the title is being edited", async () => { + sessions.list = oneSession; + renderPanel(); + const rename = await screen.findByRole("button", { name: "chat.renameConversation" }); + fireEvent.click(rename); + await screen.findByDisplayValue("Conversation 1"); + expect(rename).toHaveAttribute("aria-disabled", "true"); + expect(rename).not.toBeDisabled(); + expect((await tooltipOn(rename)).text).toBe("chat.renameConversation"); + }); + + it("keeps Compact tooltip-able while there is no conversation, and does nothing", async () => { + renderPanel(); + const compact = await screen.findByRole("button", { name: "chat.compactContext" }); + expect(compact).toHaveAttribute("aria-disabled", "true"); + expect(compact).not.toBeDisabled(); + expect((await tooltipOn(compact)).text).toBe("chat.compactContext"); + fireEvent.click(compact); + expect(chatCompact).not.toHaveBeenCalled(); + }); +}); diff --git a/src/components/ai-edition/LeftPanel.tsx b/src/components/ai-edition/LeftPanel.tsx index 81c62c306..82e809268 100644 --- a/src/components/ai-edition/LeftPanel.tsx +++ b/src/components/ai-edition/LeftPanel.tsx @@ -2,6 +2,7 @@ import { ArrowLeft, Check, Loader2, X } from "lucide-react"; import { useCallback, useEffect, useRef, useState } from "react"; import { createPortal } from "react-dom"; import { toast } from "sonner"; +import { Tooltip } from "@/components/ui/tooltip"; import { useEditorDialogActions, useEditorDialogSection } from "@/contexts/EditorDialogsContext"; import { useScopedT } from "@/contexts/I18nContext"; import { @@ -471,6 +472,7 @@ export function ChatStripPanel() { const [connectedProviders, setConnectedProviders] = useState(null); // unknown ≠ none; see chatAvailability.ts. const canChat = canSendChat(llmConfig, connectedProviders); + const sendDisabled = busy || !input.trim() || !canChat; const [modelPopoverOpen, setModelPopoverOpen] = useState(false); const modelButtonRef = useRef(null); const [modelPopoverRect, setModelPopoverRect] = useState<{ @@ -1006,92 +1008,95 @@ export function ChatStripPanel() { {t("chat.contextPercent", { percent: Math.min(100, Math.round(budget.ratio * 100)) })} - - - + + + - + + + + + + + + + + + + +
@@ -1160,61 +1165,66 @@ export function ChatStripPanel() { t("chat.untitledConversation")} )} - - + + + + + + + + + + + + ) : null} @@ -1293,21 +1303,59 @@ export function ChatStripPanel() { }} > {m.role === "user" && m.checkpointId ? ( + // The name stays "Rewind to this message"; the tip adds what it does, which + // only the confirmation dialog said before. + + + + ) : null} + - ) : null} - + {m.toolCalls?.length ? (
@@ -1546,28 +1560,34 @@ export function ChatStripPanel() { onOpenFullSettings={openProviderSettings} /> ) : null} - + + + + + +
{ + vi.spyOn(SVGElement.prototype, "getBoundingClientRect").mockReturnValue( + DOMRect.fromRect({ x: 0, y: 0, width: 1000, height: 500 }), + ); +}); +afterAll(() => vi.restoreAllMocks()); + +const BLUR = { + id: "b", + startMs: 0, + endMs: 1000, + type: "blur", + content: "", + position: { x: 10, y: 20 }, + size: { width: 30, height: 40 }, + style: {}, + zIndex: 1, + blurData: { type: "mosaic", shape: "rectangle", color: "white", intensity: 12, blockSize: 12 }, +} as unknown as AxcutAnnotationRegion; + +// The footage zoomed ×2 around its middle: one unit of footage is 1.6 units of frame. +const ZOOMED = rectQuad(-0.3, -0.3, 1.6, 1.6); + +function gimbal() { + const onChange = vi.fn(); + const onCommit = vi.fn(); + const onClick = vi.fn(); + const { container } = render( + , + ); + return { container, onChange, onCommit, onClick }; +} + +const at = (x: number, y: number) => ({ clientX: x, clientY: y, pointerId: 1 }); + +describe("MaskGimbal", () => { + it("moves the mask by what the pointer covers of the footage, zoomed", () => { + const { container, onChange, onCommit } = gimbal(); + const body = container.querySelector("polygon") as SVGPolygonElement; + fireEvent.pointerDown(body, at(300, 200)); + // 160 px of a 1000 px frame, zoomed ×2 over 1.6: a tenth of the footage. + fireEvent.pointerMove(body, at(460, 200)); + fireEvent.pointerUp(body, at(460, 200)); + const last = onChange.mock.lastCall?.[1]; + expect(last.position.x).toBeCloseTo(20, 9); + expect(last.position.y).toBeCloseTo(20, 9); + expect(onCommit).toHaveBeenCalledTimes(1); + }); + + it("resizes from a corner while the opposite one stays", () => { + const { container, onChange } = gimbal(); + const bottomRight = container.querySelectorAll("circle")[2]; + fireEvent.pointerDown(bottomRight, at(0, 0)); + // Frame point (0.5, 0.5) is the middle of the footage. + fireEvent.pointerMove(bottomRight, at(500, 250)); + const last = onChange.mock.lastCall?.[1]; + expect(last.position).toEqual({ x: 10, y: 20 }); + expect(last.size.width).toBeCloseTo(40, 9); + expect(last.size.height).toBeCloseTo(30, 9); + }); + + it("selects through on a click that moves nothing", () => { + const { container, onChange, onCommit, onClick } = gimbal(); + const body = container.querySelector("polygon") as SVGPolygonElement; + fireEvent.pointerDown(body, at(300, 200)); + fireEvent.pointerUp(body, at(300, 200)); + expect(onChange).not.toHaveBeenCalled(); + expect(onCommit).not.toHaveBeenCalled(); + expect(onClick).toHaveBeenCalledWith("b"); + }); +}); diff --git a/src/components/ai-edition/MaskGimbal.tsx b/src/components/ai-edition/MaskGimbal.tsx new file mode 100644 index 000000000..35645f154 --- /dev/null +++ b/src/components/ai-edition/MaskGimbal.tsx @@ -0,0 +1,189 @@ +// Le gimbal d'un flou de confidentialité. Le masque suit le contenu : le compositeur le pose sur +// le métrage zoomé, et incliné sous la 3D (`privacy_mask`). Le gimbal passe donc par le métrage +// de l'image (`FootageQuad`) : contour, poignées et gestes vivent dans le repère du métrage, et +// chaque point est projeté comme le compositeur le peint. Un `Rnd` droit, posé sur le métrage au +// repos, s'écartait du masque dès le premier zoom. + +import type { PointerEvent as ReactPointerEvent } from "react"; +import { useRef } from "react"; +import { + type FootageQuad, + footageAt, + footagePoint, +} from "@/lib/ai-edition/annotations/footageQuad"; +import type { AxcutAnnotationRegion } from "@/lib/ai-edition/schema"; + +type Region = AxcutAnnotationRegion; + +interface MaskGimbalProps { + annotation: Region; + /** Le métrage dans l'image, en fractions de l'image. */ + quad: FootageQuad; + /** L'image de l'aperçu, px. */ + frameWidth: number; + frameHeight: number; + onChange: (id: string, patch: Partial) => void; + /** Écriture disque, une fois en fin de geste. */ + onCommit?: () => void; + onClick: (id: string) => void; + zIndex: number; +} + +const GREEN = "#34B27B"; +/** Le plus petit masque que laissent les poignées, en % du métrage. */ +const MIN_PCT = 1; +/** Les poignées dans l'ordre des coins : TL, TR, BR, BL. */ +const CORNER_CURSORS = ["nwse-resize", "nesw-resize", "nwse-resize", "nesw-resize"] as const; + +type Gesture = + | { + kind: "move"; + start: readonly [number, number]; + origin: { x: number; y: number }; + moved: boolean; + } + | { kind: "corner"; fixed: readonly [number, number]; moved: boolean }; + +const clamp = (value: number, lo: number, hi: number) => Math.min(hi, Math.max(lo, value)); + +export function MaskGimbal({ + annotation, + quad, + frameWidth, + frameHeight, + onChange, + onCommit, + onClick, + zIndex, +}: MaskGimbalProps) { + const svgRef = useRef(null); + const gesture = useRef(null); + const { position, size } = annotation; + // Le masque en fractions du métrage. + const [u0, v0] = [position.x / 100, position.y / 100]; + const [u1, v1] = [u0 + size.width / 100, v0 + size.height / 100]; + const onFrame = (u: number, v: number) => { + const [x, y] = footagePoint(quad, u, v); + return [x * frameWidth, y * frameHeight] as const; + }; + const corners = [ + [u0, v0], + [u1, v0], + [u1, v1], + [u0, v1], + ] as const; + // Un ovale se dessine en ovale, projeté ; le tracé libre, comme le masque, par sa boîte. + const outline = + annotation.blurData?.shape === "oval" + ? Array.from({ length: 48 }, (_, k) => { + const t = (k / 48) * 2 * Math.PI; + return onFrame( + u0 + (u1 - u0) * (0.5 + 0.5 * Math.cos(t)), + v0 + (v1 - v0) * (0.5 + 0.5 * Math.sin(t)), + ); + }) + : corners.map(([u, v]) => onFrame(u, v)); + + /** Le point du métrage sous le pointeur, en fractions du métrage. */ + const under = (event: ReactPointerEvent): readonly [number, number] => { + const r = svgRef.current?.getBoundingClientRect(); + if (!r || r.width <= 0 || r.height <= 0) return [0, 0]; + return footageAt(quad, (event.clientX - r.left) / r.width, (event.clientY - r.top) / r.height); + }; + + const begin = (event: ReactPointerEvent, next: Gesture) => { + event.stopPropagation(); + event.currentTarget.setPointerCapture?.(event.pointerId); + gesture.current = next; + }; + + const onPointerMove = (event: ReactPointerEvent) => { + const g = gesture.current; + if (!g) return; + const [u, v] = under(event); + if (g.kind === "move") { + const x = clamp(g.origin.x + (u - g.start[0]) * 100, 0, 100 - size.width); + const y = clamp(g.origin.y + (v - g.start[1]) * 100, 0, 100 - size.height); + g.moved ||= x !== g.origin.x || y !== g.origin.y; + // En direct : c'est le natif qui peint, il suit le geste. L'écriture disque attend + // le relâchement (`onCommit`). + onChange(annotation.id, { position: { x, y } }); + return; + } + // Le coin opposé reste en place ; la poignée peut passer de l'autre côté, sans que le + // masque descende sous le minimum ni sorte du métrage. + const span = (c: number, fixed: number): [number, number] => { + const [lo, hi] = + c < fixed ? [Math.min(c, fixed - MIN_PCT), fixed] : [fixed, Math.max(c, fixed + MIN_PCT)]; + const top = Math.min(hi, 100); + return [Math.max(0, Math.min(lo, top - MIN_PCT)), top]; + }; + const [x0, x1] = span(clamp(u * 100, 0, 100), g.fixed[0]); + const [y0, y1] = span(clamp(v * 100, 0, 100), g.fixed[1]); + g.moved = true; + onChange(annotation.id, { + position: { x: x0, y: y0 }, + size: { width: x1 - x0, height: y1 - y0 }, + }); + }; + + const onPointerUp = (event: ReactPointerEvent) => { + const g = gesture.current; + gesture.current = null; + if (!g) return; + (event.target as Element).releasePointerCapture?.(event.pointerId); + if (g.moved) onCommit?.(); + else if (g.kind === "move") onClick(annotation.id); + }; + + const points = (list: readonly (readonly [number, number])[]) => + list.map(([x, y]) => `${x},${y}`).join(" "); + + return ( + + + begin(event, { kind: "move", start: under(event), origin: { ...position }, moved: false }) + } + /> + {corners.map(([u, v], k) => { + const [cx, cy] = onFrame(u, v); + const [ou, ov] = corners[(k + 2) % 4]; + return ( + + begin(event, { kind: "corner", fixed: [ou * 100, ov * 100], moved: false }) + } + /> + ); + })} + + ); +} diff --git a/src/components/ai-edition/McpServerSettings.test.tsx b/src/components/ai-edition/McpServerSettings.test.tsx new file mode 100644 index 000000000..453a4961d --- /dev/null +++ b/src/components/ai-edition/McpServerSettings.test.tsx @@ -0,0 +1,102 @@ +// @vitest-environment jsdom +import "@testing-library/jest-dom"; +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import type { AiEditionMcpStatus } from "@/native/contracts"; + +const TOKEN = "secret-token"; +const URL = "http://127.0.0.1:47821/mcp"; + +function status(enabled: boolean, allowEdits = false): AiEditionMcpStatus { + return { + enabled, + port: 47821, + allowEdits, + running: enabled, + url: URL, + token: enabled ? TOKEN : null, + error: null, + }; +} + +const bridge = vi.hoisted(() => ({ + mcpGetStatus: vi.fn(), + mcpSetEnabled: vi.fn(), + mcpSetPort: vi.fn(), + mcpSetAllowEdits: vi.fn(), + mcpRegenerateToken: vi.fn(), +})); + +vi.mock("@/native/client", () => ({ nativeBridgeClient: { aiEdition: bridge } })); + +import { claudeCodeCommand, codexCommand, McpServerSettings } from "./McpServerSettings"; + +const copyToClipboard = vi.fn(() => Promise.resolve()); + +function renderSection() { + return render( + + + , + ); +} + +describe("McpServerSettings", () => { + beforeEach(() => { + for (const fn of Object.values(bridge)) fn.mockReset(); + copyToClipboard.mockClear(); + (window as unknown as { electronAPI: unknown }).electronAPI = { copyToClipboard }; + }); + afterEach(cleanup); + + it("shows the server as off, with no token or commands", async () => { + bridge.mcpGetStatus.mockResolvedValue(status(false)); + renderSection(); + expect(await screen.findByTestId("mcp-server-settings")).toBeInTheDocument(); + expect(screen.getByTestId("mcp-server-toggle")).toHaveAttribute("aria-pressed", "false"); + expect(screen.queryByText(/claude mcp add/)).not.toBeInTheDocument(); + }); + + it("turns the server on and shows how to connect, token masked", async () => { + bridge.mcpGetStatus.mockResolvedValue(status(false)); + bridge.mcpSetEnabled.mockResolvedValue(status(true)); + renderSection(); + fireEvent.click(await screen.findByTestId("mcp-server-toggle")); + await waitFor(() => expect(bridge.mcpSetEnabled).toHaveBeenCalledWith(true)); + expect(await screen.findByText(codexCommand(URL))).toBeInTheDocument(); + expect(screen.queryByText(new RegExp(TOKEN))).not.toBeInTheDocument(); + }); + + it("keeps MCP edits off until they are turned on", async () => { + bridge.mcpGetStatus.mockResolvedValue(status(true)); + bridge.mcpSetAllowEdits.mockResolvedValue(status(true, true)); + renderSection(); + const toggle = await screen.findByTestId("mcp-edits-toggle"); + expect(toggle).toHaveAttribute("aria-pressed", "false"); + fireEvent.click(toggle); + await waitFor(() => expect(bridge.mcpSetAllowEdits).toHaveBeenCalledWith(true)); + await waitFor(() => + expect(screen.getByTestId("mcp-edits-toggle")).toHaveAttribute("aria-pressed", "true"), + ); + }); + + it("copies the real token into the Claude Code command", async () => { + bridge.mcpGetStatus.mockResolvedValue(status(true)); + renderSection(); + await screen.findByText(codexCommand(URL)); + const copyButtons = screen.getAllByRole("button", { name: "Copy" }); + // Token, Claude Code, Codex — in that order. + fireEvent.click(copyButtons[1]); + expect(copyToClipboard).toHaveBeenCalledWith(claudeCodeCommand(URL, TOKEN)); + }); + + it("stays out of the way when the status cannot be read", async () => { + bridge.mcpGetStatus.mockImplementation(() => { + throw new Error("no bridge"); + }); + renderSection(); + await waitFor(() => expect(bridge.mcpGetStatus).toHaveBeenCalled()); + expect(screen.queryByTestId("mcp-server-settings")).not.toBeInTheDocument(); + }); +}); diff --git a/src/components/ai-edition/McpServerSettings.tsx b/src/components/ai-edition/McpServerSettings.tsx new file mode 100644 index 000000000..785911bea --- /dev/null +++ b/src/components/ai-edition/McpServerSettings.tsx @@ -0,0 +1,311 @@ +// The MCP server section of AI settings: turn the local server on, pick its +// port, and copy what an MCP client needs to connect. The server itself lives +// in electron/mcp/; this only drives it through the native bridge. + +import { AlertCircle, Check, Copy, Eye, EyeOff, Loader2, RefreshCw } from "lucide-react"; +import { useCallback, useEffect, useState } from "react"; +import { toast } from "sonner"; +import { useScopedT } from "@/contexts/I18nContext"; +import { nativeBridgeClient } from "@/native/client"; +import type { AiEditionMcpStatus } from "@/native/contracts"; +import styles from "./NewEditorShell.module.css"; + +const CODEX_TOKEN_ENV = "OPENSCREEN_MCP_TOKEN"; +const MASKED_TOKEN = "••••••••"; + +export function claudeCodeCommand(url: string, token: string): string { + return `claude mcp add --transport http openscreen ${url} --header "Authorization: Bearer ${token}"`; +} + +export function codexCommand(url: string): string { + return `codex mcp add openscreen --url ${url} --bearer-token-env-var ${CODEX_TOKEN_ENV}`; +} + +// Commands and the token wrap onto as many lines as they need. `nowrap` here made +// the `1fr` grid column as wide as the longest command, which pushed the copy +// buttons out of the dialog and gave the whole modal a horizontal scrollbar. +const codeStyle: React.CSSProperties = { + flex: 1, + minWidth: 0, + whiteSpace: "pre-wrap", + overflowWrap: "anywhere", + padding: "6px 8px", + borderRadius: 6, + background: "var(--bg-2, rgba(127,127,127,0.12))", + font: "12px/1.5 var(--font-mono)", + color: "var(--fg-2)", + // A command is copied and typed as-is, so it has to render as-is: Geist Mono's + // contextual alternates drew " --header" as "--header", hiding the space. + fontVariantLigatures: "none", + fontFeatureSettings: '"liga" 0, "calt" 0', + userSelect: "text", +}; + +const rowStyle: React.CSSProperties = { + display: "flex", + alignItems: "flex-start", + gap: 8, + minWidth: 0, +}; + +export function McpServerSettings({ open }: { open: boolean }) { + const te = useScopedT("editor"); + const [status, setStatus] = useState(null); + const [portDraft, setPortDraft] = useState(""); + const [busy, setBusy] = useState(false); + const [showToken, setShowToken] = useState(false); + + const adopt = useCallback((next: AiEditionMcpStatus) => { + setStatus(next); + setPortDraft(String(next.port)); + }, []); + + useEffect(() => { + if (!open) { + setShowToken(false); + return; + } + // Any failure, even a synchronous one, just hides the section: it must never + // take the provider settings around it down with it. + void Promise.resolve() + .then(() => nativeBridgeClient.aiEdition.mcpGetStatus()) + .then(adopt, () => setStatus(null)); + }, [open, adopt]); + + const run = async (task: () => Promise) => { + setBusy(true); + try { + adopt(await task()); + } catch (err) { + toast.error(te("mcpServer.updateFailed"), { + description: err instanceof Error ? err.message : String(err), + }); + } finally { + setBusy(false); + } + }; + + const copy = (text: string) => { + const bridge = window.electronAPI?.copyToClipboard; + const write = bridge ? bridge(text) : navigator.clipboard.writeText(text); + void write.then( + () => toast.success(te("mcpServer.copied")), + () => toast.error(te("mcpServer.copyFailed")), + ); + }; + + if (!status) return null; + + const port = Number(portDraft); + const portChanged = portDraft !== String(status.port); + const token = status.token; + const shownToken = token && showToken ? token : MASKED_TOKEN; + + return ( +
+
+
+

{te("mcpServer.title")}

+ {status.running ? ( + + + {te("mcpServer.statusRunning")} + + ) : ( + + {status.enabled ? te("mcpServer.statusError") : te("mcpServer.statusOff")} + + )} +
+

{te("mcpServer.description")}

+ + run(() => nativeBridgeClient.aiEdition.mcpSetEnabled(!status.enabled))} + /> + + + run(() => nativeBridgeClient.aiEdition.mcpSetAllowEdits(!status.allowEdits)) + } + /> + +
+ +
+ setPortDraft(e.target.value)} + disabled={busy} + style={{ width: 120 }} + /> + {portChanged ? ( + + ) : null} +
+
+ + {status.error ? ( +

+ + {status.error} +

+ ) : null} + + {status.enabled && token ? ( + <> +
+ +
+ {shownToken} + setShowToken((v) => !v)} + > + {showToken ? : } + + copy(token)}> + + + run(() => nativeBridgeClient.aiEdition.mcpRegenerateToken())} + > + {busy ? : } + +
+

{te("mcpServer.tokenHint")}

+
+ +
+ +
+ {claudeCodeCommand(status.url, shownToken)} + copy(claudeCodeCommand(status.url, token))} + > + + +
+
+ +
+ +
+ {codexCommand(status.url)} + copy(codexCommand(status.url))} + > + + +
+

{te("mcpServer.codexHint", { env: CODEX_TOKEN_ENV })}

+
+ + ) : null} +
+
+ ); +} + +/** The panes' switch, not a system checkbox, inside its label so the text toggles it too. */ +function ToggleField({ + label, + text, + hint, + testId, + on, + disabled, + onToggle, +}: { + label: string; + text: string; + hint?: string; + testId: string; + on: boolean; + disabled: boolean; + onToggle: () => void; +}) { + return ( +
+ +
+ ); +} + +function IconButton({ + label, + onClick, + disabled, + children, +}: { + label: string; + onClick: () => void; + disabled?: boolean; + children: React.ReactNode; +}) { + return ( + + ); +} diff --git a/src/components/ai-edition/NativeCompositorOverlay.test.tsx b/src/components/ai-edition/NativeCompositorOverlay.test.tsx new file mode 100644 index 000000000..f776feab7 --- /dev/null +++ b/src/components/ai-edition/NativeCompositorOverlay.test.tsx @@ -0,0 +1,315 @@ +// @vitest-environment jsdom +/** + * How the overlay steers the native view while the transport plays. + * + * The view runs its own clock and crosses clip boundaries by itself. The overlay used to + * re-send the clip at every cut anyway, which made the view seek back to a place it had just + * left, or drop the next clip it had preloaded: a hitch at every cut of an edited take. Now it + * reads where the view is from the position each frame reports, and only steps in when the + * view is somewhere else. + */ +import { act, cleanup, render } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AxcutClip, AxcutDocument } from "@/lib/ai-edition/schema"; +import { axcutSchemaVersion } from "@/lib/ai-edition/schema"; +import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { publishNativePosition } from "@/native/nativeSync"; + +const native = vi.hoisted(() => ({ + setActiveClip: vi.fn(async () => ({ ok: true })), + setNativePlaying: vi.fn(), + setNativeScene: vi.fn(), +})); +const i18n = vi.hoisted(() => ({ locale: "en" })); + +vi.mock("@/native", () => ({ + pushAllNativeParams: vi.fn(), + setActiveClip: native.setActiveClip, + setCurrentNativeViewId: vi.fn(), + setNativePlaying: native.setNativePlaying, + setNativeScene: native.setNativeScene, + subscribeNativeCompositor: () => () => undefined, + useIsCpuCompositor: () => false, + useNativeCompositorView: () => ({ viewId: 7, error: null }), +})); + +vi.mock("@/contexts/I18nContext", () => ({ + useI18n: () => ({ locale: i18n.locale }), + useScopedT: () => (key: string) => key, +})); + +import { NativeCompositorOverlay } from "./NativeCompositorOverlay"; + +/** One take cut in two: source 0-5 s, then 7-12 s after a 2 s cut. */ +const FIRST: AxcutClip = { + id: "clip_a", + assetId: "asset_take", + sourceStartSec: 0, + sourceEndSec: 5, + timelineStartSec: 0, + timelineEndSec: 5, + wordRefs: [], + origin: "system", + reason: "", +}; +const SECOND: AxcutClip = { + ...FIRST, + id: "clip_b", + sourceStartSec: 7, + sourceEndSec: 12, + timelineStartSec: 5, + timelineEndSec: 10, +}; + +function makeDocument(): AxcutDocument { + return { + schemaVersion: axcutSchemaVersion, + project: { + id: "proj_sync", + title: "Sync", + createdAt: "2026-10-02T08:00:00.000Z", + updatedAt: "2026-10-02T08:00:00.000Z", + primaryAssetId: "asset_take", + }, + assets: [ + { + id: "asset_take", + kind: "video", + label: "take", + originalPath: "/take.mp4", + cameraTrack: null, + }, + ], + transcript: null, + transcripts: [], + timeline: { + clips: [FIRST, SECOND], + gaps: [], + trimRanges: [], + muteRanges: [], + speedRanges: [], + captionRanges: [], + }, + annotations: [], + zoomRanges: [], + audioTracks: [], + legacyEditor: null, + }; +} + +function setPlayhead(currentTimeSec: number, playing: boolean) { + act(() => { + useProjectStore.setState({ currentTimeSec, playing }); + }); +} + +describe("NativeCompositorOverlay while playing", () => { + let now = 1000; + + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(performance, "now").mockImplementation(() => now); + useProjectStore.setState({ + projectId: "proj_sync", + document: makeDocument(), + revision: 1, + status: "ready", + error: null, + sourceDurationSec: 12, + currentTimeSec: 4.9, + playing: true, + dirty: false, + lastSavedAt: new Date(), + }); + }); + + afterEach(() => { + cleanup(); + vi.restoreAllMocks(); + publishNativePosition(null); + useProjectStore.getState().clear(); + }); + + async function mountAtFirstClip() { + render(); + // The first mount sends the starting clip, and its reply resumes the view: let both + // settle, so only what follows is under test. + await act(async () => { + await Promise.resolve(); + }); + native.setActiveClip.mockClear(); + native.setNativePlaying.mockClear(); + } + + it("lets the view cross a cut by itself", async () => { + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.99 }, now); + + setPlayhead(5.02, true); + + expect(native.setActiveClip).not.toHaveBeenCalled(); + expect(native.setNativePlaying).not.toHaveBeenCalled(); + }); + + it("follows a jump to another clip at once, without pausing the view", async () => { + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.9 }, now); + + setPlayhead(8, true); + + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 1, 10, []); + expect(native.setNativePlaying).not.toHaveBeenCalledWith(false); + }); + + it("sends the asset's extra cameras with the clip", async () => { + const document = makeDocument(); + document.assets[0] = { + ...document.assets[0], + additionalCameraTracks: [ + { sourcePath: "/cam-2.mp4", startMs: 500, offsetMs: 0, visible: true, label: "" }, + { sourcePath: "/cam-3.mp4", startMs: 0, offsetMs: 0, visible: false, label: "" }, + ], + }; + useProjectStore.setState({ document }); + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.9 }, now); + + setPlayhead(8, true); + + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 1, 10, [ + { path: "/cam-2.mp4", offsetSec: 0.5 }, + { path: "", offsetSec: 0 }, + ]); + }); + + it("re-anchors a view that stays behind, once the gap holds", async () => { + await mountAtFirstClip(); + // The view stalled 0.4 s behind the playhead, inside the first clip. + publishNativePosition({ clipIndex: 0, sourceTimeSec: 2 }, now); + setPlayhead(2.4, true); + expect(native.setActiveClip).not.toHaveBeenCalled(); + + now += 120; + publishNativePosition({ clipIndex: 0, sourceTimeSec: 2.12 }, now); + setPlayhead(2.52, true); + + expect(native.setActiveClip).toHaveBeenCalledTimes(1); + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 0, 2.52, []); + }); + + // At 16× a frame that takes 30 ms to arrive shows the playhead 0.48 s of programme ago. + it("leaves alone a view keeping pace inside a speed region", async () => { + useProjectStore.setState({ + document: { + ...makeDocument(), + legacyEditor: { speedRegions: [{ id: "speed_a", startMs: 0, endMs: 12000, speed: 16 }] }, + }, + currentTimeSec: 0.2, + }); + await mountAtFirstClip(); + + for (let frame = 0; frame < 8; frame++) { + now += 30; + publishNativePosition({ clipIndex: 0, sourceTimeSec: 0.2 + frame * 0.48 }, now); + setPlayhead(0.2 + (frame + 1) * 0.48, true); + } + + expect(native.setActiveClip).not.toHaveBeenCalled(); + }); + + // Two copies of one take: only the clip index tells them apart, and the scrub enters the left + // copy by its end. The view must be sent there, or it keeps the right copy's crop and zoom. + it("sends the left copy back when scrubbing into it while paused", async () => { + const left: AxcutClip = { + ...FIRST, + id: "clip_left", + sourceStartSec: 0, + sourceEndSec: 14.5, + timelineStartSec: 0, + timelineEndSec: 14.5, + }; + const right: AxcutClip = { + ...left, + id: "clip_right", + timelineStartSec: 14.5, + timelineEndSec: 29, + }; + const document = makeDocument(); + useProjectStore.setState({ + document: { ...document, timeline: { ...document.timeline, clips: [left, right] } }, + sourceDurationSec: 14.5, + currentTimeSec: 20, + playing: false, + }); + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 1, sourceTimeSec: 5.5 }, now); + + for (const time of [16, 15, 14.6, 14.4, 12, 9]) { + now += 16; + setPlayhead(time, false); + } + + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 0, 14.4, []); + expect(native.setActiveClip).toHaveBeenLastCalledWith(7, "/take.mp4", "", 0, 0, 14.4, []); + }); + + it("still sends the clip on a change while paused", async () => { + useProjectStore.setState({ playing: false }); + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.9 }, now); + + setPlayhead(6, false); + + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 1, 8, []); + }); + + // An addon that reports no position cannot be read, so it is driven as before. + it("pauses the view across the swap when it reports no position", async () => { + await mountAtFirstClip(); + + setPlayhead(5.02, true); + + expect(native.setNativePlaying).toHaveBeenCalledWith(false); + expect(native.setActiveClip).toHaveBeenCalledTimes(1); + }); +}); + +// The desk-view label is generated text inside the scene, so the scene is rebuilt when the +// user switches language — otherwise the preview keeps the old label until the next edit. +describe("NativeCompositorOverlay on a language change", () => { + beforeEach(() => { + vi.clearAllMocks(); + i18n.locale = "en"; + useProjectStore.setState({ + projectId: "proj_sync", + document: makeDocument(), + revision: 1, + status: "ready", + error: null, + sourceDurationSec: 12, + currentTimeSec: 1, + playing: false, + dirty: false, + lastSavedAt: new Date(), + }); + }); + + afterEach(() => { + cleanup(); + i18n.locale = "en"; + useProjectStore.getState().clear(); + }); + + it("pushes the scene again when the locale changes", async () => { + const { rerender } = render(); + await act(async () => { + await Promise.resolve(); + }); + native.setNativeScene.mockClear(); + + i18n.locale = "de"; + rerender(); + + expect(native.setNativeScene).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/components/ai-edition/NativeCompositorOverlay.tsx b/src/components/ai-edition/NativeCompositorOverlay.tsx index e4af64102..84a67ce39 100644 --- a/src/components/ai-edition/NativeCompositorOverlay.tsx +++ b/src/components/ai-edition/NativeCompositorOverlay.tsx @@ -1,9 +1,11 @@ import { useEffect, useMemo, useRef, useSyncExternalStore } from "react"; -import { useScopedT } from "@/contexts/I18nContext"; +import { useI18n, useScopedT } from "@/contexts/I18nContext"; +import { readSpeedRegions } from "@/lib/ai-edition/document/timeline"; import { noteUiProbeClipSwitch } from "@/lib/ai-edition/perf/uiFrameProbe"; import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; -import { assetCameraSource } from "@/lib/ai-edition/timeline/camera"; +import { assetAdditionalCameraSources, assetCameraSource } from "@/lib/ai-edition/timeline/camera"; +import { findActiveSpeedRegion, type SpeedRegion } from "@/lib/ai-edition/timeline/speed"; import { resolveNativePosition } from "@/lib/ai-edition/timeline/timelineMap"; import { pushAllNativeParams, @@ -15,6 +17,14 @@ import { useIsCpuCompositor, useNativeCompositorView, } from "@/native"; +import { + type DriftWatch, + getNativePosition, + IDLE_DRIFT_WATCH, + NATIVE_DRIFT_TOLERANCE_SEC, + nativeLeadSec, + watchDrift, +} from "@/native/nativeSync"; import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription"; import { getWebcamNativeSize, @@ -119,6 +129,8 @@ export function NativeCompositorOverlay() { sources: sources ?? undefined, }); const t = useScopedT("editor"); + // The scene carries translated text (the desk-view label): a language switch rebuilds it. + const { locale } = useI18n(); // No usable GPU: the preview still renders every effect, just slowly (~8 fps with // everything on). Nothing is disabled — the output stays identical to the GPU path — // so this is a notice, not a degradation warning. @@ -137,7 +149,7 @@ export function NativeCompositorOverlay() { // `_webcamSizeRevision` ci-dessus) : le layout preset et cie pilotent le rendu (remplace le // layout fixture). Effet APRÈS celui du viewId ci-dessus → currentViewId est déjà publié // quand on pousse. - // biome-ignore lint/correctness/useExhaustiveDependencies: size revision + // biome-ignore lint/correctness/useExhaustiveDependencies: size revision, locale useEffect(() => { if (viewId === null || !document) { return; @@ -153,8 +165,9 @@ export function NativeCompositorOverlay() { // re-trigger this effect when the probed-size cache mutates; the actual value is // re-read fresh via getWebcamNativeSize() above on every run (biome flags this as // an "unnecessary" dependency, but removing it would mean a probed webcam size - // arriving after mount never gets pushed to native). - }, [viewId, document, sources, _webcamSizeRevision]); + // arriving after mount never gets pushed to native). `locale` is the same kind of + // trigger: the label text is read inside `buildSceneDescription`. + }, [viewId, document, sources, _webcamSizeRevision, locale]); // SYNCHRO COMPLETE DES PARAMS, en un seul endroit. // @@ -183,12 +196,27 @@ export function NativeCompositorOverlay() { const activeClipId = activeClip?.id ?? null; const activeClipIndex = activePosition?.clipIndex ?? null; const activeSourceTimeSec = activePosition?.sourceTimeSec ?? null; + // Matched on the raw ruler, like `currentTimeSec` and the regions' pills. + const speedRegions = useMemo( + () => (document ? readSpeedRegions(document) : []), + [document], + ); + const playbackSpeed = findActiveSpeedRegion(speedRegions, currentTimeSec * 1000)?.speed ?? 1; const pendingTargetClipIdRef = useRef(null); const playing = useProjectStore((s) => s.playing); + const wasPlayingRef = useRef(playing); - // Change les décodeurs screen/webcam uniquement quand le playhead entre dans un autre clip. + // Change les décodeurs screen/webcam quand le playhead entre dans un autre clip, à l'arrêt. + // + // En lecture, la vue enchaîne seule les clips, préchargement compris. Lui renvoyer le clip à + // chaque coupe la faisait rechercher une position qu'elle avait déjà, ou jeter le + // préchargement qu'elle s'apprêtait à utiliser : un recul de quelques images et un arrêt à + // chaque coupe. Elle n'est recalée que si elle est réellement ailleurs (guet de dérive + // ci-dessous), d'après la position que porte chaque frame. useEffect(() => { + const justPaused = wasPlayingRef.current && !playing; + wasPlayingRef.current = playing; if ( viewId === null || !document || @@ -199,7 +227,13 @@ export function NativeCompositorOverlay() { ) { return; } - if (previousActiveClipIdRef.current === activeClipId) { + const native = getNativePosition(); + const clipChanged = previousActiveClipIdRef.current !== activeClipId; + // Mise en pause pendant que la vue était sur un autre clip que la tête de lecture : elle + // y retourne, sans quoi le recalage en temps (`setNativeTime`) chercherait dans le + // mauvais fichier. + const nativeElsewhere = justPaused && native !== null && native.clipIndex !== activeClipIndex; + if (!clipChanged && !nativeElsewhere) { return; } const asset = document.assets.find((candidate) => candidate.id === activeClip.assetId); @@ -210,13 +244,31 @@ export function NativeCompositorOverlay() { const targetClipId = activeClipId; // Sonde de fluidité (diagnostic) : sépare les mesures d'avant et d'après un // franchissement de clip, qui se sont déjà révélées non comparables. - noteUiProbeClipSwitch(previousActiveClipIdRef.current, activeClipId); + if (clipChanged) { + noteUiProbeClipSwitch(previousActiveClipIdRef.current, activeClipId); + } pendingTargetClipIdRef.current = targetClipId; previousActiveClipIdRef.current = targetClipId; + if (playing && native !== null) { + const lead = nativeLeadSec( + native, + { clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec }, + nativeClips, + playbackSpeed, + performance.now(), + ); + // Une coupe franchie en lecture : la vue la franchit d'elle-même, au même endroit du + // programme. Un saut (clic ailleurs sur la timeline) la met loin : suivi tout de suite, + // sans attendre le délai du guet de dérive. + if (lead !== null && Math.abs(lead) <= NATIVE_DRIFT_TOLERANCE_SEC) { + return; + } + } - // Pause native across the decoder swap. Deliberately NOT kept in a variable to - // resume from later — see the `.then` below, which re-reads the live transport. - if (playing) { + // Sans position rapportée (addon antérieur), la vue est encore pilotée comme avant : en + // pause pendant le changement de décodeurs. Deliberately NOT kept in a variable to resume + // from later — see the `.then` below, which re-reads the live transport. + if (playing && native === null) { setNativePlaying(false); } @@ -227,6 +279,7 @@ export function NativeCompositorOverlay() { camera.offsetSec, activeClipIndex, activeSourceTimeSec, + assetAdditionalCameraSources(asset), ) .then(() => { if (pendingTargetClipIdRef.current !== targetClipId) { @@ -248,7 +301,75 @@ export function NativeCompositorOverlay() { previousActiveClipIdRef.current = null; } }); - }, [viewId, document, activeClipId, activeClip, activeClipIndex, activeSourceTimeSec, playing]); + }, [ + viewId, + document, + activeClipId, + activeClip, + activeClipIndex, + activeSourceTimeSec, + playing, + nativeClips, + playbackSpeed, + ]); + + // Guet de dérive, en lecture : la vue tourne sur sa propre horloge, et n'est recalée que si + // la position que porte sa dernière frame s'écarte de la tête de lecture et le reste. Couvre + // un saut de l'utilisateur pendant la lecture comme un blocage du thread de rendu. L'ancien + // recalage devinait la dérive à l'horloge murale, à vitesse 1 : dans une région à 2× il + // relançait une recherche dix fois par seconde. + const driftRef = useRef(IDLE_DRIFT_WATCH); + useEffect(() => { + if ( + viewId === null || + !document || + !playing || + !activeClip || + activeClipIndex === null || + activeSourceTimeSec === null + ) { + driftRef.current = { ...driftRef.current, outSinceMs: null }; + return; + } + const now = performance.now(); + const lead = nativeLeadSec( + getNativePosition(), + { clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec }, + nativeClips, + playbackSpeed, + now, + ); + const { watch, resync } = watchDrift(driftRef.current, lead, now); + driftRef.current = watch; + if (!resync) { + return; + } + const asset = document.assets.find((candidate) => candidate.id === activeClip.assetId); + if (!asset?.originalPath) { + return; + } + const camera = assetCameraSource(asset); + setActiveClip( + viewId, + asset.originalPath, + camera.path, + camera.offsetSec, + activeClipIndex, + activeSourceTimeSec, + assetAdditionalCameraSources(asset), + ).catch((error: unknown) => { + console.warn("[compositor-view] re-anchoring the preview failed:", error); + }); + }, [ + viewId, + document, + playing, + activeClip, + activeClipIndex, + activeSourceTimeSec, + nativeClips, + playbackSpeed, + ]); if (!ready) { return null; diff --git a/src/components/ai-edition/NewEditorShell.chatOpen.test.tsx b/src/components/ai-edition/NewEditorShell.chatOpen.test.tsx index 27fb776f3..ead8fed69 100644 --- a/src/components/ai-edition/NewEditorShell.chatOpen.test.tsx +++ b/src/components/ai-edition/NewEditorShell.chatOpen.test.tsx @@ -2,6 +2,7 @@ import "@testing-library/jest-dom"; import { act, cleanup, fireEvent, render, screen } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { TooltipProvider } from "@/components/ui/tooltip"; import { EditorDialogsProvider } from "@/contexts/EditorDialogsContext"; import { useChatPromptBus } from "@/lib/ai-edition/store/useChatPromptBus"; @@ -48,9 +49,11 @@ import { NewEditorShell } from "./NewEditorShell"; function renderShell() { return render( - - - , + + + + + , ); } diff --git a/src/components/ai-edition/NewEditorShell.dialogShortcuts.test.tsx b/src/components/ai-edition/NewEditorShell.dialogShortcuts.test.tsx index 7c4201ead..10834f969 100644 --- a/src/components/ai-edition/NewEditorShell.dialogShortcuts.test.tsx +++ b/src/components/ai-edition/NewEditorShell.dialogShortcuts.test.tsx @@ -49,14 +49,17 @@ vi.mock("@/contexts/I18nContext", () => ({ useScopedT: () => (key: string) => key, })); +import { TooltipProvider } from "@/components/ui/tooltip"; import { EditorDialogsProvider } from "@/contexts/EditorDialogsContext"; import { NewEditorShell } from "./NewEditorShell"; function renderShell() { return render( - - - , + + + + + , ); } diff --git a/src/components/ai-edition/NewEditorShell.module.css b/src/components/ai-edition/NewEditorShell.module.css index 9b24797e6..162591937 100644 --- a/src/components/ai-edition/NewEditorShell.module.css +++ b/src/components/ai-edition/NewEditorShell.module.css @@ -361,6 +361,10 @@ overflow: hidden; display: grid; place-items: center; + /* Its own stacking context: the layers inside it (gimbals, zoom focus, webcam drag) order + themselves with z-indexes up to 1000 and more, which must not reach past the preview. A + selected annotation's gimbal painted over the export dialog (z-index 100). */ + isolation: isolate; } /* Once the native canvas has painted, it covers the whole card, and anything painted UNDER the same rounded clip leaks through its anti-aliased edge: the wallpaper placeholder and the inset @@ -722,7 +726,8 @@ position: relative; z-index: 1; /* Pixels come from the native canvas now; this