diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 493436c360..a3d80f71ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,11 +8,15 @@ name: CI # #169 — 30k+ lines of deletion and refactor — merged into release/1.8.0 with # every one of those jobs skipped, because a release branch matched neither # pattern. A release branch is the LAST place to run a build unguarded. +# +# integration/** for the same reason again: a feature that spans several PRs +# (integration/shader-effects) collects them there before it goes to main, and +# those PRs change native code for platforms their author cannot build locally. on: pull_request: - branches: [main, feat/ai-edition, "release/**"] + branches: [main, feat/ai-edition, "release/**", "integration/**"] push: - branches: [main, feat/ai-edition, "release/**"] + branches: [main, feat/ai-edition, "release/**", "integration/**"] jobs: lint: diff --git a/crates/Cargo.lock b/crates/Cargo.lock index da41d7070d..ad32f5dc25 100644 --- a/crates/Cargo.lock +++ b/crates/Cargo.lock @@ -994,6 +994,7 @@ dependencies = [ "cosmic-text", "image", "metal 0.29.0", + "naga", "ndarray 0.16.1", "objc", "ort", diff --git a/crates/Cargo.toml b/crates/Cargo.toml index a65f9aaef2..48a77d7756 100644 --- a/crates/Cargo.toml +++ b/crates/Cargo.toml @@ -60,6 +60,7 @@ version = "0.58" features = [ "Win32_Foundation", "Win32_Graphics_Direct3D", + "Win32_Graphics_Direct3D_Fxc", # D3DCompile à l'exécution des effets shader (shader_fx.rs) "Win32_Graphics_Direct3D11", "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs index ed98a71046..f1e4bd8fa6 100644 --- a/crates/compositor-view-napi/src/lib.rs +++ b/crates/compositor-view-napi/src/lib.rs @@ -388,6 +388,56 @@ pub fn set_active_clip( } } +/// Ce que l'outil de l'agent apprend d'un effet avant de l'accepter (`docs/shader-effects.md`, E.3). +#[napi(object)] +pub struct ShaderEffectCheck { + /// L'erreur telle que l'agent doit la lire (ligne, colonne, extrait), ou `None` s'il est accepté. + pub error: Option, + /// Le palier de coût : "light", "medium", "heavy" ou "very-heavy". Indicatif, jamais un refus. + pub tier: Option, + /// Lectures de `frame()` par pixel, boucles multipliées par leur borne. + pub frame_reads: Option, + /// Itérations de boucle par pixel. + pub iterations: Option, +} + +pub struct CheckShaderEffectTask { + code: String, + params_json: String, +} + +impl Task for CheckShaderEffectTask { + type Output = ShaderEffectCheck; + type JsValue = ShaderEffectCheck; + + fn compute(&mut self) -> Result { + Ok(match openscreen_compositor::shader_fx::check_for_agent_json(&self.code, &self.params_json) { + Ok(report) => ShaderEffectCheck { + error: None, + tier: Some(report.tier.name().to_string()), + frame_reads: Some(report.frame_reads as f64), + iterations: Some(report.iterations as f64), + }, + Err(error) => ShaderEffectCheck { error: Some(error), tier: None, frame_reads: None, iterations: None }, + }) + } + + fn resolve(&mut self, _env: Env, check: Self::Output) -> Result { + Ok(check) + } +} + +/// Vérifie le WGSL d'un effet shader : validation, bornes des boucles, compilation réelle pour le +/// backend de la machine (FXC sur Windows, Metal sur macOS, SPIR-V sur Linux), et palier de coût. +/// `params_json` : les paramètres déclarés, `[{ "name": "strength", "type": "number" }, …]`. +/// +/// `AsyncTask`, comme `segment_frame` : la compilation Metal d'une source neuve prend 120 ms, et +/// jusqu'à 375 ms pour la première du processus. Ce temps n'a rien à faire sur le thread principal. +#[napi] +pub fn check_shader_effect(code: String, params_json: String) -> AsyncTask { + AsyncTask::new(CheckShaderEffectTask { code, params_json }) +} + /// Installe la scène de l'app (JSON `SceneDescription`) sur la vue : layout preset piloté par /// l'app au lieu de la fixture. JSON invalide → ignoré côté natif. #[napi] @@ -614,21 +664,7 @@ impl Task for ExportMultiTask { // libopenh264 au lieu d'AMF, plutôt que d'échouer. let gpu = Gpu::create_auto(false).map_err(|e| Error::from_reason(format!("{e:#}")))?; self.control.check().map_err(mp4_task_error)?; - let mut cfg = config::all().pop().expect("au moins une config"); // C8 - cfg.zoom = false; - cfg.layout_anim = false; - cfg.mblur_n = 1; // layout statique → pas de motion blur de layout (pas de surcoût) - - // scène de l'app = même chemin que la preview live : fond, layout, webcam, curseur. - // JSON absent/invalide → pas de scène (fixture), pareil que si la preview n'en avait - // jamais reçu — jamais un fallback masquant, juste rien de configuré. - let scene = self.scene_json.as_deref().and_then(|j| Scene::from_json(j).ok()); - if let Some(scene) = &scene { - cfg.bg_blur = scene.effects.blur; - cfg.cursor = scene.cursor.show; - } else { - cfg.cursor = false; - } + let (cfg, scene) = export_cfg_and_scene(self.scene_json.as_deref()); let mut export_params = pipeline::ExportParams::default(); let (width, height) = resolve_output_size( @@ -713,6 +749,121 @@ fn mp4_task_error(error: anyhow::Error) -> Error { } } +/// La config et la scène d'un rendu d'export (MP4, GIF, frames) : la C8 en layout statique, et ce +/// que la scène de l'app règle. Scène = même chemin que la preview live : fond, layout, webcam, +/// curseur. JSON absent ou invalide → pas de scène (fixture), pareil que si la preview n'en avait +/// jamais reçu : jamais un fallback masquant, juste rien de configuré. +fn export_cfg_and_scene(scene_json: Option<&str>) -> (config::Cfg, Option) { + let mut cfg = config::all().pop().expect("au moins une config"); // C8 + cfg.zoom = false; + cfg.layout_anim = false; + cfg.mblur_n = 1; // layout statique → pas de motion blur de layout (pas de surcoût) + let scene = scene_json.and_then(|j| Scene::from_json(j).ok()); + match &scene { + Some(scene) => { + cfg.bg_blur = scene.effects.blur; + cfg.cursor = scene.cursor.show; + } + None => cfg.cursor = false, + } + (cfg, scene) +} + +fn clip_sources(clips: Vec) -> Vec { + clips + .into_iter() + .map(|c| pipeline::ClipSource { + screen: c.screen_path, + webcam: c.webcam_path, + source_start_sec: c.source_start_sec, + source_end_sec: c.source_end_sec, + webcam_offset_sec: c.webcam_offset_sec, + has_audio: c.has_audio, + }) + .collect() +} + +/// Une frame de `render_frames`. +#[napi(object)] +pub struct RenderedFrameOutput { + /// Son indice sur la grille de sortie. + pub index: u32, + pub width: u32, + pub height: u32, + /// JPEG, qualité 85. + pub jpeg: Buffer, +} + +pub struct RenderFramesTask { + clips: Vec, + scene_json: Option, + fps: u32, + frames: Vec, + max_width: u32, +} + +impl Task for RenderFramesTask { + type Output = Vec<(u32, u32, u32, Vec)>; + type JsValue = Vec; + + fn compute(&mut self) -> Result { + let reason = |e: anyhow::Error| Error::from_reason(format!("{e:#}")); + let gpu = Gpu::create_auto(false).map_err(reason)?; + let (cfg, scene) = export_cfg_and_scene(self.scene_json.as_deref()); + let default = pipeline::ExportParams::default(); + let (width, height) = resolve_output_size(scene.as_ref(), None, None, (default.width, default.height))?; + // Un compositeur neuf attend la compilation des effets shader, comme celui de l'export. + let comp = Compositor::new_sized(&gpu, width, height).map_err(reason)?; + if let Some(scene) = &scene { + comp.set_live_params(live_params_from_scene(scene)); + } + comp.set_scene(scene); + let wanted: Vec = self.frames.iter().map(|&f| u64::from(f)).collect(); + let frames = unsafe { + openscreen_compositor::timeline_walk::render_frames( + &self.clips, + &gpu, + &comp, + &cfg, + self.fps.max(1) as i32, + &wanted, + ) + } + .map_err(reason)?; + frames + .into_iter() + .map(|frame| { + let index = frame.index as u32; + let (w, h, jpeg) = frame.to_jpeg(self.max_width).map_err(reason)?; + Ok((index, w, h, jpeg)) + }) + .collect() + } + + fn resolve(&mut self, _env: Env, out: Self::Output) -> Result { + Ok(out + .into_iter() + .map(|(index, width, height, jpeg)| RenderedFrameOutput { index, width, height, jpeg: jpeg.into() }) + .collect()) + } +} + +/// Rend des frames de la timeline sans l'exporter : `getFrames` quand l'export manque ou date +/// (`docs/shader-effects.md`, E.4). Mêmes clips et même scène qu'`export_multi`, mêmes pixels : +/// la marche est celle de l'export (`timeline_walk::render_frames`), sans encodage. `frames` sont +/// des indices sur la grille `fps` ; ceux au-delà de la timeline ne rendent rien. Les images sont +/// en JPEG, réduites à `max_width` de large. +#[napi] +pub fn render_frames( + clips: Vec, + scene_json: Option, + fps: u32, + frames: Vec, + max_width: u32, +) -> AsyncTask { + AsyncTask::new(RenderFramesTask { clips: clip_sources(clips), scene_json, fps, frames, max_width }) +} + /// Lance un export multiclip natif (vraie timeline → MP4) et résout `Promise`. /// `scene_json` : même `SceneDescription` que la preview (fond/layout/webcam/effets/curseur). /// `params` : taille/cadence/codec de sortie voulus (absent → 1920x1080/fps du 1er clip/h264). @@ -729,17 +880,7 @@ pub fn export_multi( on_progress: Option, control: Option>, ) -> Result> { - let clips = clips - .into_iter() - .map(|c| pipeline::ClipSource { - screen: c.screen_path, - webcam: c.webcam_path, - source_start_sec: c.source_start_sec, - source_end_sec: c.source_end_sec, - webcam_offset_sec: c.webcam_offset_sec, - has_audio: c.has_audio, - }) - .collect(); + let clips = clip_sources(clips); Ok(AsyncTask::new(ExportMultiTask { out_path, clips, @@ -833,18 +974,7 @@ impl Task for ExportGifTask { // where the CPU backend exists specifically so the export still completes. let gpu = Gpu::create_auto(false).map_err(|e| Error::from_reason(format!("{e:#}")))?; self.control.check().map_err(gif_task_error)?; - let mut cfg = config::all().pop().expect("au moins une config"); // C8 - cfg.zoom = false; - cfg.layout_anim = false; - cfg.mblur_n = 1; - - let scene = self.scene_json.as_deref().and_then(|j| Scene::from_json(j).ok()); - if let Some(scene) = &scene { - cfg.bg_blur = scene.effects.blur; - cfg.cursor = scene.cursor.show; - } else { - cfg.cursor = false; - } + let (cfg, scene) = export_cfg_and_scene(self.scene_json.as_deref()); let (width, height) = resolve_output_size( scene.as_ref(), @@ -911,17 +1041,7 @@ pub fn export_gif( // Deliberately the same argument shape as `export_multi`: the caller builds // one clip list and one scene, and picks the container. Cursor comes from // the scene like every other effect — there is no GIF-specific input left. - let clips = clips - .into_iter() - .map(|c| pipeline::ClipSource { - screen: c.screen_path, - webcam: c.webcam_path, - source_start_sec: c.source_start_sec, - source_end_sec: c.source_end_sec, - webcam_offset_sec: c.webcam_offset_sec, - has_audio: c.has_audio, - }) - .collect(); + let clips = clip_sources(clips); let gif_params = params .map(|p| GifExportParams { width: p.width, diff --git a/crates/compositor/Cargo.toml b/crates/compositor/Cargo.toml index cc9ef64593..e33fa29323 100644 --- a/crates/compositor/Cargo.toml +++ b/crates/compositor/Cargo.toml @@ -37,6 +37,9 @@ ndarray = { workspace = true, optional = true } serde.workspace = true serde_json.workspace = true image.workspace = true +# Effets shader écrits à l'exécution (`shader_fx`) : la version que wgpu 24 tire déjà. Pas au-delà +# de 24 : dès la 25, la sortie HLSL ne passe plus FXC en SM 5.0 (voir `shader_fx::to_hlsl`). +naga = { version = "24", features = ["wgsl-in", "hlsl-out"] } # Acces Vulkan brut, UNIQUEMENT pour ouvrir le device avec les extensions de # memoire externe (cf. `d3d_linux::open_device_with_dmabuf_export`). Les versions @@ -50,6 +53,8 @@ image.workspace = true [target.'cfg(target_os = "linux")'.dependencies] ash = "0.38" wgpu-hal = { version = "24", features = ["vulkan"] } +# `shader_fx::check_for_agent` traduit l'effet en SPIR-V, comme wgpu le fera pour Vulkan. +naga = { version = "24", features = ["spv-out"] } # Windows : D3D11 + D3D11VA + Direct2D/DirectWrite + HLSL à l'exécution. [target.'cfg(windows)'.dependencies] @@ -77,6 +82,9 @@ metal = "0.29" objc = "0.2" block = "0.1" core-foundation = "0.9" +# Effets shader (`shader_fx::to_msl`) : naga vers MSL. wgpu l'active déjà par l'unification des +# features, mais rien ne le garantit : déclaré ici, à la même version (24) que plus haut. +naga = { version = "24", features = ["msl-out"] } # Linux : wgpu (Vulkan) pour le rendu, pollster pour block_on les ops wgpu, # cosmic-text pour la rastérisation du texte (remplace DirectWrite/CoreText). diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index e3176ad8c5..dc7567373e 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -47,7 +47,7 @@ use crate::frame_geometry::{ plan_frame, tilted_screen_cb, CursorPlacement, CursorPlanInput, FrameGeometryInput, ShadowCaster, SpriteShape, }; -use crate::scene::{Scene, SceneBackground, WallpaperMotion}; +use crate::scene::{Scene, SceneBackground, SceneShaderEffect, ShaderEffectTarget, WallpaperMotion}; const LAYER_WGSL: &str = include_str!("vk_shaders/layer.wgsl"); const BLUR_WGSL: &str = include_str!("vk_shaders/blur.wgsl"); @@ -116,6 +116,24 @@ struct BgCache { tex: wgpu::Texture, } +/// Un effet shader de la scène (`shader_fx`), par hash de sa source. +enum ShaderFx { + /// Module et pipeline se construisent sur un thread de fond (`spawn_shader_fx`). + Pending(std::sync::mpsc::Receiver>), + Ready(wgpu::RenderPipeline), + /// Erreur déjà écrite ; l'effet ne se dessine pas. + Failed, +} + +struct ShaderFxEntry { + state: ShaderFx, + /// Dernier usage, pour l'éviction (`SHADER_FX_CACHE`). + used: u64, +} + +/// Combien d'effets compilés le compositeur garde. Un agent qui itère en crée un par version. +const SHADER_FX_CACHE: usize = 64; + /// Une copie RT -> staging DEJA SOUMISE, dont le mapping est arme mais pas /// encore recolte. On garde `idx` (l'index de soumission rendu par /// `Queue::submit`) pour n'attendre QUE cette soumission-la, et les dimensions @@ -435,6 +453,20 @@ pub struct Compositor { seg_scratch: RefCell>, /// Le chargement du modele a echoue : ne pas reessayer a chaque frame. seg_failed: RefCell, + + /// Layout des effets shader : les quatre bindings du prélude (`shader_fx::PRELUDE_V1`). + shader_fx_bgl: wgpu::BindGroupLayout, + shader_fx_layout: wgpu::PipelineLayout, + /// Effets shader de la scène, compilés une fois par source, les moins récents évincés au-delà + /// de `SHADER_FX_CACHE`. + shader_fx: RefCell>, + shader_fx_tick: std::cell::Cell, + /// Vrai par défaut : une frame dont l'effet compile encore attend son pipeline, parce qu'un + /// export ne doit jamais rendre de frame sans l'effet. La preview le passe à faux et saute + /// l'effet le temps de la compilation, puis recompose (`shader_fx_landed`). + shader_fx_blocking: std::cell::Cell, + /// Le calque transparent des textes, alloué à la première frame qui porte un effet `text`. + text_layer: RefCell>, } impl Compositor { @@ -699,6 +731,40 @@ impl Compositor { multiview: None, cache: None, }); + // Effets shader (`shader_fx`) : un layout explicite, celui du prélude. Avec `layout: None`, + // un effet qui ne lit pas `ctx` perdrait ce binding, et son bind group ne validerait plus. + let fx_entry = |binding: u32, ty: wgpu::BindingType| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty, + count: None, + }; + let fx_uniform = wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }; + let shader_fx_bgl = gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("shader-fx"), + entries: &[ + fx_entry( + 0, + wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + ), + fx_entry(1, wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering)), + fx_entry(2, fx_uniform), + fx_entry(3, fx_uniform), + ], + }); + let shader_fx_layout = gpu.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("shader-fx"), + bind_group_layouts: &[&shader_fx_bgl], + push_constant_ranges: &[], + }); let (blur_half, blur_qtr, blur_oct, trail_view) = Self::make_blur_targets(&gpu, w, h); let (rt, rt_view, accum, accum_view, readback_bpr) = Self::make_targets(&gpu, w, h); let (ann_copy, ann_copy_view, ann_copy_mips) = Self::make_ann_copy(&gpu, w, h); @@ -772,6 +838,12 @@ impl Compositor { seg_rate: RefCell::new(crate::segmentation::RateLimiter::new(SEGMENTATION_HZ)), seg_scratch: RefCell::new(Vec::new()), seg_failed: RefCell::new(false), + shader_fx_bgl, + shader_fx_layout, + shader_fx: RefCell::new(std::collections::HashMap::new()), + shader_fx_tick: std::cell::Cell::new(0), + shader_fx_blocking: std::cell::Cell::new(true), + text_layer: RefCell::new(None), }) } @@ -819,6 +891,7 @@ impl Compositor { ann_copy_view, ann_copy_mips, bg_cache: RefCell::new(None), + text_layer: RefCell::new(None), ..self }) } @@ -1166,6 +1239,244 @@ impl Compositor { pub fn set_scene(&self, s: Option) { *self.scene.borrow_mut() = s; + // Les effets nouveaux partent compiler tout de suite, hors du thread de rendu : à la + // première frame qui les montre, ils sont en général déjà prêts. Après la pose de la + // scène : l'éviction épargne ce qu'elle montre. + if let Some(scene) = self.scene.borrow().as_ref() { + for effect in &scene.shader_effects { + self.shader_fx_entry(effect); + } + } + } + + /// Faux pour la preview : voir `shader_fx_blocking`. + pub fn set_shader_fx_blocking(&self, on: bool) { + self.shader_fx_blocking.set(on); + } + + /// Fait aboutir les compilations terminées. Vrai si un effet vient d'être prêt : la preview + /// en pause doit alors recomposer, sinon elle resterait sur la frame sans l'effet. + pub fn shader_fx_landed(&self) -> bool { + let mut landed = false; + for entry in self.shader_fx.borrow_mut().values_mut() { + let got = match &entry.state { + ShaderFx::Pending(rx) => rx.try_recv(), + _ => continue, + }; + match got { + Ok(result) => { + entry.state = Self::finish_shader_fx(result); + landed = true; + } + Err(std::sync::mpsc::TryRecvError::Empty) => {} + Err(std::sync::mpsc::TryRecvError::Disconnected) => entry.state = ShaderFx::Failed, + } + } + landed + } + + /// La clé de cache d'`effect`, sa compilation lancée s'il est neuf, et son dernier usage noté. + /// `None` pour un contrat que ce compositeur ne connaît pas : la région ne se dessine pas. + fn shader_fx_entry(&self, effect: &SceneShaderEffect) -> Option { + if effect.contract != crate::shader_fx::CONTRACT { + return None; + } + let key = crate::shader_fx::source_hash(effect); + let tick = self.shader_fx_tick.get() + 1; + self.shader_fx_tick.set(tick); + let mut fx = self.shader_fx.borrow_mut(); + fx.entry(key) + .or_insert_with(|| ShaderFxEntry { + state: ShaderFx::Pending(self.spawn_shader_fx(effect)), + used: tick, + }) + .used = tick; + // Un agent qui itère crée une entrée par version : on garde les plus récentes, et toutes + // celles que la scène montre. + if fx.len() > SHADER_FX_CACHE { + let scene = self.scene.borrow(); + let shown = scene.as_ref().map_or(&[][..], |s| &s.shader_effects[..]); + if let Some(oldest) = crate::shader_fx::eviction_victim(fx.iter().map(|(k, e)| (*k, e.used)), shown) { + fx.remove(&oldest); + } + } + Some(key) + } + + /// Construit le pipeline d'`effect` sur un thread de fond : `Device` est `Send + Sync`, et le + /// pilote compile le SPIR-V dans `create_render_pipeline`. naga valide le code avant wgpu : + /// aucun `on_uncaptured_error` n'est posé, et une erreur de validation wgpu fait paniquer. Si + /// wgpu panique malgré tout, seul ce thread meurt, et le canal fermé vaut un échec. + fn spawn_shader_fx( + &self, + effect: &SceneShaderEffect, + ) -> std::sync::mpsc::Receiver> { + let (tx, rx) = std::sync::mpsc::channel(); + let (device, layout) = (self.gpu.device.clone(), self.shader_fx_layout.clone()); + let (code, params) = (effect.code.clone(), effect.params.clone()); + std::thread::Builder::new() + .name("shader-fx".into()) + .spawn(move || { + let built = crate::shader_fx::analyze(&code, ¶ms).map(|_| { + let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("shader-fx"), + source: wgpu::ShaderSource::Wgsl( + crate::shader_fx::module_source(&code, ¶ms).into(), + ), + }); + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("shader-fx"), + layout: Some(&layout), + vertex: wgpu::VertexState { + module: &module, + entry_point: Some("fx_vs"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some("fx_fs"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }) + }); + let _ = tx.send(built); + }) + .expect("thread shader-fx"); + rx + } + + fn finish_shader_fx(result: Result) -> ShaderFx { + match result { + Ok(pipeline) => ShaderFx::Ready(pipeline), + Err(e) => { + eprintln!("[shader-fx] {e}"); + ShaderFx::Failed + } + } + } + + /// Le pipeline d'`effect`. Le compositeur d'export attend sa compilation ; la preview + /// (`shader_fx_blocking` faux) rend `None` tant qu'il compile. `None` aussi s'il a échoué. + fn shader_fx_ready(&self, effect: &SceneShaderEffect) -> Option { + let key = self.shader_fx_entry(effect)?; + let mut fx = self.shader_fx.borrow_mut(); + let entry = fx.get_mut(&key)?; + if let ShaderFx::Pending(rx) = &entry.state { + let got = if self.shader_fx_blocking.get() { + rx.recv().map_err(|e| e.to_string()) + } else { + match rx.try_recv() { + Ok(result) => Ok(result), + Err(std::sync::mpsc::TryRecvError::Empty) => return None, + Err(e) => Err(e.to_string()), + } + }; + entry.state = Self::finish_shader_fx(got.and_then(|result| result)); + } + match &entry.state { + ShaderFx::Ready(pipeline) => Some(pipeline.clone()), + _ => None, + } + } + + /// Une passe d'effet sur `dst` (le RT, ou le calque des textes) : `dst` est copié dans le + /// niveau 0 d'`ann_copy`, libre à ce stade (les flous et le cristal de Prism Glow l'ont déjà + /// lu, plus tôt dans l'encodeur), puis l'effet le repeint en entier. + fn encode_shader_effect( + &self, + encoder: &mut wgpu::CommandEncoder, + effect: &SceneShaderEffect, + dst: (&wgpu::Texture, &wgpu::TextureView), + ctx: &crate::shader_fx::FxContext, + ) { + let Some(pipeline) = self.shader_fx_ready(effect) else { return }; + let ctx_bytes = unsafe { + std::slice::from_raw_parts( + ctx as *const crate::shader_fx::FxContext as *const u8, + std::mem::size_of::(), + ) + }; + let params: Vec = + crate::shader_fx::pack_params(effect).iter().flat_map(|v| v.to_ne_bytes()).collect(); + let uniform = |label: &str, contents: &[u8]| { + self.gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some(label), + contents, + usage: wgpu::BufferUsages::UNIFORM, + }) + }; + let (ctx_buf, params_buf) = (uniform("shader-fx-ctx", ctx_bytes), uniform("shader-fx-params", ¶ms)); + // Les bindings du prélude : la frame (0), le sampler (1), `ctx` (2), `params` (3). + let bind = self.gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("shader-fx"), + layout: &self.shader_fx_bgl, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&self.ann_copy_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + wgpu::BindGroupEntry { binding: 2, resource: ctx_buf.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() }, + ], + }); + encoder.copy_texture_to_texture( + dst.0.as_image_copy(), + self.ann_copy.as_image_copy(), + wgpu::Extent3d { width: self.render_w, height: self.render_h, depth_or_array_layers: 1 }, + ); + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("shader-fx-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: dst.1, + resolve_target: None, + ops: wgpu::Operations { load: wgpu::LoadOp::Load, store: wgpu::StoreOp::Store }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&pipeline); + rpass.set_bind_group(0, &bind, &[]); + rpass.draw(0..3, 0..1); + } + + /// Le calque transparent des textes, à la taille de rendu, alloué à la première frame qui en a + /// besoin (un effet `ShaderEffectTarget::Text`). + fn text_layer(&self) -> (wgpu::Texture, wgpu::TextureView) { + if let Some(layer) = self.text_layer.borrow().as_ref() { + return layer.clone(); + } + let tex = self.gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("text-layer"), + size: wgpu::Extent3d { width: self.render_w, height: self.render_h, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + // COPY_SRC : un effet `text` le copie dans `ann_copy` avant de le repeindre. + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); + *self.text_layer.borrow_mut() = Some((tex.clone(), view.clone())); + (tex, view) } pub fn set_cursor(&self, track: crate::cursor::CursorTrack) { @@ -2919,25 +3230,28 @@ impl Compositor { /// Le cristal de Prism Glow : il lit la copie de l'image composee (`CursorPlan::glass`). glass: bool, } + // La pointe et les clics récents, tels que les effets shader les lisent : ceux de + // l'enregistrement, que le curseur soit dessiné ou non (`effect_pointer`). + let mut fx_pointer = crate::frame_geometry::EffectPointer::default(); let cursor_draw: Option = (|| { let track = cursor_ref.as_ref()?; - let plan = plan_cursor( - &g, - &CursorPlanInput { - render_px: [rw, rh], - u_max, - v_max, - cfg, - live: lp, - scene: scene_ref.as_ref(), - track, - t: self - .cursor_time - .borrow() - .unwrap_or(frame / crate::frame_geometry::FPS), - }, - )? - .for_backend(self.gpu.backend == crate::d3d::Backend::Cpu); + let input = CursorPlanInput { + render_px: [rw, rh], + u_max, + v_max, + cfg, + live: lp, + scene: scene_ref.as_ref(), + track, + t: self + .cursor_time + .borrow() + .unwrap_or(frame / crate::frame_geometry::FPS), + }; + if scene_ref.as_ref().is_some_and(|s| !s.shader_effects.is_empty()) { + fx_pointer = crate::frame_geometry::effect_pointer(&g, &input); + } + let plan = plan_cursor(&g, &input)?.for_backend(self.gpu.backend == crate::d3d::Backend::Cpu); // `taps == 1` : un seul placement, celui de l'instant rendu -- le // chemin net d'avant, inchange. Au-dela, on echelonne les copies @@ -3431,21 +3745,49 @@ impl Compositor { self.draw_layer(&mut rpass, layer, false); } } + // Effets shader de l'agent, autour des textes selon leur cible (`docs/shader-effects.md`, + // D.1) : `image` avant les annotations, `text` sur un calque des textes a part, `both` sur + // l'image finale. Meme ordre que `compositor_windows.rs`. + let fx_inputs = crate::shader_fx::FxFrameInputs { + cursor: fx_pointer.cursor, + clicks: fx_pointer.clicks, + footage: g.footage_quad([rw, rh]), + footage_aspect: g.footage_aspect([rw, rh]), + camera: lp.has_webcam.then_some(g.w_dst), + }; + let effects = scene_ref + .as_ref() + .map(|s| crate::shader_fx::active_effects(&s.shader_effects, g.source_t)) + .unwrap_or_default(); + let fx_context = |e: &SceneShaderEffect| { + crate::shader_fx::context(e, g.source_t, g.programme_t, [rw, rh], &fx_inputs) + }; + let with_target = |target: ShaderEffectTarget| -> Vec<&SceneShaderEffect> { + effects.iter().copied().filter(|e| e.target == target).collect() + }; + for e in with_target(ShaderEffectTarget::Image) { + self.encode_shader_effect(&mut encoder, e, (&self.rt, &self.rt_view), &fx_context(e)); + } + // Un effet `text` : les textes sur un calque transparent, l'effet sur ce calque seul, puis + // le calque pose sur l'image (« over » premultiplie). L'image dessous reste intacte. + let text_effects = with_target(ShaderEffectTarget::Text); + let text_layer = (!text_effects.is_empty()).then(|| self.text_layer()); // Passe 3 : les autres annotations, par-dessus tout le reste (les flous sont passes avant // le curseur). Elle existe meme sans annotation : deux passes consecutives sur la MEME // cible avec `LoadOp::Load` ne coutent rien de plus qu'une seule sur un GPU desktop, et un // seul chemin de code vaut mieux qu'un branchement qui ne serait exerce que dans un projet // sur dix. { + let (view, load) = match &text_layer { + Some((_, view)) => (view, wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT)), + None => (&self.rt_view, wgpu::LoadOp::Load), + }; let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("ann-pass"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: &self.rt_view, + view, resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Load, - store: wgpu::StoreOp::Store, - }, + ops: wgpu::Operations { load, store: wgpu::StoreOp::Store }, })], depth_stencil_attachment: None, timestamp_writes: None, @@ -3456,6 +3798,47 @@ impl Compositor { self.draw_layer(&mut rpass, &a.layer, false); } } + if let Some((tex, view)) = &text_layer { + for e in text_effects { + self.encode_shader_effect(&mut encoder, e, (tex, view), &fx_context(e)); + } + // Le calque pose sur le RT par la passe qui recopie `accum` : layout du blur, et un + // uniforme que `fs_copy` ne lit pas. + let uniform = self.gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("text-layer-uniform"), + contents: layer_bytes(&LayerCB::default()), + usage: wgpu::BufferUsages::UNIFORM, + }); + let bind = self.gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("text-layer"), + layout: &self.blur_bgl, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: uniform.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(view) }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + ], + }); + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("text-layer-composite"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { load: wgpu::LoadOp::Load, store: wgpu::StoreOp::Store }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline_copy); + rpass.set_bind_group(0, &bind, &[]); + rpass.draw(0..3, 0..1); + } + for e in with_target(ShaderEffectTarget::Both) { + self.encode_shader_effect(&mut encoder, e, (&self.rt, &self.rt_view), &fx_context(e)); + } self.gpu.context.submit(std::iter::once(encoder.finish())); Ok(()) } @@ -6543,6 +6926,216 @@ mod tests { assert!(n > 50, "trop peu de points visibles : {n}"); assert!(worst <= 0.25, "le rendu s'ecarte de la projection de {worst:.2} px"); } + + // ----------------------------------------------------------------------- + // Effets shader (`shader_fx`) : les tests de `compositor_windows.rs`, sur lavapipe. + // ----------------------------------------------------------------------- + + /// Une scène à fond rouge plein, l'écran rétréci par le padding : les coins sont du fond. + fn shader_fx_scene(effects: serde_json::Value) -> Scene { + let json = serde_json::json!({ + "clips": [], + "layout": {"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}, + "effects": {"padding":0.5,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}, + "background": {"kind":"color","color":"#ff0000"}, + "zoomRegions": [], + "cursor": {"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}, + "cropByClip": [], + "output": {"width":320,"height":180,"fps":null}, + "shaderEffects": effects, + }); + Scene::from_json(&json.to_string()).expect("scène") + } + + /// Un fragment d'effet de 1 s à 3 s, tel que l'app l'envoie. + fn fx(code: &str, target: &str) -> serde_json::Value { + serde_json::json!({ + "clipIndex": 0, "startSec": 1.0, "endSec": 3.0, "target": target, + "contract": 1, "code": code, + "params": [{"name": "level", "type": "number"}, {"name": "tint", "type": "color"}], + "values": {"level": 1.0, "tint": [0.0, 1.0, 0.0, 1.0]}, + }) + } + + const INVERT: &str = "fn effect(uv: vec2) -> vec4 {\n let c = frame(uv);\n return vec4(vec3(params.level) - c.rgb, 1.0);\n}\n"; + const BLUE: &str = "fn effect(uv: vec2) -> vec4 { return vec4(0.0, 0.0, 1.0, 1.0); }"; + const TINT: &str = "fn effect(uv: vec2) -> vec4 { return params.tint; }"; + const SAME: &str = "fn effect(uv: vec2) -> vec4 { return frame(uv); }"; + + fn fx_cfg() -> Cfg { + let mut cfg = Cfg::c8(); + cfg.bg_blur = 0.0; + cfg.mblur_n = 1; + cfg + } + + fn fx_compositor(gpu: &Gpu, w: u32, h: u32, effects: serde_json::Value) -> Compositor { + let comp = Compositor::new_sized(gpu, w, h).expect("compositeur"); + let scene = shader_fx_scene(effects); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + comp + } + + /// Le pixel du coin haut-gauche (du fond) après une frame composée au temps source `t`. + fn corner_at(comp: &Compositor, gpu: &Gpu, t: f32) -> [u8; 3] { + let screen = FakeFrame::new(gpu, 64, 64, |_, _| 126); + comp.set_timeline_time(Some(t)); + comp.set_programme_time(Some(t)); + unsafe { comp.compose_frame(screen.as_ptr(), std::ptr::null(), 0.0, &fx_cfg()) }.expect("compose"); + let (_, _, rgba) = unsafe { comp.readback_direct() }.expect("readback"); + [rgba[0], rgba[1], rgba[2]] + } + + /// Le coin à 2 s, dans la région, sous les effets donnés. + fn corner_under(effects: serde_json::Value) -> Option<[u8; 3]> { + let gpu = gpu()?; + Some(corner_at(&fx_compositor(&gpu, 320, 180, effects), &gpu, 2.0)) + } + + /// La passe de l'agent repeint l'image composée, dans sa région et nulle part ailleurs. Le + /// compositeur par défaut est celui de l'export : il attend le pipeline, jamais de frame sans. + #[test] + fn a_shader_effect_repaints_the_composed_frame_inside_its_region() { + let Some(gpu) = gpu() else { return }; + let comp = fx_compositor(&gpu, 320, 180, serde_json::json!([fx(INVERT, "image")])); + assert_eq!(corner_at(&comp, &gpu, 0.5), [255, 0, 0], "avant la région : le fond rouge"); + assert_eq!(corner_at(&comp, &gpu, 2.0), [0, 255, 255], "dans la région : rouge inversé"); + assert_eq!(corner_at(&comp, &gpu, 3.5), [255, 0, 0], "après la région : le fond rouge"); + } + + /// La preview ne bloque pas : elle saute l'effet tant qu'il compile, et `shader_fx_landed` + /// lui dit quand recomposer. Ensuite la frame est celle de l'export, au pixel. + #[test] + fn the_preview_shows_the_effect_once_it_landed() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + comp.set_shader_fx_blocking(false); + let scene = shader_fx_scene(serde_json::json!([fx(INVERT, "image")])); + comp.set_live_params(live_params_from_scene(&scene)); + let start = std::time::Instant::now(); + comp.set_scene(Some(scene)); + while !comp.shader_fx_landed() { + assert!(start.elapsed().as_secs() < 30, "la compilation n'a jamais abouti"); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + eprintln!("effet prêt en {:.1} ms", start.elapsed().as_secs_f64() * 1000.0); + assert_eq!(corner_at(&comp, &gpu, 2.0), [0, 255, 255]); + } + + /// Un WGSL invalide, une binding de plus ou un contrat inconnu ne cassent pas le rendu : + /// l'effet ne se dessine pas, la frame reste celle de la composition. Aucun n'atteint wgpu, + /// qui paniquerait sur une erreur de validation. + #[test] + fn a_broken_or_unknown_effect_leaves_the_frame_untouched() { + let broken = fx("fn effect(uv: vec2) -> vec4 { nope }", "image"); + let extra = fx( + "@group(0) @binding(4) var extra: vec4;\nfn effect(uv: vec2) -> vec4 { return extra; }", + "image", + ); + let mut future = fx(BLUE, "image"); + future["contract"] = serde_json::json!(2); + for effect in [broken, extra, future] { + let Some(corner) = corner_under(serde_json::json!([effect])) else { return }; + assert_eq!(corner, [255, 0, 0]); + } + } + + /// Les paramètres arrivent dans `params` au layout de la struct générée : la couleur `tint` + /// suit le nombre `level` dans le buffer, et se lit telle que l'app l'a écrite. + #[test] + fn a_color_parameter_reaches_the_shader() { + let Some(corner) = corner_under(serde_json::json!([fx(TINT, "image")])) else { return }; + assert_eq!(corner, [0, 255, 0]); + } + + /// La scène de ces tests masque le curseur (`cursor.show` faux) : rien n'est dessiné, mais un + /// effet voit toujours la pointe et le clic de l'enregistrement (`effect_pointer`). Il peint + /// `ctx.cursor.z` en rouge et `ctx.clicks[0].w` en vert : jaune, il a vu les deux. + #[test] + fn a_hidden_cursor_still_hands_the_effect_its_pointer_and_clicks() { + let Some(gpu) = gpu() else { return }; + const SEES: &str = "fn effect(uv: vec2) -> vec4 { return vec4(ctx.cursor.z, ctx.clicks[0].w, 0.0, 1.0); }"; + let comp = fx_compositor(&gpu, 320, 180, serde_json::json!([fx(SEES, "image")])); + comp.set_cursor(crate::cursor::CursorTrack::new_with_visibility( + vec![(0.0, 0.5, 0.5), (4.0, 0.5, 0.5)], + vec![1.5], + vec![], + vec![(0.0, true)], + )); + comp.set_cursor_time(Some(2.0)); + assert_eq!(corner_at(&comp, &gpu, 2.0), [255, 255, 0]); + } + + /// `text` ne touche que le calque des textes, posé ensuite sur l'image ; `image` et `both` + /// repeignent l'image. Sans annotation, le calque est transparent : un effet `text` qui le + /// rend tel quel laisse l'image intacte, un effet `text` opaque la recouvre. + #[test] + fn the_three_targets_keep_to_their_layer() { + let Some(image_only) = corner_under(serde_json::json!([fx(BLUE, "image")])) else { return }; + assert_eq!(image_only, [0, 0, 255]); + let text_untouched = corner_under(serde_json::json!([fx(SAME, "text")])).unwrap(); + assert_eq!(text_untouched, [255, 0, 0], "un calque transparent ne change pas l'image"); + let text_opaque = corner_under(serde_json::json!([fx(TINT, "text")])).unwrap(); + assert_eq!(text_opaque, [0, 255, 0], "le calque des textes, opaque, posé sur l'image"); + let both = corner_under(serde_json::json!([fx(INVERT, "both")])).unwrap(); + assert_eq!(both, [0, 255, 255], "`both` repeint l'image finale"); + } + + /// Calibre les paliers de coût : le surcoût par frame d'un effet qui lit `frame()` N fois, + /// à 1080p et en 4K, sur le GPU de la machine. Opt-in, comme les autres mesures : + /// + /// OPENSCREEN_FX_BENCH=1 cargo test -p openscreen-compositor --release --lib + /// shader_fx_tier_calibration -- --nocapture + /// + /// 200 frames par mesure ; `OPENSCREEN_FX_BENCH_FRAMES=5` sur lavapipe, où 512 lectures en 4K + /// prennent 5 s par frame. + #[test] + fn shader_fx_tier_calibration() { + if std::env::var_os("OPENSCREEN_FX_BENCH").is_none() { + eprintln!("OPENSCREEN_FX_BENCH absent — saute"); + return; + } + let gpu = Gpu::create_auto(false).expect("device Vulkan"); + eprintln!("backend : {:?}", gpu.backend); + let screen = FakeFrame::new(&gpu, 1920, 1080, |x, y| ((x ^ y) & 0xff) as u8); + let cfg = fx_cfg(); + let reads_effect = |n: u32| { + format!( + "fn effect(uv: vec2) -> vec4 {{\n var acc = vec4(0.0);\n for (var i = 0; i < {n}; i++) {{\n let a = f32(i) * 2.399;\n acc += frame(uv + vec2(cos(a), sin(a)) * fx_px(f32(i)));\n }}\n return acc / {n}.0;\n}}\n" + ) + }; + let frames: u32 = std::env::var("OPENSCREEN_FX_BENCH_FRAMES").ok().and_then(|v| v.parse().ok()).unwrap_or(200); + let per_frame = |w: u32, h: u32, effects: serde_json::Value| -> f64 { + let comp = fx_compositor(&gpu, w, h, effects); + comp.set_timeline_time(Some(2.0)); + comp.set_programme_time(Some(2.0)); + unsafe { comp.compose_frame(screen.as_ptr(), std::ptr::null(), 0.0, &cfg) }.unwrap(); + unsafe { comp.readback_direct() }.unwrap(); + let t0 = std::time::Instant::now(); + for f in 0..frames { + unsafe { comp.compose_frame(screen.as_ptr(), std::ptr::null(), f as f32, &cfg) }.unwrap(); + } + unsafe { comp.readback_direct() }.unwrap(); + t0.elapsed().as_secs_f64() * 1000.0 / frames as f64 + }; + for (w, h) in [(1920u32, 1080u32), (3840, 2160)] { + let base = (0..3).map(|_| per_frame(w, h, serde_json::json!([]))).fold(f64::MAX, f64::min); + eprintln!("{w}x{h} : sans effet {base:.3} ms/frame"); + for reads in [1u32, 8, 32, 128, 512] { + let code = reads_effect(reads); + let report = crate::shader_fx::analyze(&code, &[]).expect("analyse").report; + let with = (0..3) + .map(|_| per_frame(w, h, serde_json::json!([fx(&code, "image")]))) + .fold(f64::MAX, f64::min); + eprintln!( + "{w}x{h} : {reads:>3} lectures ({:>10}) : +{:.3} ms/frame", + report.tier.name(), + with - base + ); + } + } + } } // --------------------------------------------------------------------------- diff --git a/crates/compositor/src/compositor_macos.rs b/crates/compositor/src/compositor_macos.rs index 07ea2dd7c2..822d1d417b 100644 --- a/crates/compositor/src/compositor_macos.rs +++ b/crates/compositor/src/compositor_macos.rs @@ -37,7 +37,7 @@ pub use crate::frame_geometry::{ use crate::frame_geometry::{parse_hex, FrameGeometryInput, ShadowCaster, SCREEN_SHADOW_SPREAD_FRAC, WEBCAM_SHADOW_OFFSET_FRAC, WEBCAM_SHADOW_OPACITY, WEBCAM_SHADOW_SPREAD_FRAC}; -use crate::scene::{Scene, SceneBackground, WallpaperMotion}; +use crate::scene::{Scene, SceneBackground, SceneShaderEffect, ShaderEffectTarget, WallpaperMotion}; use anyhow::{anyhow, Result}; use metal::foreign_types::ForeignType; use std::cell::RefCell; @@ -361,6 +361,18 @@ pub struct Compositor { text_cache: RefCell>, text_raster: Option, + // --- Effets shader de l'agent (`shader_fx`, `docs/shader-effects.md`) --- + /// Compilés une fois par source, les moins récents évincés au-delà de `SHADER_FX_CACHE`. + shader_fx: RefCell>, + shader_fx_tick: std::cell::Cell, + /// Le calque transparent des textes, pour les effets qui ne visent qu'eux. Alloué à la + /// première frame qui porte un effet `text`, à la taille de rendu. + text_layer: RefCell>, + /// Vrai par défaut : une frame dont l'effet compile encore attend sa pipeline, parce qu'un + /// export ne doit jamais rendre de frame sans l'effet. La preview le passe à faux et saute + /// l'effet le temps de la compilation, puis recompose (`shader_fx_landed`). + shader_fx_blocking: std::cell::Cell, + // --- Segmentation du sujet webcam (cf. `pump_segmentation`) --- /// Masque du sujet, R8 à la résolution du modèle. Écrit par `set_webcam_mask`, lu au /// moment de dessiner la webcam. `None` tant qu'aucune frame n'a été segmentée — l'effet @@ -540,7 +552,7 @@ fn make_targets(device: &metal::Device, rw: u32, rh: u32) -> Targets { /// Comment un draw se mélange à ce qui est déjà dans la cible. #[derive(Clone, Copy, PartialEq)] -enum Blend { +pub(crate) enum Blend { /// Opaque : la conversion NV12 et le composite fullscreen écrasent. Replace, /// « over » alpha prémultiplié — la passe de composition normale. @@ -562,8 +574,28 @@ struct BgCache { tex: metal::Texture, } -/// Un pipeline state à une seule pièce jointe couleur. -fn make_pipeline( +/// Un effet shader de la scène (`shader_fx`), par hash de sa source. Cf. +/// `compositor_windows::ShaderFx`. +enum ShaderFx { + /// naga, le compilateur de Metal et le pipeline state tournent sur un thread de fond. + Pending(std::sync::mpsc::Receiver>), + Ready(metal::RenderPipelineState), + /// Erreur déjà écrite ; l'effet ne se dessine pas. + Failed, +} + +struct ShaderFxEntry { + state: ShaderFx, + /// Dernier usage, pour l'éviction (`SHADER_FX_CACHE`). + used: u64, +} + +/// Combien d'effets compilés le compositeur garde. Un agent qui itère en crée un par version. +const SHADER_FX_CACHE: usize = 64; + +/// Un pipeline state à une seule pièce jointe couleur. Aussi celui des effets shader +/// (`shader_fx::compile_metal`), construit sur leur thread de fond. +pub(crate) fn make_pipeline( device: &metal::Device, library: &metal::Library, vs: &str, @@ -764,6 +796,10 @@ impl Compositor { ann_img_cache: RefCell::new(std::collections::HashMap::new()), text_cache: RefCell::new(std::collections::HashMap::new()), text_raster: crate::text::TextRasterizer::new().ok(), + shader_fx: RefCell::new(std::collections::HashMap::new()), + shader_fx_tick: std::cell::Cell::new(0), + text_layer: RefCell::new(None), + shader_fx_blocking: std::cell::Cell::new(true), webcam_mask: RefCell::new(None), seg_capture: RefCell::new(None), seg_worker: RefCell::new(None), @@ -811,6 +847,7 @@ impl Compositor { blur_eighth, ann_copy, bg_cache: RefCell::new(None), + text_layer: RefCell::new(None), ..self }) } @@ -834,8 +871,155 @@ impl Compositor { self.live_params.borrow_mut().has_webcam = v; } + /// Installe (ou retire) la scène de l'app. Les effets shader nouveaux partent compiler tout de + /// suite, hors du thread de rendu : à la première frame qui les montre, ils sont en général + /// déjà prêts. Après la pose de la scène : l'éviction épargne ce qu'elle montre. pub fn set_scene(&self, s: Option) { *self.scene.borrow_mut() = s; + if let Some(scene) = self.scene.borrow().as_ref() { + for effect in &scene.shader_effects { + self.shader_fx_entry(effect); + } + } + } + + /// Faux pour la preview : voir `shader_fx_blocking`. + pub fn set_shader_fx_blocking(&self, on: bool) { + self.shader_fx_blocking.set(on); + } + + /// Fait aboutir les compilations terminées. Vrai si un effet vient d'être prêt : la preview + /// en pause doit alors recomposer, sinon elle resterait sur la frame sans l'effet. + pub fn shader_fx_landed(&self) -> bool { + let mut landed = false; + for entry in self.shader_fx.borrow_mut().values_mut() { + let got = match &entry.state { + ShaderFx::Pending(rx) => rx.try_recv(), + _ => continue, + }; + match got { + Ok(result) => { + entry.state = Self::finish_shader_fx(result); + landed = true; + } + Err(std::sync::mpsc::TryRecvError::Empty) => {} + Err(std::sync::mpsc::TryRecvError::Disconnected) => entry.state = ShaderFx::Failed, + } + } + landed + } + + /// La clé de cache d'`effect`, sa compilation lancée s'il est neuf, et son dernier usage noté. + /// `None` pour un contrat que ce compositeur ne connaît pas : la région ne se dessine pas. + fn shader_fx_entry(&self, effect: &SceneShaderEffect) -> Option { + if effect.contract != crate::shader_fx::CONTRACT { + return None; + } + let key = crate::shader_fx::source_hash(effect); + let tick = self.shader_fx_tick.get() + 1; + self.shader_fx_tick.set(tick); + let mut fx = self.shader_fx.borrow_mut(); + fx.entry(key) + .or_insert_with(|| ShaderFxEntry { + state: ShaderFx::Pending(crate::shader_fx::spawn_compile(effect, &self.gpu.device)), + used: tick, + }) + .used = tick; + // Un agent qui itère crée une entrée par version : on garde les plus récentes, et toutes + // celles que la scène montre. + if fx.len() > SHADER_FX_CACHE { + let scene = self.scene.borrow(); + let shown = scene.as_ref().map_or(&[][..], |s| &s.shader_effects[..]); + if let Some(oldest) = crate::shader_fx::eviction_victim(fx.iter().map(|(k, e)| (*k, e.used)), shown) { + fx.remove(&oldest); + } + } + Some(key) + } + + /// Sur Metal, le thread de fond rend la pipeline elle-même : il ne reste qu'à noter l'erreur. + fn finish_shader_fx(result: Result) -> ShaderFx { + match result { + Ok(pipeline) => ShaderFx::Ready(pipeline), + Err(e) => { + eprintln!("[shader-fx] {e}"); + ShaderFx::Failed + } + } + } + + /// La pipeline d'`effect`. Le compositeur d'export attend sa compilation ; la preview + /// (`shader_fx_blocking` faux) rend `None` tant qu'elle compile. `None` aussi si elle a échoué. + fn shader_fx_ready(&self, effect: &SceneShaderEffect) -> Option { + let key = self.shader_fx_entry(effect)?; + let mut fx = self.shader_fx.borrow_mut(); + let entry = fx.get_mut(&key)?; + if let ShaderFx::Pending(rx) = &entry.state { + let got = if self.shader_fx_blocking.get() { + rx.recv().map_err(|e| e.to_string()) + } else { + match rx.try_recv() { + Ok(result) => Ok(result), + Err(std::sync::mpsc::TryRecvError::Empty) => return None, + Err(e) => Err(e.to_string()), + } + }; + entry.state = Self::finish_shader_fx(got.and_then(|r| r)); + } + match &entry.state { + ShaderFx::Ready(pipeline) => Some(pipeline.clone()), + _ => None, + } + } + + /// Une passe d'effet sur `dst` (le RT, ou le calque des textes), dans le command buffer de la + /// frame : `dst` est copié dans `ann_copy`, libre à ce stade, puis l'effet le repeint en entier + /// dans une passe qui CHARGE la cible. + unsafe fn apply_shader_effect( + &self, + cmd: &metal::CommandBufferRef, + effect: &SceneShaderEffect, + dst: &metal::Texture, + ctx: &crate::shader_fx::FxContext, + ) -> Result<()> { + let Some(pipeline) = self.shader_fx_ready(effect) else { return Ok(()) }; + // On ne lit pas la cible sur laquelle on dessine : copie dans `ann_copy` (les flous + // d'annotation et le cristal de Prism Glow l'ont déjà lue). + self.snapshot(cmd, dst, false); + let params = crate::shader_fx::pack_params(effect); + let e = self.begin_pass(cmd, dst, None, &pipeline)?; + e.set_fragment_texture(0, Some(&self.ann_copy)); + e.set_fragment_bytes( + crate::shader_fx::MSL_CTX_BUFFER, + std::mem::size_of::() as u64, + ctx as *const crate::shader_fx::FxContext as *const std::ffi::c_void, + ); + e.set_fragment_bytes( + crate::shader_fx::MSL_PARAMS_BUFFER, + (params.len() * 4) as u64, + params.as_ptr() as *const std::ffi::c_void, + ); + e.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 3); + e.end_encoding(); + Ok(()) + } + + /// Le calque transparent des textes, à la taille de rendu, alloué à la première frame qui en a + /// besoin (un effet `ShaderEffectTarget::Text`). + fn text_layer(&self) -> metal::Texture { + self.text_layer + .borrow_mut() + .get_or_insert_with(|| { + make_texture( + &self.gpu.device, + metal::MTLPixelFormat::RGBA8Unorm, + self.render_w, + self.render_h, + metal::MTLStorageMode::Private, + metal::MTLTextureUsage::RenderTarget | metal::MTLTextureUsage::ShaderRead, + ) + }) + .clone() } pub fn set_cursor(&self, track: crate::cursor::CursorTrack) { @@ -1497,9 +1681,14 @@ impl Compositor { /// Fige l'image composée dans `ann_copy` (mip 0), qu'y lisent les flous (mips dérivés, /// `mips`) et le cristal de Prism Glow. unsafe fn snapshot_rt(&self, cmd: &metal::CommandBufferRef, mips: bool) { + self.snapshot(cmd, &self.rt, mips); + } + + /// Copie `src` (le RT, ou le calque des textes des effets shader) dans `ann_copy` (mip 0). + unsafe fn snapshot(&self, cmd: &metal::CommandBufferRef, src: &metal::Texture, mips: bool) { let blit = cmd.new_blit_command_encoder(); blit.copy_from_texture( - &self.rt, 0, 0, + src, 0, 0, metal::MTLOrigin { x: 0, y: 0, z: 0 }, metal::MTLSize { width: self.render_w as u64, height: self.render_h as u64, depth: 1 }, &self.ann_copy, 0, 0, @@ -1522,9 +1711,14 @@ impl Compositor { /// /// `privacy` : les flous de confidentialité seuls, dessinés sur le métrage avant le curseur ; /// sinon toutes les autres annotations, le calque le plus haut. + /// + /// `target` : le RT, ou le calque transparent des textes quand un effet shader `text` les vise + /// (`compose_frame`). Les flous de confidentialité visent toujours le RT, qu'ils relisent. + #[allow(clippy::too_many_arguments)] unsafe fn draw_annotations( &self, cmd: &metal::CommandBufferRef, + target: &metal::Texture, scene: Option<&Scene>, t: f32, s_ann: [f32; 4], @@ -1546,7 +1740,7 @@ impl Compositor { self.snapshot_rt(cmd, true); } - let enc = self.begin_pass(cmd, &self.rt, None, &self.pipeline_main)?; + let enc = self.begin_pass(cmd, target, None, &self.pipeline_main)?; // La liste arrive déjà triée par zIndex côté app : l'ordre d'itération EST l'ordre // de peinture. for a in &scene.annotations { @@ -2548,24 +2742,29 @@ impl Compositor { // --- flous de confidentialité : sur le métrage, AVANT le curseur (parité Windows). Le // curseur reste net par-dessus, et le cristal de Prism Glow, qui réfracte l'image // composée, n'y voit que des pixels déjà floutés. - self.draw_annotations(cmd_buf, scene_ref.as_ref(), g.source_t, g.s_ann, &g, true)?; + self.draw_annotations(cmd_buf, &self.rt, scene_ref.as_ref(), g.source_t, g.s_ann, &g, true)?; + + // La pointe et les clics récents, tels que les effets shader les lisent : ceux de + // l'enregistrement, que le curseur soit dessiné ou non (`effect_pointer`). + let mut fx_pointer = crate::frame_geometry::EffectPointer::default(); // --- curseur --- (parité `compositor_windows.rs`, section « curseur custom ») if let Some(track) = cursor_ref.as_ref() { - let plan = crate::frame_geometry::plan_cursor( - &g, - &crate::frame_geometry::CursorPlanInput { - render_px: [rw, rh], - u_max, - v_max, - cfg, - live: lp, - scene: scene_ref.as_ref(), - track, - t: self.cursor_time.borrow().unwrap_or(frame / crate::frame_geometry::FPS), - }, - ) - .map(|p| p.for_backend(self.gpu.backend == crate::d3d::Backend::Cpu)); + let input = crate::frame_geometry::CursorPlanInput { + render_px: [rw, rh], + u_max, + v_max, + cfg, + live: lp, + scene: scene_ref.as_ref(), + track, + t: self.cursor_time.borrow().unwrap_or(frame / crate::frame_geometry::FPS), + }; + if scene_ref.as_ref().is_some_and(|s| !s.shader_effects.is_empty()) { + fx_pointer = crate::frame_geometry::effect_pointer(&g, &input); + } + let plan = crate::frame_geometry::plan_cursor(&g, &input) + .map(|p| p.for_backend(self.gpu.backend == crate::d3d::Backend::Cpu)); if let Some(plan) = plan { let sprites = scene_ref .as_ref() @@ -2764,12 +2963,64 @@ impl Compositor { enc.end_encoding(); + // --- effets shader de l'agent, autour des textes selon leur cible (`docs/shader-effects.md`, + // D.1) : `image` avant les annotations et les sous-titres, `text` sur un calque des textes + // à part, `both` sur l'image finale. Tout dans le command buffer de la frame. + let fx_inputs = crate::shader_fx::FxFrameInputs { + cursor: fx_pointer.cursor, + clicks: fx_pointer.clicks, + footage: g.footage_quad([rw, rh]), + footage_aspect: g.footage_aspect([rw, rh]), + camera: lp.has_webcam.then_some(g.w_dst), + }; + let effects = scene_ref + .as_ref() + .map(|s| crate::shader_fx::active_effects(&s.shader_effects, g.source_t)) + .unwrap_or_default(); + let fx_context = |e: &SceneShaderEffect| { + crate::shader_fx::context(e, g.source_t, g.programme_t, [rw, rh], &fx_inputs) + }; + let with_target = |target: ShaderEffectTarget| -> Vec<&SceneShaderEffect> { + effects.iter().copied().filter(|e| e.target == target).collect() + }; + for e in with_target(ShaderEffectTarget::Image) { + self.apply_shader_effect(cmd_buf, e, &self.rt, &fx_context(e))?; + } + // --- annotations : calque le plus haut, ancré sur le rect ÉCRAN SANS ZOOM --- // `s_ann`, pas `s_dst` : le zoom vit dans la boîte depuis l'issue #179, donc `s_dst` // grandit avec lui et emmenait annotations et sous-titres dans le mouvement. Le flou de // confidentialité est l'exception : il suit le contenu, d'où `&g` ; il est passé avant le // curseur (plus haut). - self.draw_annotations(cmd_buf, scene_ref.as_ref(), g.source_t, g.s_ann, &g, false)?; + let text_effects = with_target(ShaderEffectTarget::Text); + if text_effects.is_empty() { + self.draw_annotations(cmd_buf, &self.rt, scene_ref.as_ref(), g.source_t, g.s_ann, &g, false)?; + } else { + // Les textes sur un calque transparent, l'effet sur ce calque seul, puis le calque + // posé sur l'image (« over » prémultiplié, `pipeline_fs_tex`) : l'image dessous reste + // intacte. Le calque est effacé dans sa propre passe : `draw_annotations` n'en ouvre + // aucune quand rien n'est visible. + let layer = self.text_layer(); + self.begin_pass( + cmd_buf, + &layer, + Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 0.0)), + &self.pipeline_main, + )? + .end_encoding(); + self.draw_annotations(cmd_buf, &layer, scene_ref.as_ref(), g.source_t, g.s_ann, &g, false)?; + for e in text_effects { + self.apply_shader_effect(cmd_buf, e, &layer, &fx_context(e))?; + } + let c = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_fs_tex)?; + c.set_fragment_texture(0, Some(&layer)); + c.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 3); + c.end_encoding(); + } + + for e in with_target(ShaderEffectTarget::Both) { + self.apply_shader_effect(cmd_buf, e, &self.rt, &fx_context(e))?; + } // Ni miroir RGBA ni attente ici : le miroir ne sert qu'à `readback_direct` (la // preview), et l'export ne lit jamais le RGBA — le blit pleine résolution était payé @@ -4519,4 +4770,358 @@ mod tests { let (rw, rh, rgba) = unsafe { comp.readback_direct() }.expect("readback"); assert_eq!((rw, rh, rgba.len()), (182, 322, 182 * 322 * 4)); } + + // ----------------------------------------------------------------------- + // Effets shader de l'agent (`shader_fx`) : le pendant des tests GPU de + // `compositor_windows.rs`, sur le device Metal du système. + // ----------------------------------------------------------------------- + + /// Une scène à fond rouge plein, l'écran rétréci par le padding : les coins sont du fond. + /// `annotations` : celles de la scène, pour les tests de cible. + fn shader_fx_scene_with(effects: serde_json::Value, annotations: serde_json::Value) -> Scene { + Scene::from_json(&shader_fx_json(effects, annotations).to_string()).expect("scène") + } + + fn shader_fx_json(effects: serde_json::Value, annotations: serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "clips": [], + "layout": {"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}, + "effects": {"padding":0.5,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}, + "background": {"kind":"color","color":"#ff0000"}, + "zoomRegions": [], + "annotations": annotations, + "cursor": {"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}, + "cropByClip": [], + "output": {"width":320,"height":180,"fps":null}, + "shaderEffects": effects, + }) + } + + fn shader_fx_scene(effects: serde_json::Value) -> Scene { + shader_fx_scene_with(effects, serde_json::json!([])) + } + + /// Un fragment d'effet de 1 s à 3 s, tel que l'app l'envoie. + fn fx(code: &str, target: &str) -> serde_json::Value { + serde_json::json!({ + "clipIndex": 0, "startSec": 1.0, "endSec": 3.0, "target": target, + "contract": 1, "code": code, + "params": [{"name": "level", "type": "number"}, {"name": "tint", "type": "color"}], + "values": {"level": 1.0, "tint": [0.0, 1.0, 0.0, 1.0]}, + }) + } + + const INVERT: &str = "fn effect(uv: vec2) -> vec4 {\n let c = frame(uv);\n return vec4(vec3(params.level) - c.rgb, 1.0);\n}\n"; + const BLUE: &str = "fn effect(uv: vec2) -> vec4 { return vec4(0.0, 0.0, 1.0, 1.0); }"; + const TINT: &str = "fn effect(uv: vec2) -> vec4 { return params.tint; }"; + const SAME: &str = "fn effect(uv: vec2) -> vec4 { return frame(uv); }"; + + fn fx_cfg() -> crate::config::Cfg { + let mut cfg = crate::config::Cfg::c8(); + cfg.bg_blur = 0.0; + cfg.mblur_n = 1; + cfg + } + + /// L'image composée au temps source `t`, en RGBA. + fn compose_fx_at(comp: &super::Compositor, t: f32) -> Vec { + let screen = FakeFrame::new(64, 64, |_, _| 126); + comp.set_timeline_time(Some(t)); + comp.set_programme_time(Some(t)); + unsafe { comp.compose_frame(screen.as_ptr(), screen.as_ptr(), 0.0, &fx_cfg()) }.expect("compose"); + let (_, _, rgba) = unsafe { comp.readback_direct() }.expect("readback"); + rgba + } + + /// Le pixel du coin haut-gauche (du fond) au temps source `t`. + fn corner_at(comp: &super::Compositor, t: f32) -> [u8; 3] { + let rgba = compose_fx_at(comp, t); + [rgba[0], rgba[1], rgba[2]] + } + + /// Le coin à 2 s, dans la région, sous les effets donnés. + fn corner_under(effects: serde_json::Value) -> Option<[u8; 3]> { + let gpu = crate::d3d::Gpu::create(false).ok()?; + let comp = super::Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + comp.set_scene(Some(shader_fx_scene(effects))); + Some(corner_at(&comp, 2.0)) + } + + /// La passe de l'agent repeint l'image composée, dans sa région et nulle part ailleurs. Le + /// compositeur par défaut est celui de l'export : il attend la pipeline, jamais de frame sans. + #[test] + fn a_shader_effect_repaints_the_composed_frame_inside_its_region() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let comp = super::Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + comp.set_scene(Some(shader_fx_scene(serde_json::json!([fx(INVERT, "image")])))); + assert_eq!(corner_at(&comp, 0.5), [255, 0, 0], "avant la région : le fond rouge"); + assert_eq!(corner_at(&comp, 2.0), [0, 255, 255], "dans la région : rouge inversé"); + assert_eq!(corner_at(&comp, 3.5), [255, 0, 0], "après la région : le fond rouge"); + } + + /// La preview ne bloque pas : elle saute l'effet tant qu'il compile, et `shader_fx_landed` + /// lui dit quand recomposer. Ensuite la frame est celle de l'export, au pixel. + #[test] + fn the_preview_shows_the_effect_once_it_landed() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let comp = super::Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + comp.set_shader_fx_blocking(false); + comp.set_scene(Some(shader_fx_scene(serde_json::json!([fx(INVERT, "image")])))); + let start = std::time::Instant::now(); + while !comp.shader_fx_landed() { + assert!(start.elapsed().as_secs() < 10, "la compilation n'a jamais abouti"); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + eprintln!("effet prêt en {:.1} ms", start.elapsed().as_secs_f64() * 1000.0); + assert_eq!(corner_at(&comp, 2.0), [0, 255, 255]); + } + + /// Un WGSL invalide, ou un contrat inconnu, ne casse pas le rendu : l'effet ne se dessine + /// pas, la frame reste celle de la composition. + #[test] + fn a_broken_or_unknown_effect_leaves_the_frame_untouched() { + let broken = fx("fn effect(uv: vec2) -> vec4 { nope }", "image"); + let mut future = fx(BLUE, "image"); + future["contract"] = serde_json::json!(2); + for effect in [broken, future] { + let Some(corner) = corner_under(serde_json::json!([effect])) else { return }; + assert_eq!(corner, [255, 0, 0]); + } + } + + /// Les paramètres arrivent dans `params` au layout de la struct générée : la couleur `tint` + /// suit le nombre `level` dans le buffer, et se lit telle que l'app l'a écrite. + #[test] + fn a_color_parameter_reaches_the_shader() { + let Some(corner) = corner_under(serde_json::json!([fx(TINT, "image")])) else { return }; + assert_eq!(corner, [0, 255, 0]); + } + + /// `fx_to_footage` retrouve le point du métrage que le compositeur a dessiné sous un pixel, + /// à plat, sous un angle fixe (warp bilinéaire) et sous la caméra réelle (homographie), et + /// `fx_from_footage` le ramène sur ce pixel. En 1080p, l'arrondi 8 bits et le demi-pixel + /// tiennent sous 0,005 ; la pose orbit testée écarte l'homographie du bilinéaire de ~0,01. + /// Le pendant du test Windows : les deux fonctions passent ici par la traduction MSL. + #[test] + fn the_footage_plane_maps_back_to_what_the_compositor_drew() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + const PLANE: &str = "fn effect(uv: vec2) -> vec4 {\n let p = fx_to_footage(uv);\n return vec4(p, distance(fx_from_footage(p), uv) * 100.0, 1.0);\n}\n"; + for (rotation, projective) in [(None, false), (Some("left"), false), (Some("orbit"), true)] { + let mut json = shader_fx_json(serde_json::json!([fx(PLANE, "image")]), serde_json::json!([])); + json["clips"] = serde_json::json!([{"screenPath": "/s.mp4", "webcamPath": "", "sourceStartSec": 0, + "sourceEndSec": 10, "webcamOffsetSec": 0, "hasAudio": false}]); + json["layout"]["screenRect"] = serde_json::json!({"x": 0.1, "y": 0.1, "width": 0.8, "height": 0.8}); + json["cropByClip"] = serde_json::json!([null]); + if let Some(rotation) = rotation { + json["zoomRegions"] = serde_json::json!([{"clipIndex": 0, "startSec": 0, "endSec": 10, + "scale": 1.5, "focusX": 0.5, "focusY": 0.5, "rotation": rotation}]); + } + let comp = super::Compositor::new_sized(&gpu, 1920, 1080).expect("compositeur"); + let scene = Scene::from_json(&json.to_string()).expect("scène"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + let screen = FakeFrame::new(64, 64, |_, _| 126); + comp.set_timeline_time(Some(2.0)); + comp.set_programme_time(Some(2.0)); + unsafe { comp.compose_frame(screen.as_ptr(), screen.as_ptr(), 0.0, &fx_cfg()) }.expect("compose"); + let quad = comp.footage_quad().expect("le métrage de la frame"); + assert_eq!(quad.projective, projective, "{rotation:?}"); + let (w, h, rgba) = unsafe { comp.readback_direct() }.expect("readback"); + let [tl, tr, br, bl] = quad.corners.map(|[x, y]| (x, y)); + for (s, t) in [(0.25, 0.3), (0.7, 0.4), (0.4, 0.75)] { + let (x, y) = if projective { + crate::regions::square_to_quad(&[tl, tr, br, bl], s, t) + } else { + let top = (tl.0 + (tr.0 - tl.0) * s, tl.1 + (tr.1 - tl.1) * s); + let bottom = (bl.0 + (br.0 - bl.0) * s, bl.1 + (br.1 - bl.1) * s); + (top.0 + (bottom.0 - top.0) * t, top.1 + (bottom.1 - top.1) * t) + }; + let i = ((y * h as f32) as usize * w as usize + (x * w as f32) as usize) * 4; + let got = (rgba[i] as f32 / 255.0, rgba[i + 1] as f32 / 255.0); + assert!( + (got.0 - s).abs() < 0.005 && (got.1 - t).abs() < 0.005, + "{rotation:?} ({s}, {t}) : lu {got:?}, coins {:?}", + quad.corners + ); + assert!(rgba[i + 2] < 3, "{rotation:?} ({s}, {t}) : aller-retour à {}", rgba[i + 2]); + } + } + } + + /// La scène de ces tests masque le curseur (`cursor.show` faux) : rien n'est dessiné, mais un + /// effet voit toujours la pointe et le clic de l'enregistrement (`effect_pointer`). Il peint + /// `ctx.cursor.z` en rouge et `ctx.clicks[0].w` en vert : jaune, il a vu les deux. + #[test] + fn a_hidden_cursor_still_hands_the_effect_its_pointer_and_clicks() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + const SEES: &str = "fn effect(uv: vec2) -> vec4 { return vec4(ctx.cursor.z, ctx.clicks[0].w, 0.0, 1.0); }"; + let comp = super::Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + comp.set_scene(Some(shader_fx_scene(serde_json::json!([fx(SEES, "image")])))); + comp.set_cursor(crate::cursor::CursorTrack::new_with_visibility( + vec![(0.0, 0.5, 0.5), (4.0, 0.5, 0.5)], + vec![1.5], + vec![], + vec![(0.0, true)], + )); + comp.set_cursor_time(Some(2.0)); + assert_eq!(corner_at(&comp, 2.0), [255, 255, 0]); + } + + /// `text` ne touche que le calque des textes, posé ensuite sur l'image ; `image` et `both` + /// repeignent l'image. Sans annotation, le calque est transparent : un effet `text` qui le + /// rend tel quel laisse l'image intacte, un effet `text` opaque la recouvre. + #[test] + fn the_three_targets_keep_to_their_layer() { + let Some(image_only) = corner_under(serde_json::json!([fx(BLUE, "image")])) else { return }; + assert_eq!(image_only, [0, 0, 255]); + let text_untouched = corner_under(serde_json::json!([fx(SAME, "text")])).unwrap(); + assert_eq!(text_untouched, [255, 0, 0], "un calque transparent ne change pas l'image"); + let text_opaque = corner_under(serde_json::json!([fx(TINT, "text")])).unwrap(); + assert_eq!(text_opaque, [0, 255, 0], "le calque des textes, opaque, posé sur l'image"); + let both = corner_under(serde_json::json!([fx(INVERT, "both")])).unwrap(); + assert_eq!(both, [0, 255, 255], "`both` repeint l'image finale"); + } + + /// Avec une vraie annotation : un effet `image` passe dessous sans la toucher, un effet `text` + /// la repeint sans toucher l'image. La flèche blanche est dessinée dans le calque des textes, + /// pas dans le RT : c'est ce que `draw_annotations(…, target, …)` doit garantir. + #[test] + fn an_annotation_stays_out_of_image_effects_and_inside_text_effects() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let arrow = serde_json::json!([{ + "id": "a", "startSec": 0.0, "endSec": 10.0, "kind": "figure", "space": "frame", + "x": 0.1, "y": 0.1, "w": 0.8, "h": 0.8, "zIndex": 0, + "figure": {"direction": "right", "color": "#ffffff", "strokeWidth": 30}, + }]); + // Le calque des textes est prémultiplié : l'effet garde l'alpha et peint en vert ce qui + // est couvert. + const GREEN_TEXT: &str = "fn effect(uv: vec2) -> vec4 { let a = frame(uv).a; return vec4(0.0, a, 0.0, a); }"; + let count = |rgba: &[u8], rgb: [u8; 3]| rgba.chunks_exact(4).filter(|p| p[..3] == rgb).count(); + let rgba_under = |effects: serde_json::Value| { + let comp = super::Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + comp.set_scene(Some(shader_fx_scene_with(effects, arrow.clone()))); + compose_fx_at(&comp, 2.0) + }; + + let bare = rgba_under(serde_json::json!([])); + let arrow_px = count(&bare, [255, 255, 255]); + assert!(arrow_px > 500, "la flèche est dessinée : {arrow_px} px blancs"); + + let image = rgba_under(serde_json::json!([fx(BLUE, "image")])); + assert_eq!(count(&image, [255, 255, 255]), arrow_px, "l'effet `image` passe sous la flèche"); + assert_eq!([image[0], image[1], image[2]], [0, 0, 255]); + + let text = rgba_under(serde_json::json!([fx(GREEN_TEXT, "text")])); + assert_eq!(count(&text, [255, 255, 255]), 0, "la flèche est passée par l'effet `text`"); + assert_eq!(count(&text, [0, 255, 0]), arrow_px, "et elle seule est verte"); + assert_eq!([text[0], text[1], text[2]], [255, 0, 0], "l'image dessous est intacte"); + } + + /// Calibre les paliers de coût : le surcoût par frame d'un effet qui lit `frame()` N fois, + /// à 1080p et en 4K, sur le GPU de la machine ; puis le temps de compilation d'un effet. + /// Opt-in, comme les autres mesures : + /// + /// OPENSCREEN_FX_BENCH=1 cargo test -p openscreen-compositor --release --lib \ + /// shader_fx_tier_calibration -- --nocapture + #[test] + fn shader_fx_tier_calibration() { + if std::env::var_os("OPENSCREEN_FX_BENCH").is_none() { + eprintln!("OPENSCREEN_FX_BENCH absent — saute"); + return; + } + let gpu = crate::d3d::Gpu::create(false).expect("device Metal"); + let screen = FakeFrame::new(1920, 1080, |x, y| ((x ^ y) & 0xff) as u8); + let cfg = fx_cfg(); + let reads_effect = |n: u32| { + format!( + "fn effect(uv: vec2) -> vec4 {{\n var acc = vec4(0.0);\n for (var i = 0; i < {n}; i++) {{\n let a = f32(i) * 2.399;\n acc += frame(uv + vec2(cos(a), sin(a)) * fx_px(f32(i)));\n }}\n return acc / {n}.0;\n}}\n" + ) + }; + let per_frame = |w: u32, h: u32, effects: serde_json::Value| -> f64 { + let comp = super::Compositor::new_sized(&gpu, w, h).expect("compositeur"); + comp.set_scene(Some(shader_fx_scene(effects))); + comp.set_timeline_time(Some(2.0)); + comp.set_programme_time(Some(2.0)); + unsafe { comp.compose_frame(screen.as_ptr(), screen.as_ptr(), 0.0, &cfg) }.unwrap(); + unsafe { comp.readback_direct() }.unwrap(); + let n = 200; + let t0 = std::time::Instant::now(); + for f in 0..n { + unsafe { comp.compose_frame(screen.as_ptr(), screen.as_ptr(), f as f32, &cfg) }.unwrap(); + } + unsafe { comp.readback_direct() }.unwrap(); + t0.elapsed().as_secs_f64() * 1000.0 / n as f64 + }; + for (w, h) in [(1920u32, 1080u32), (3840, 2160)] { + let base = (0..5).map(|_| per_frame(w, h, serde_json::json!([]))).fold(f64::MAX, f64::min); + eprintln!("{w}x{h} : sans effet {base:.3} ms/frame"); + for reads in [1u32, 8, 32, 128, 512] { + let code = reads_effect(reads); + let report = crate::shader_fx::analyze(&code, &[]).expect("analyse").report; + let with = (0..5) + .map(|_| per_frame(w, h, serde_json::json!([fx(&code, "image")]))) + .fold(f64::MAX, f64::min); + eprintln!( + "{w}x{h} : {reads:>3} lectures ({:>10}) : +{:.3} ms/frame", + report.tier.name(), + with - base + ); + } + } + + // La compilation : naga, le compilateur de Metal et le pipeline state, sur le ripple de la + // spec. Une fonction de plus, au nom unique, à chaque tour : sans elle le MSL serait le + // même (naga ne garde pas les commentaires) et Metal le servirait depuis son cache. + let ripple = include_str!("../../../docs/shader-effects.md") + .split("```wgsl\n") + .nth(2) + .and_then(|s| s.split("```").next()) + .expect("l'exemple C.6"); + let params: Vec = ["strength", "radius", "duration"] + .into_iter() + .map(|name| crate::scene::ShaderParamDecl { + name: name.into(), + ty: crate::scene::ShaderParamType::Number, + }) + .collect(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + let mut times: Vec = (0..20) + .map(|i| { + let code = format!("{ripple}\nfn fx_nonce_{nonce}_{i}() -> f32 {{ return {i}.0; }}\n"); + let t0 = std::time::Instant::now(); + crate::shader_fx::compile_metal(&gpu.device, &code, ¶ms).expect("compile"); + t0.elapsed().as_secs_f64() * 1000.0 + }) + .collect(); + let first = times[0]; + times.sort_by(f64::total_cmp); + eprintln!( + "compilation du ripple ({} lignes) : première {first:.1} ms, médiane {:.1} ms, max {:.1} ms", + ripple.lines().count(), + times[times.len() / 2], + times[times.len() - 1] + ); + // Et la même source une seconde fois : ce que coûte un effet que Metal a déjà vu. + let code = format!("{ripple}\nfn fx_nonce_{nonce}_0() -> f32 {{ return 0.0; }}\n"); + let t0 = std::time::Instant::now(); + crate::shader_fx::compile_metal(&gpu.device, &code, ¶ms).expect("compile"); + eprintln!("recompilation d'une source déjà vue : {:.1} ms", t0.elapsed().as_secs_f64() * 1000.0); + } } diff --git a/crates/compositor/src/compositor_windows.rs b/crates/compositor/src/compositor_windows.rs index 26dc8f67b8..81ba78b743 100644 --- a/crates/compositor/src/compositor_windows.rs +++ b/crates/compositor/src/compositor_windows.rs @@ -16,7 +16,10 @@ use crate::frame_geometry::{ WEBCAM_SHADOW_OPACITY, WEBCAM_SHADOW_SPREAD_FRAC, }; use crate::cursor::CursorTrack; -use crate::scene::{Scene, SceneBackground, SceneCrop, SceneCursorSprite, WallpaperMotion}; +use crate::scene::{ + Scene, SceneBackground, SceneCrop, SceneCursorSprite, SceneShaderEffect, ShaderEffectTarget, + WallpaperMotion, +}; use crate::d3d::Gpu; use crate::ffi::AVFrame; use anyhow::{bail, Result}; @@ -26,7 +29,8 @@ use std::ffi::c_void; use windows::core::Interface; use windows::Win32::Foundation::RECT; use windows::Win32::Graphics::Direct3D::{ - D3D11_SRV_DIMENSION_TEXTURE2D, D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP, + D3D11_SRV_DIMENSION_TEXTURE2D, D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST, + D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP, }; use windows::Win32::Graphics::Direct3D11::*; use windows::Win32::Graphics::Dxgi::Common::*; @@ -85,6 +89,31 @@ struct BgCache { tex: ID3D11Texture2D, } +/// Un effet shader de la scène (`shader_fx`), par hash de sa source. +enum ShaderFx { + /// naga + FXC tournent sur un thread de fond. + Pending(std::sync::mpsc::Receiver>), + Ready(ID3D11VertexShader, ID3D11PixelShader), + /// Erreur déjà écrite ; l'effet ne se dessine pas. + Failed, +} + +struct ShaderFxEntry { + state: ShaderFx, + /// Dernier usage, pour l'éviction (`SHADER_FX_CACHE`). + used: u64, +} + +/// Combien d'effets compilés le compositeur garde. Un agent qui itère en crée un par version. +const SHADER_FX_CACHE: usize = 64; + +/// Le calque transparent des textes, pour les effets qui ne visent qu'eux. +struct TextLayer { + tex: ID3D11Texture2D, + rtv: ID3D11RenderTargetView, + srv: ID3D11ShaderResourceView, +} + struct DofPyramid { rtv: ID3D11RenderTargetView, srv: ID3D11ShaderResourceView, @@ -264,6 +293,19 @@ pub struct Compositor { /// Pyramide de profondeur de champ, allouée au premier écran incliné et recréée quand la /// texture décodeur change de taille. Rien n'est alloué tant qu'aucun tilt n'est rendu. dof_pyramid: RefCell>, + /// Effets shader de la scène, compilés une fois par source, les moins récents évincés au-delà + /// de `SHADER_FX_CACHE`. + shader_fx: RefCell>, + shader_fx_tick: Cell, + /// `ctx` (b2) et `params` (b3) de la passe d'effet. + shader_fx_cb: ID3D11Buffer, + shader_fx_params_cb: ID3D11Buffer, + /// Alloué à la première frame qui porte un effet `text` ; à la taille de rendu. + text_layer: RefCell>, + /// Vrai par défaut : une frame dont l'effet compile encore attend son binaire, parce qu'un + /// export ne doit jamais rendre de frame sans l'effet. La preview le passe à faux et saute + /// l'effet le temps de la compilation (~10 ms), puis recompose (`shader_fx_landed`). + shader_fx_blocking: Cell, /// Device WARP : la profondeur de champ y obéit à `DOF_ON_CPU_BACKEND`. cpu_backend: bool, } @@ -426,6 +468,7 @@ impl Compositor { bg_cache: RefCell::new(None), live_readback_staging: RefCell::new(None), nv12_readback_staging: RefCell::new(None), + text_layer: RefCell::new(None), ..self }) } @@ -654,6 +697,18 @@ impl Compositor { }; let mut cbuf: Option = None; dev.CreateBuffer(&bd, None, Some(&mut cbuf))?; + let fx_bd = D3D11_BUFFER_DESC { + ByteWidth: std::mem::size_of::() as u32, + ..bd + }; + let mut shader_fx_cb: Option = None; + dev.CreateBuffer(&fx_bd, None, Some(&mut shader_fx_cb))?; + let fx_params_bd = D3D11_BUFFER_DESC { + ByteWidth: crate::shader_fx::PARAMS_BUFFER_BYTES as u32, + ..bd + }; + let mut shader_fx_params_cb: Option = None; + dev.CreateBuffer(&fx_params_bd, None, Some(&mut shader_fx_params_cb))?; // --- blend alpha prémultiplié --- let mut bl = D3D11_BLEND_DESC::default(); @@ -791,6 +846,12 @@ impl Compositor { seg_failed: RefCell::new(false), nv12_readback_staging: RefCell::new(None), dof_pyramid: RefCell::new(None), + shader_fx: RefCell::new(HashMap::new()), + shader_fx_tick: Cell::new(0), + shader_fx_cb: shader_fx_cb.unwrap(), + shader_fx_params_cb: shader_fx_params_cb.unwrap(), + shader_fx_blocking: Cell::new(true), + text_layer: RefCell::new(None), cpu_backend: gpu.backend == crate::d3d::Backend::Cpu, }) } @@ -840,6 +901,208 @@ impl Compositor { /// depuis le layout preset au lieu du planning fixture. pub fn set_scene(&self, s: Option) { *self.scene.borrow_mut() = s; + // Les effets nouveaux partent compiler tout de suite, hors du thread de rendu : à la + // première frame qui les montre, ils sont en général déjà prêts. Après la pose de la + // scène : l'éviction épargne ce qu'elle montre. + if let Some(scene) = self.scene.borrow().as_ref() { + for effect in &scene.shader_effects { + self.shader_fx_entry(effect); + } + } + } + + /// Faux pour la preview : voir `shader_fx_blocking`. + pub fn set_shader_fx_blocking(&self, on: bool) { + self.shader_fx_blocking.set(on); + } + + /// Fait aboutir les compilations terminées. Vrai si un effet vient d'être prêt : la preview + /// en pause doit alors recomposer, sinon elle resterait sur la frame sans l'effet. + pub fn shader_fx_landed(&self) -> bool { + let mut landed = false; + for entry in self.shader_fx.borrow_mut().values_mut() { + let got = match &entry.state { + ShaderFx::Pending(rx) => rx.try_recv(), + _ => continue, + }; + match got { + Ok(result) => { + entry.state = self.finish_shader_fx(result); + landed = true; + } + Err(std::sync::mpsc::TryRecvError::Empty) => {} + Err(std::sync::mpsc::TryRecvError::Disconnected) => entry.state = ShaderFx::Failed, + } + } + landed + } + + /// La clé de cache d'`effect`, sa compilation lancée s'il est neuf, et son dernier usage noté. + /// `None` pour un contrat que ce compositeur ne connaît pas : la région ne se dessine pas. + fn shader_fx_entry(&self, effect: &SceneShaderEffect) -> Option { + if effect.contract != crate::shader_fx::CONTRACT { + return None; + } + let key = crate::shader_fx::source_hash(effect); + let tick = self.shader_fx_tick.get() + 1; + self.shader_fx_tick.set(tick); + let mut fx = self.shader_fx.borrow_mut(); + fx.entry(key) + .or_insert_with(|| ShaderFxEntry { + state: ShaderFx::Pending(crate::shader_fx::spawn_compile(effect)), + used: tick, + }) + .used = tick; + // Un agent qui itère crée une entrée par version : on garde les plus récentes, et toutes + // celles que la scène montre. + if fx.len() > SHADER_FX_CACHE { + let scene = self.scene.borrow(); + let shown = scene.as_ref().map_or(&[][..], |s| &s.shader_effects[..]); + if let Some(oldest) = crate::shader_fx::eviction_victim(fx.iter().map(|(k, e)| (*k, e.used)), shown) { + fx.remove(&oldest); + } + } + Some(key) + } + + fn finish_shader_fx(&self, result: Result) -> ShaderFx { + let created = result.and_then(|c| unsafe { + let (mut vs, mut ps) = (None, None); + self.dev + .CreateVertexShader(&c.vs, None, Some(&mut vs)) + .and_then(|()| self.dev.CreatePixelShader(&c.ps, None, Some(&mut ps))) + .map_err(|e| e.to_string())?; + Ok((vs.unwrap(), ps.unwrap())) + }); + match created { + Ok((vs, ps)) => ShaderFx::Ready(vs, ps), + Err(e) => { + eprintln!("[shader-fx] {e}"); + ShaderFx::Failed + } + } + } + + /// Les shaders d'`effect`. Le compositeur d'export attend leur compilation ; la preview + /// (`shader_fx_blocking` faux) rend `None` tant qu'ils compilent. `None` aussi s'ils ont échoué. + fn shader_fx_ready(&self, effect: &SceneShaderEffect) -> Option<(ID3D11VertexShader, ID3D11PixelShader)> { + let key = self.shader_fx_entry(effect)?; + let mut fx = self.shader_fx.borrow_mut(); + let entry = fx.get_mut(&key)?; + if let ShaderFx::Pending(rx) = &entry.state { + let got = if self.shader_fx_blocking.get() { + rx.recv().map_err(|e| e.to_string()) + } else { + match rx.try_recv() { + Ok(result) => Ok(result), + Err(std::sync::mpsc::TryRecvError::Empty) => return None, + Err(e) => Err(e.to_string()), + } + }; + entry.state = match got { + Ok(result) => self.finish_shader_fx(result), + Err(e) => { + eprintln!("[shader-fx] {e}"); + ShaderFx::Failed + } + }; + } + match &entry.state { + ShaderFx::Ready(vs, ps) => Some((vs.clone(), ps.clone())), + _ => None, + } + } + + unsafe fn upload_bytes(&self, buf: &ID3D11Buffer, bytes: &[u8]) -> bool { + let mut m = D3D11_MAPPED_SUBRESOURCE::default(); + if self.ctx.Map(buf, 0, D3D11_MAP_WRITE_DISCARD, 0, Some(&mut m)).is_err() { + return false; + } + std::ptr::copy_nonoverlapping(bytes.as_ptr(), m.pData as *mut u8, bytes.len()); + self.ctx.Unmap(buf, 0); + true + } + + /// Une passe d'effet sur `dst` (le RT principal, ou le calque des textes) : `dst` est copié + /// dans `ann_copy`, libre à ce stade, puis l'effet le repeint en entier. + unsafe fn apply_shader_effect( + &self, + effect: &SceneShaderEffect, + dst: (&ID3D11Texture2D, &ID3D11RenderTargetView), + cb: &crate::shader_fx::FxContext, + ) { + let Some((vs, ps)) = self.shader_fx_ready(effect) else { return }; + let ctx_bytes = std::slice::from_raw_parts( + cb as *const crate::shader_fx::FxContext as *const u8, + std::mem::size_of::(), + ); + let params = crate::shader_fx::pack_params(effect); + let params_bytes = std::slice::from_raw_parts( + params.as_ptr() as *const u8, + (params.len() * 4).min(crate::shader_fx::PARAMS_BUFFER_BYTES), + ); + if !self.upload_bytes(&self.shader_fx_cb, ctx_bytes) + || !self.upload_bytes(&self.shader_fx_params_cb, params_bytes) + { + return; + } + // On ne lit pas la cible sur laquelle on dessine : copie dans `ann_copy` (les flous + // d'annotation et le cristal de Prism Glow l'ont déjà lue). + self.ctx.CopySubresourceRegion(&self.ann_copy, 0, 0, 0, 0, dst.0, 0, None); + self.ctx.OMSetRenderTargets(Some(&[Some(dst.1.clone())]), None); + let vp = D3D11_VIEWPORT { + TopLeftX: 0.0, + TopLeftY: 0.0, + Width: self.rw(), + Height: self.rh(), + MinDepth: 0.0, + MaxDepth: 1.0, + }; + self.ctx.RSSetViewports(Some(&[vp])); + self.ctx.IASetPrimitiveTopology(D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST); + self.ctx.VSSetShader(&vs, None); + self.ctx.PSSetShader(&ps, None); + // Les registres du prélude tels que naga les sort : t0, s1, b2 (ctx), b3 (params). + self.ctx.PSSetShaderResources(0, Some(&[Some(self.ann_copy_srv.clone())])); + self.ctx.PSSetSamplers(1, Some(&[Some(self.sampler.clone())])); + self.ctx.PSSetConstantBuffers( + 2, + Some(&[Some(self.shader_fx_cb.clone()), Some(self.shader_fx_params_cb.clone())]), + ); + self.ctx.OMSetBlendState(&self.blend_none, None, 0xffffffff); + self.ctx.Draw(3, 0); + self.ctx.PSSetShaderResources(0, Some(&[None])); + self.ctx.PSSetConstantBuffers(2, Some(&[None, None])); + self.bind_compose_state(); + } + + /// Le calque transparent des textes, à la taille de rendu, alloué à la première frame qui en a + /// besoin (un effet `ShaderEffectTarget::Text`). + unsafe fn text_layer(&self) -> Result<(ID3D11Texture2D, ID3D11RenderTargetView, ID3D11ShaderResourceView)> { + if let Some(l) = self.text_layer.borrow().as_ref() { + return Ok((l.tex.clone(), l.rtv.clone(), l.srv.clone())); + } + let mut d = D3D11_TEXTURE2D_DESC::default(); + self.rt.GetDesc(&mut d); + d.BindFlags = (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32; + d.MiscFlags = 0; + d.CPUAccessFlags = 0; + d.Usage = D3D11_USAGE_DEFAULT; + let mut tex: Option = None; + self.dev.CreateTexture2D(&d, None, Some(&mut tex))?; + let tex = tex.ok_or_else(|| anyhow::anyhow!("CreateTexture2D (calque des textes)"))?; + let mut rtv: Option = None; + self.dev.CreateRenderTargetView(&tex, None, Some(&mut rtv))?; + let mut srv: Option = None; + self.dev.CreateShaderResourceView(&tex, None, Some(&mut srv))?; + let layer = TextLayer { + tex, + rtv: rtv.ok_or_else(|| anyhow::anyhow!("RTV (calque des textes)"))?, + srv: srv.ok_or_else(|| anyhow::anyhow!("SRV (calque des textes)"))?, + }; + let out = (layer.tex.clone(), layer.rtv.clone(), layer.srv.clone()); + *self.text_layer.borrow_mut() = Some(layer); + Ok(out) } /// The Y (R8) and UV (R8G8) views of the decoder frame — over a private COPY, never @@ -2315,23 +2578,27 @@ impl Compositor { // pixels déjà floutés. Les autres annotations restent le calque le plus haut (plus bas). self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, true); + // La pointe et les clics récents, tels que les effets shader les lisent : ceux de + // l'enregistrement, que le curseur soit dessiné ou non (`effect_pointer`). + let mut fx_pointer = crate::frame_geometry::EffectPointer::default(); + // --- curseur custom : suit le mapping src/dst (zoom+layout), click bounce, // et flou de mouvement (parité `compositor_macos.rs` et `compositor_linux.rs`) --- if let Some(track) = cursor_ref.as_ref() { - let plan = crate::frame_geometry::plan_cursor( - &g, - &crate::frame_geometry::CursorPlanInput { - render_px: [self.rw(), self.rh()], - u_max, - v_max, - cfg, - live: lp, - scene: scene_ref.as_ref(), - track, - t: self.cursor_t_override.borrow().unwrap_or(frame / FPS), - }, - ) - .map(|p| p.for_backend(self.cpu_backend)); + let input = crate::frame_geometry::CursorPlanInput { + render_px: [self.rw(), self.rh()], + u_max, + v_max, + cfg, + live: lp, + scene: scene_ref.as_ref(), + track, + t: self.cursor_t_override.borrow().unwrap_or(frame / FPS), + }; + if scene_ref.as_ref().is_some_and(|s| !s.shader_effects.is_empty()) { + fx_pointer = crate::frame_geometry::effect_pointer(&g, &input); + } + let plan = crate::frame_geometry::plan_cursor(&g, &input).map(|p| p.for_backend(self.cpu_backend)); if let Some(plan) = plan { let cursor_sprites: HashMap = scene_ref .as_ref() @@ -2540,6 +2807,30 @@ impl Compositor { } } + // --- effets shader de l'agent, autour des textes selon leur cible (`docs/shader-effects.md`, + // D.1) : `image` avant les annotations et les sous-titres, `text` sur un calque des textes + // à part, `both` sur l'image finale. + let fx_inputs = crate::shader_fx::FxFrameInputs { + cursor: fx_pointer.cursor, + clicks: fx_pointer.clicks, + footage: g.footage_quad([self.rw(), self.rh()]), + footage_aspect: g.footage_aspect([self.rw(), self.rh()]), + camera: lp.has_webcam.then_some(w_dst), + }; + let effects = scene_ref + .as_ref() + .map(|s| crate::shader_fx::active_effects(&s.shader_effects, source_t)) + .unwrap_or_default(); + let fx_context = |e: &SceneShaderEffect| { + crate::shader_fx::context(e, source_t, g.programme_t, [self.rw(), self.rh()], &fx_inputs) + }; + let with_target = |target: ShaderEffectTarget| -> Vec<&SceneShaderEffect> { + effects.iter().copied().filter(|e| e.target == target).collect() + }; + for e in with_target(ShaderEffectTarget::Image) { + self.apply_shader_effect(e, (&self.rt, &self.rtv), &fx_context(e)); + } + // --- annotations : calque le plus haut, comme dans le DOM de la preview (le calque y est // monté après la vidéo). Ancrées sur `s_ann`, le rect ÉCRAN SANS ZOOM — c'est le conteneur // que reçoit l'overlay web (`layout.screenRect`) — et volontairement pas sur le rect de @@ -2553,7 +2844,34 @@ impl Compositor { // `source_t`, la même base de temps que les zoom/speed regions : le temps SOURCE du clip, // pas le compteur de frames. C'est ce qui garde une annotation alignée sur l'image quand // une speed region répète ou saute des frames. - self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, false); + let text_effects = with_target(ShaderEffectTarget::Text); + if text_effects.is_empty() { + self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, false); + } else { + // Les textes sur un calque transparent, l'effet sur ce calque seul, puis le calque + // posé sur l'image (« over » prémultiplié) : l'image dessous reste intacte. + let (tex, rtv, srv) = self.text_layer()?; + self.bind_compose_state(); + self.ctx.OMSetRenderTargets(Some(&[Some(rtv.clone())]), None); + self.ctx.ClearRenderTargetView(&rtv, &[0.0, 0.0, 0.0, 0.0]); + self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, false); + for e in text_effects { + self.apply_shader_effect(e, (&tex, &rtv), &fx_context(e)); + } + self.ctx.OMSetRenderTargets(Some(&[Some(self.rtv.clone())]), None); + self.ctx.PSSetShaderResources(0, Some(&[Some(srv)])); + self.ctx.VSSetShader(&self.vs_fs, None); + self.ctx.PSSetShader(&self.ps_tex, None); + self.ctx.PSSetSamplers(0, Some(&[Some(self.sampler.clone())])); + self.ctx.OMSetBlendState(&self.blend, None, 0xffffffff); + self.ctx.Draw(3, 0); + self.ctx.PSSetShaderResources(0, Some(&[None])); + self.bind_compose_state(); + } + + for e in with_target(ShaderEffectTarget::Both) { + self.apply_shader_effect(e, (&self.rt, &self.rtv), &fx_context(e)); + } Ok(()) } @@ -3494,6 +3812,10 @@ mod tests { let mut frame: Box = Box::new(std::mem::zeroed()); frame.data[0] = tex.as_raw() as *mut u8; frame.data[1] = std::ptr::null_mut(); + // La taille visible, comme le décodeur la pose : sans elle le métrage s'aplatit à une + // largeur nulle, et rien de ce qui s'y place (le pointeur des effets) n'y tombe. + frame.width = w as i32; + frame.height = h as i32; (frame, tex) } } @@ -3554,4 +3876,261 @@ mod tests { let (rw, rh, rgba) = unsafe { comp.readback_direct() }.expect("readback"); assert_eq!((rw, rh, rgba.len()), (182, 322, 182 * 322 * 4)); } + + /// Une scène à fond rouge plein, l'écran rétréci par le padding : les coins sont du fond. + fn shader_fx_scene(effects: serde_json::Value) -> Scene { + Scene::from_json(&shader_fx_json(effects).to_string()).expect("scène") + } + + fn shader_fx_json(effects: serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "clips": [], + "layout": {"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}, + "effects": {"padding":0.5,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}, + "background": {"kind":"color","color":"#ff0000"}, + "zoomRegions": [], + "cursor": {"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}, + "cropByClip": [], + "output": {"width":320,"height":180,"fps":null}, + "shaderEffects": effects, + }) + } + + /// Un fragment d'effet de 1 s à 3 s, tel que l'app l'envoie. + fn fx(code: &str, target: &str) -> serde_json::Value { + serde_json::json!({ + "clipIndex": 0, "startSec": 1.0, "endSec": 3.0, "target": target, + "contract": 1, "code": code, + "params": [{"name": "level", "type": "number"}, {"name": "tint", "type": "color"}], + "values": {"level": 1.0, "tint": [0.0, 1.0, 0.0, 1.0]}, + }) + } + + const INVERT: &str = "fn effect(uv: vec2) -> vec4 {\n let c = frame(uv);\n return vec4(vec3(params.level) - c.rgb, 1.0);\n}\n"; + const BLUE: &str = "fn effect(uv: vec2) -> vec4 { return vec4(0.0, 0.0, 1.0, 1.0); }"; + const TINT: &str = "fn effect(uv: vec2) -> vec4 { return params.tint; }"; + const SAME: &str = "fn effect(uv: vec2) -> vec4 { return frame(uv); }"; + + fn fx_cfg() -> crate::config::Cfg { + let mut cfg = crate::config::Cfg::c8(); + cfg.bg_blur = 0.0; + cfg.mblur_n = 1; + cfg + } + + /// Le pixel du coin haut-gauche (du fond) après une frame composée au temps source `t`. + fn corner_at(comp: &Compositor, gpu: &crate::d3d::Gpu, t: f32) -> [u8; 3] { + let (screen, _tex) = nv12_frame(gpu, 64, 64); + comp.set_timeline_time(Some(t)); + comp.set_programme_time(Some(t)); + unsafe { comp.compose_frame(&*screen, &*screen, 0.0, &fx_cfg()) }.expect("compose"); + let (_, _, rgba) = unsafe { comp.readback_direct() }.expect("readback"); + [rgba[0], rgba[1], rgba[2]] + } + + /// Le coin à 2 s, dans la région, sous les effets donnés. + fn corner_under(effects: serde_json::Value) -> Option<[u8; 3]> { + let gpu = crate::d3d::Gpu::create_auto(false).ok()?; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + comp.set_scene(Some(shader_fx_scene(effects))); + Some(corner_at(&comp, &gpu, 2.0)) + } + + /// La passe de l'agent repeint l'image composée, dans sa région et nulle part ailleurs. Le + /// compositeur par défaut est celui de l'export : il attend le shader, jamais de frame sans. + #[test] + fn a_shader_effect_repaints_the_composed_frame_inside_its_region() { + let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else { + eprintln!("pas de device D3D11 — test sauté"); + return; + }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + comp.set_scene(Some(shader_fx_scene(serde_json::json!([fx(INVERT, "image")])))); + assert_eq!(corner_at(&comp, &gpu, 0.5), [255, 0, 0], "avant la région : le fond rouge"); + assert_eq!(corner_at(&comp, &gpu, 2.0), [0, 255, 255], "dans la région : rouge inversé"); + assert_eq!(corner_at(&comp, &gpu, 3.5), [255, 0, 0], "après la région : le fond rouge"); + } + + /// La preview ne bloque pas : elle saute l'effet tant qu'il compile, et `shader_fx_landed` + /// lui dit quand recomposer. Ensuite la frame est celle de l'export, au pixel. + #[test] + fn the_preview_shows_the_effect_once_it_landed() { + let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else { + eprintln!("pas de device D3D11 — test sauté"); + return; + }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + comp.set_shader_fx_blocking(false); + comp.set_scene(Some(shader_fx_scene(serde_json::json!([fx(INVERT, "image")])))); + let start = std::time::Instant::now(); + while !comp.shader_fx_landed() { + assert!(start.elapsed().as_secs() < 10, "la compilation n'a jamais abouti"); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + eprintln!("effet prêt en {:.1} ms", start.elapsed().as_secs_f64() * 1000.0); + assert_eq!(corner_at(&comp, &gpu, 2.0), [0, 255, 255]); + } + + /// Un WGSL invalide, ou un contrat inconnu, ne casse pas le rendu : l'effet ne se dessine + /// pas, la frame reste celle de la composition. + #[test] + fn a_broken_or_unknown_effect_leaves_the_frame_untouched() { + let broken = fx("fn effect(uv: vec2) -> vec4 { nope }", "image"); + let mut future = fx(BLUE, "image"); + future["contract"] = serde_json::json!(2); + for effect in [broken, future] { + let Some(corner) = corner_under(serde_json::json!([effect])) else { return }; + assert_eq!(corner, [255, 0, 0]); + } + } + + /// Les paramètres arrivent dans `params` au layout de la struct générée : la couleur `tint` + /// suit le nombre `level` dans le buffer, et se lit telle que l'app l'a écrite. + #[test] + fn a_color_parameter_reaches_the_shader() { + let Some(corner) = corner_under(serde_json::json!([fx(TINT, "image")])) else { return }; + assert_eq!(corner, [0, 255, 0]); + } + + /// `fx_to_footage` retrouve le point du métrage que le compositeur a dessiné sous un pixel, + /// à plat, sous un angle fixe (warp bilinéaire) et sous la caméra réelle (homographie), et + /// `fx_from_footage` le ramène sur ce pixel. En 1080p, l'arrondi 8 bits et le demi-pixel + /// tiennent sous 0,005 ; la pose orbit testée écarte l'homographie du bilinéaire de ~0,01. + #[test] + fn the_footage_plane_maps_back_to_what_the_compositor_drew() { + let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else { + eprintln!("pas de device D3D11 — test sauté"); + return; + }; + const PLANE: &str = "fn effect(uv: vec2) -> vec4 {\n let p = fx_to_footage(uv);\n return vec4(p, distance(fx_from_footage(p), uv) * 100.0, 1.0);\n}\n"; + for (rotation, projective) in [(None, false), (Some("left"), false), (Some("orbit"), true)] { + let mut json = shader_fx_json(serde_json::json!([fx(PLANE, "image")])); + json["clips"] = serde_json::json!([{"screenPath": "/s.mp4", "webcamPath": "", "sourceStartSec": 0, + "sourceEndSec": 10, "webcamOffsetSec": 0, "hasAudio": false}]); + json["layout"]["screenRect"] = serde_json::json!({"x": 0.1, "y": 0.1, "width": 0.8, "height": 0.8}); + json["cropByClip"] = serde_json::json!([null]); + if let Some(rotation) = rotation { + json["zoomRegions"] = serde_json::json!([{"clipIndex": 0, "startSec": 0, "endSec": 10, + "scale": 1.5, "focusX": 0.5, "focusY": 0.5, "rotation": rotation}]); + } + let comp = Compositor::new_sized(&gpu, 1920, 1080).expect("compositeur"); + let scene = Scene::from_json(&json.to_string()).expect("scène"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + let (screen, _tex) = nv12_frame(&gpu, 64, 64); + comp.set_timeline_time(Some(2.0)); + comp.set_programme_time(Some(2.0)); + unsafe { comp.compose_frame(&*screen, &*screen, 0.0, &fx_cfg()) }.expect("compose"); + let quad = comp.footage_quad().expect("le métrage de la frame"); + assert_eq!(quad.projective, projective, "{rotation:?}"); + let (w, h, rgba) = unsafe { comp.readback_direct() }.expect("readback"); + let [tl, tr, br, bl] = quad.corners.map(|[x, y]| (x, y)); + for (s, t) in [(0.25, 0.3), (0.7, 0.4), (0.4, 0.75)] { + let (x, y) = if projective { + crate::regions::square_to_quad(&[tl, tr, br, bl], s, t) + } else { + let top = (tl.0 + (tr.0 - tl.0) * s, tl.1 + (tr.1 - tl.1) * s); + let bottom = (bl.0 + (br.0 - bl.0) * s, bl.1 + (br.1 - bl.1) * s); + (top.0 + (bottom.0 - top.0) * t, top.1 + (bottom.1 - top.1) * t) + }; + let i = ((y * h as f32) as usize * w as usize + (x * w as f32) as usize) * 4; + let got = (rgba[i] as f32 / 255.0, rgba[i + 1] as f32 / 255.0); + assert!( + (got.0 - s).abs() < 0.005 && (got.1 - t).abs() < 0.005, + "{rotation:?} ({s}, {t}) : lu {got:?}, coins {:?}", + quad.corners + ); + assert!(rgba[i + 2] < 3, "{rotation:?} ({s}, {t}) : aller-retour à {}", rgba[i + 2]); + } + } + } + + /// La scène de ces tests masque le curseur (`cursor.show` faux) : rien n'est dessiné, mais un + /// effet voit toujours la pointe et le clic de l'enregistrement (`effect_pointer`). Il peint + /// `ctx.cursor.z` en rouge et `ctx.clicks[0].w` en vert : jaune, il a vu les deux. + #[test] + fn a_hidden_cursor_still_hands_the_effect_its_pointer_and_clicks() { + let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else { + eprintln!("pas de device D3D11 — test sauté"); + return; + }; + const SEES: &str = "fn effect(uv: vec2) -> vec4 { return vec4(ctx.cursor.z, ctx.clicks[0].w, 0.0, 1.0); }"; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + comp.set_scene(Some(shader_fx_scene(serde_json::json!([fx(SEES, "image")])))); + comp.set_cursor(crate::cursor::CursorTrack::new_with_visibility( + vec![(0.0, 0.5, 0.5), (4.0, 0.5, 0.5)], + vec![1.5], + vec![], + vec![(0.0, true)], + )); + comp.set_cursor_time(Some(2.0)); + assert_eq!(corner_at(&comp, &gpu, 2.0), [255, 255, 0]); + } + + /// `text` ne touche que le calque des textes, posé ensuite sur l'image ; `image` et `both` + /// repeignent l'image. Sans annotation, le calque est transparent : un effet `text` qui le + /// rend tel quel laisse l'image intacte, un effet `text` opaque la recouvre. + #[test] + fn the_three_targets_keep_to_their_layer() { + let Some(image_only) = corner_under(serde_json::json!([fx(BLUE, "image")])) else { return }; + assert_eq!(image_only, [0, 0, 255]); + let text_untouched = corner_under(serde_json::json!([fx(SAME, "text")])).unwrap(); + assert_eq!(text_untouched, [255, 0, 0], "un calque transparent ne change pas l'image"); + let text_opaque = corner_under(serde_json::json!([fx(TINT, "text")])).unwrap(); + assert_eq!(text_opaque, [0, 255, 0], "le calque des textes, opaque, posé sur l'image"); + let both = corner_under(serde_json::json!([fx(INVERT, "both")])).unwrap(); + assert_eq!(both, [0, 255, 255], "`both` repeint l'image finale"); + } + + /// Calibre les paliers de coût : le surcoût par frame d'un effet qui lit `frame()` N fois, + /// à 1080p et en 4K, sur le GPU de la machine. Opt-in, comme les autres mesures : + /// + /// set OPENSCREEN_FX_BENCH=1 && cargo test -p openscreen-compositor --release + /// shader_fx_tier_calibration -- --nocapture + #[test] + fn shader_fx_tier_calibration() { + if std::env::var_os("OPENSCREEN_FX_BENCH").is_none() { + eprintln!("OPENSCREEN_FX_BENCH absent — saute"); + return; + } + let gpu = crate::d3d::Gpu::create_backend(crate::d3d::Backend::Hardware, false) + .expect("device D3D11"); + let (screen, _tex) = nv12_frame(&gpu, 1920, 1080); + let cfg = fx_cfg(); + let reads_effect = |n: u32| { + format!( + "fn effect(uv: vec2) -> vec4 {{\n var acc = vec4(0.0);\n for (var i = 0; i < {n}; i++) {{\n let a = f32(i) * 2.399;\n acc += frame(uv + vec2(cos(a), sin(a)) * fx_px(f32(i)));\n }}\n return acc / {n}.0;\n}}\n" + ) + }; + let per_frame = |w: u32, h: u32, effects: serde_json::Value| -> f64 { + let comp = Compositor::new_sized(&gpu, w, h).expect("compositeur"); + comp.set_scene(Some(shader_fx_scene(effects))); + comp.set_timeline_time(Some(2.0)); + comp.set_programme_time(Some(2.0)); + unsafe { comp.compose_frame(&*screen, &*screen, 0.0, &cfg) }.unwrap(); + unsafe { comp.readback_direct() }.unwrap(); + let n = 200; + let t0 = std::time::Instant::now(); + for f in 0..n { + unsafe { comp.compose_frame(&*screen, &*screen, f as f32, &cfg) }.unwrap(); + } + unsafe { comp.readback_direct() }.unwrap(); + t0.elapsed().as_secs_f64() * 1000.0 / n as f64 + }; + for (w, h) in [(1920u32, 1080u32), (3840, 2160)] { + let base = (0..5).map(|_| per_frame(w, h, serde_json::json!([]))).fold(f64::MAX, f64::min); + for reads in [1u32, 8, 32, 128, 512] { + let code = reads_effect(reads); + let report = crate::shader_fx::analyze(&code, &[]).expect("analyse").report; + let with = (0..5) + .map(|_| per_frame(w, h, serde_json::json!([fx(&code, "image")]))) + .fold(f64::MAX, f64::min); + eprintln!( + "{w}x{h} : {reads:>3} lectures ({:>10}) : +{:.3} ms/frame", + report.tier.name(), + with - base + ); + } + } + } } diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index 0decdb21f1..86b881f273 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -2857,12 +2857,17 @@ impl FrameGeometry { let [x, y, w, h] = self.s_dst; FootageQuad { corners: [[x, y], [x + w, y], [x + w, y + h], [x, y + h]], projective: false } } + + /// Largeur sur hauteur du métrage dans son propre plan : celles du rect qu'incline le tilt. + pub fn footage_aspect(&self, render_px: [f32; 2]) -> f32 { + self.s_dst[2] * render_px[0] / (self.s_dst[3] * render_px[1]).max(1e-6) + } } /// Le métrage dans l'image : ses coins TL, TR, BR, BL en fractions de la sortie, et la /// correspondance d'un point `(u, v)` du métrage à l'image, bilinéaire entre eux ou, sous la /// caméra réelle, l'homographie qu'ils définissent (`TiltedQuad::point_px`). -#[derive(Debug, Clone, Copy, PartialEq)] +#[derive(Debug, Clone, Copy, PartialEq, Default)] pub struct FootageQuad { pub corners: [[f32; 2]; 4], pub projective: bool, @@ -3707,6 +3712,119 @@ fn click_impact_at( ) } +/// Pose un point de la piste (repère normalisé de l'écran) sur la sortie, à travers le zoom, le +/// layout et le tilt de la frame : ce que partagent le curseur dessiné (`plan_cursor`) et les +/// effets shader (`effect_pointer`), pour qu'un effet voie la pointe là où le curseur se dessine. +struct CursorPlacer<'a> { + g: &'a FrameGeometry, + track: &'a crate::cursor::CursorTrack, + uv_max: [f32; 2], + render_px: [f32; 2], + /// Le rect d'écran non incliné, en px. + s_px: [f32; 2], + tilt: Option, + /// Centre du plan en px de sortie. + quad_center_px: [f32; 2], + model3d: bool, +} + +impl<'a> CursorPlacer<'a> { + fn new(g: &'a FrameGeometry, input: &CursorPlanInput<'a>, model3d: bool) -> Self { + let [rw, rh] = input.render_px; + let s_px = [g.s_dst[2] * rw, g.s_dst[3] * rh]; + CursorPlacer { + g, + track: input.track, + uv_max: [input.u_max, input.v_max], + render_px: input.render_px, + s_px, + tilt: g.screen_tilt(s_px), + quad_center_px: [(g.s_dst[0] + g.s_dst[2] * 0.5) * rw, (g.s_dst[1] + g.s_dst[3] * 0.5) * rh], + model3d, + } + } + + /// La pointe à `t`. Le curseur modélisé touche le plan à chaque clic : il se pose sur le point + /// cliqué brut (`pinned_at`), pas sur la piste lissée qui traîne derrière la souris. Le sprite + /// plat garde la piste telle quelle, son rendu est celui d'avant. + fn at(&self, t: f32) -> Option<(f32, f32)> { + if self.model3d { self.track.pinned_at(t) } else { self.track.at(t) } + } + + /// `None` hors de la coupe courante (zoom serré, hors écran). + fn place(&self, cxy: Option<(f32, f32)>, dst: [f32; 4]) -> Option { + let [rw, rh] = self.render_px; + let [fx, fy] = cursor_plane_point(self.g.cut, self.uv_max, cxy?)?; + Some(match self.tilt.as_ref() { + Some(&quad) => CursorPlacement::Tilted { + plane_pt: [fx, fy], + quad, + center_px: self.quad_center_px, + screen_px: self.s_px, + render_px: [rw, rh], + }, + // Le curseur modélisé a toujours besoin d'un plan : sur écran droit, un plan + // IDENTITÉ taillé dans `dst`. À rotation nulle `rotated_quad_corners_px` rend les + // coins exacts (±w/2, ±h/2, échelle 1) : même caméra, même mode 15. + None if self.model3d => { + let screen_px = [dst[2] * rw, dst[3] * rh]; + CursorPlacement::Tilted { + plane_pt: [fx, fy], + quad: crate::regions::rotated_quad_corners_px( + screen_px[0], + screen_px[1], + [0.0; 3], + [0.0; 3], + ), + center_px: [(dst[0] + dst[2] * 0.5) * rw, (dst[1] + dst[3] * 0.5) * rh], + screen_px, + render_px: [rw, rh], + } + } + None => CursorPlacement::Upright { center: [dst[0] + fx * dst[2], dst[1] + fy * dst[3]] }, + }) + } +} + +/// Ce que les effets shader lisent du pointeur (`ctx.cursor`, `ctx.clicks`), en 0..1 de SORTIE, +/// zoom, layout et tilt appliqués : ce qu'un effet ne peut pas recalculer lui-même. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct EffectPointer { + /// La pointe, là où le curseur se dessine ; `None` quand l'enregistrement l'a cachée + /// (`visible: false`) ou qu'elle sort de la coupe. + pub cursor: Option<[f32; 2]>, + /// Les clics des `shader_fx::CLICK_WINDOW_S` dernières secondes, du plus récent au plus ancien, + /// au plus `shader_fx::MAX_CLICKS` : le point de la pointe AU MOMENT du clic, et l'âge en + /// secondes. + pub clicks: Vec<([f32; 2], f32)>, +} + +/// La pointe et les clics récents pour les effets, que le curseur soit dessiné ou non. Masqué par +/// l'utilisateur (`cursor.show`, l'auto-hide, `hideCursor` d'un zoom), il reste la pointe de +/// l'enregistrement : un effet peut suivre la souris, ou remplacer le curseur par son propre +/// dessin. Seuls l'enregistrement qui la cache et la coupe qui la laisse dehors l'en privent. +pub fn effect_pointer(g: &FrameGeometry, input: &CursorPlanInput) -> EffectPointer { + let model3d = input.live.cursor_model3d + && modelled_sprite(input.scene, input.track.type_at(input.t)).is_some(); + let placer = CursorPlacer::new(g, input, model3d); + let cursor = input + .track + .visible_at(input.t) + .then(|| placer.place(placer.at(input.t), g.s_dst)) + .flatten() + .map(|p| p.upright_center()); + let clicks = input + .track + .clicks_with_points(input.t - crate::shader_fx::CLICK_WINDOW_S, input.t) + .collect::>() + .into_iter() + .rev() + .take(crate::shader_fx::MAX_CLICKS) + .filter_map(|(tc, p)| Some((placer.place(Some(p), g.s_dst)?.upright_center(), input.t - tc))) + .collect(); + EffectPointer { cursor, clicks } +} + /// `None` = rien à dessiner cette frame : curseur masqué, pointeur hors du rect source /// courant (zoom serré, hors écran), ou sprite réduit à rien au creux d'un click bounce /// extrême — un état normal en lecture, pas une erreur. @@ -3723,56 +3841,14 @@ pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option, dst: [f32; 4]| -> Option { - cxy.and_then(|p| { - let [fx, fy] = cursor_plane_point(g.cut, [input.u_max, input.v_max], p)?; - Some(match tilt.as_ref() { - Some(&quad) => CursorPlacement::Tilted { - plane_pt: [fx, fy], - quad, - center_px: quad_center_px, - screen_px: s_px, - render_px: [rw, rh], - }, - // Le curseur modélisé a toujours besoin d'un plan : sur écran droit, un plan - // IDENTITÉ taillé dans `dst`. À rotation nulle `rotated_quad_corners_px` rend les - // coins exacts (±w/2, ±h/2, échelle 1) : même caméra, même mode 15. - None if model3d => { - let screen_px = [dst[2] * rw, dst[3] * rh]; - CursorPlacement::Tilted { - plane_pt: [fx, fy], - quad: crate::regions::rotated_quad_corners_px( - screen_px[0], - screen_px[1], - [0.0; 3], - [0.0; 3], - ), - center_px: [(dst[0] + dst[2] * 0.5) * rw, (dst[1] + dst[3] * 0.5) * rh], - screen_px, - render_px: [rw, rh], - } - } - None => CursorPlacement::Upright { - center: [dst[0] + fx * dst[2], dst[1] + fy * dst[3]], - }, - }) - }) - }; - // Le curseur modélisé touche le plan à chaque clic : il se pose sur le point cliqué brut - // (`pinned_at`), pas sur la piste lissée qui traîne derrière la souris. Le sprite plat garde - // la piste telle quelle, son rendu est celui d'avant. - let at = |t: f32| if model3d { input.track.pinned_at(t) } else { input.track.at(t) }; - let placement = place(at(input.t), g.s_dst)?; + let placer = CursorPlacer::new(g, input, model3d); + let (s_px, tilt, quad_center_px) = (placer.s_px, placer.tilt, placer.quad_center_px); + let place = |cxy: Option<(f32, f32)>, dst: [f32; 4]| placer.place(cxy, dst); + let placement = place(placer.at(input.t), g.s_dst)?; let lp = input.live; // `cursor_bounce_scale` est le clickBounce brut (0..5) : au-delà de 1/0.24 ≈ 4.17, le creux @@ -3811,7 +3887,7 @@ pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option) -> crate::cursor::CursorTrack { + let mut visibility = vec![(0.0, true)]; + visibility.extend(hidden_from.map(|t| (t, false))); + crate::cursor::CursorTrack::new_with_visibility( + vec![(0.0, 0.3, 0.6), (1.0, 0.3, 0.6), (1.2, 0.5, 0.4), (2.0, 0.8, 0.2), (6.0, 0.8, 0.2)], + vec![1.0, 1.2], + vec![], + visibility, + ) + } + + fn cursor_input<'a>( + scene: &'a Scene, + cfg: &'a crate::config::Cfg, + track: &'a crate::cursor::CursorTrack, + t: f32, + ) -> CursorPlanInput<'a> { + CursorPlanInput { + render_px: [1920.0, 1080.0], + u_max: 1.0, + v_max: 1.0, + cfg, + live: LiveParams::default(), + scene: Some(scene), + track, + t, + } + } + + /// Les effets shader lisent la pointe et les clics récents en uv de SORTIE : un clic garde le + /// point où la pointe était quand il a eu lieu, le plus récent vient en premier, un clic sort + /// de la liste après la fenêtre, et la pointe est là où le curseur se dessine. + #[test] + fn effects_get_the_pointer_and_the_recent_clicks() { + let scene = model_scene(); + let cfg = crate::config::all().pop().expect("cfg"); + let fg = full_frame_geometry(); + let track = clicked_track(None); + let fx = |t: f32| effect_pointer(&fg, &cursor_input(&scene, &cfg, &track, t)); + assert!(fx(0.5).clicks.is_empty(), "pas encore de clic"); + let (first, second) = (fx(1.0).cursor.expect("pointe"), fx(1.2).cursor.expect("pointe")); + let clicks = fx(1.5).clicks; + assert_eq!(clicks.len(), 2); + assert_eq!((clicks[0].0, clicks[1].0), (second, first), "le plus récent d'abord, au point du clic"); + assert!((clicks[0].1 - 0.3).abs() < 1e-4 && (clicks[1].1 - 0.5).abs() < 1e-4, "{clicks:?}"); + assert_ne!(fx(2.5).cursor, Some(second), "la pointe a bougé depuis"); + assert_eq!(fx(1.1 + crate::shader_fx::CLICK_WINDOW_S).clicks.len(), 1, "le premier est sorti"); + let drawn = plan_cursor(&fg, &cursor_input(&scene, &cfg, &track, 2.5)).expect("visible"); + assert_eq!(fx(2.5).cursor, Some(drawn.placement.upright_center()), "là où le curseur se dessine"); + } + + /// Masqué par l'utilisateur, le curseur n'est plus dessiné, mais les effets gardent la pointe + /// et les clics : un halo suit toujours la souris, une onde part toujours du clic. Quand c'est + /// l'enregistrement qui cache la pointe, les effets la perdent, et gardent les clics d'avant. + #[test] + fn a_hidden_cursor_still_hands_effects_the_pointer_and_the_clicks() { + let shown = model_scene(); + let mut hidden = model_scene(); + hidden.cursor.show = false; + let cfg = crate::config::all().pop().expect("cfg"); + let fg = full_frame_geometry(); + let track = clicked_track(None); + assert!(plan_cursor(&fg, &cursor_input(&hidden, &cfg, &track, 1.5)).is_none(), "rien à dessiner"); + let seen = effect_pointer(&fg, &cursor_input(&hidden, &cfg, &track, 1.5)); + assert_eq!(seen, effect_pointer(&fg, &cursor_input(&shown, &cfg, &track, 1.5))); + assert!(seen.cursor.is_some() && seen.clicks.len() == 2, "{seen:?}"); + + let recorded_hidden = clicked_track(Some(1.3)); + let after = effect_pointer(&fg, &cursor_input(&shown, &cfg, &recorded_hidden, 1.5)); + assert_eq!(after.cursor, None, "l'enregistrement a caché la pointe"); + assert_eq!(after.clicks, seen.clicks, "les clics d'avant restent"); + } + /// Sous un angle fixe, l'écran tourne autour du centre de sa boîte ZOOMÉE, réduit de son /// containment : le curseur y perdait ~18 % au point de focus même, pendant que le zoom /// grossissait l'image. Il y garde sa taille à plat, à la perspective de biais près ; ailleurs, diff --git a/crates/compositor/src/lib.rs b/crates/compositor/src/lib.rs index b336797050..16ebde9d09 100644 --- a/crates/compositor/src/lib.rs +++ b/crates/compositor/src/lib.rs @@ -47,6 +47,8 @@ pub mod prism_mesh; pub mod remux; pub mod scene; pub mod sculpt; +// Effets shader écrits par un agent (docs/shader-effects.md) : WGSL -> naga -> backend. +pub mod shader_fx; // Segmentation du sujet webcam (masque -> `t3`). Le module compile toujours ; sans la feature // `segmentation` ses deux entrées échouent proprement, ce qui garde le reste du crate // indépendant du choix de packaging d'ONNX Runtime. @@ -56,7 +58,7 @@ pub mod shared_frames; pub mod text_anim; pub mod text_fonts; pub mod text_plate; -pub(crate) mod timeline_walk; +pub mod timeline_walk; // GPU backend : Windows → d3d_windows, macOS → d3d_macos. Ré-exporté sous le nom `d3d` // pour que `crate::d3d::Gpu`/`Backend` reste portable. Le module sous-jacent est diff --git a/crates/compositor/src/live.rs b/crates/compositor/src/live.rs index 16c75bd9e0..ca1645605e 100644 --- a/crates/compositor/src/live.rs +++ b/crates/compositor/src/live.rs @@ -1491,6 +1491,9 @@ unsafe fn render_thread( // Chemin de PRODUCTION : matériel si possible, backend CPU sinon (voir `create_auto`). let gpu = Gpu::create_auto(false)?; let mut comp = Compositor::new(&gpu)?; + // La preview ne bloque jamais sur la compilation d'un effet shader : elle le saute le temps + // qu'il compile (voir `shader_fx_landed` plus bas). L'export, lui, attend. + comp.set_shader_fx_blocking(false); // Vue live = le VRAI enregistrement, pas la fenêtre fixture (100s@6s, taillée pour l'ancien // fixture POC). On charge toute la piste depuis t=0 ; 24h couvre large toute recording réelle. // Gardée à part (raw_cursor) pour pouvoir régénérer une variante lissée sans relire le @@ -1781,6 +1784,9 @@ unsafe fn render_thread( continue; } + // Un effet shader vient de finir de compiler : la frame en pause doit le montrer. + let fx_landed = comp.shader_fx_landed(); + // avance : seek app-piloté (presentTime) prioritaire, sinon lecture libre (60 fps) let requested = shared.requested_frame.lock().unwrap().take(); let now = Instant::now(); @@ -1915,7 +1921,7 @@ unsafe fn render_thread( break; } } - } else if first || ip_changed || scene_changed || clip_changed || resized { + } else if first || ip_changed || scene_changed || clip_changed || resized || fx_landed { // pause : recompose la frame courante (param / scène / clip / résolution changés). (settle_until, last_settle) = open_settle_window(now); let _ = player.recompose(&comp, &cfg); diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs index 895fa6b2a2..94dee1d593 100644 --- a/crates/compositor/src/scene.rs +++ b/crates/compositor/src/scene.rs @@ -547,6 +547,73 @@ pub struct SceneCameraFullscreenRegion { pub continues_after: bool, } +/// Un fragment de région d'effet shader (`docs/shader-effects.md`, B.5). Temps source du clip, +/// comme les autres régions ; la définition (code, paramètres) voyage avec chaque fragment. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneShaderEffect { + #[serde(default)] + pub clip_index: Option, + pub start_sec: f64, + pub end_sec: f64, + /// L'étendue de la pilule ENTIÈRE en temps programme, d'où `ctx.region`. Absente : la + /// progression court sur le fragment, en temps source. + #[serde(default)] + pub programme_start_sec: Option, + #[serde(default)] + pub programme_end_sec: Option, + #[serde(default)] + pub target: ShaderEffectTarget, + /// Version du contrat. Une version que ce compositeur ne connaît pas ne se dessine pas. + #[serde(default = "default_contract")] + pub contract: u32, + /// Le WGSL de l'agent : une `fn effect(uv: vec2) -> vec4`. + pub code: String, + /// Les paramètres déclarés, dans l'ordre de la struct `FxParams` générée. + #[serde(default)] + pub params: Vec, + /// Les valeurs, déjà normalisées par l'app : un nombre, ou 2 ou 4 floats. + #[serde(default)] + pub values: std::collections::HashMap, +} + +fn default_contract() -> u32 { + crate::shader_fx::CONTRACT +} + +/// Ce qu'un effet transforme : l'image sans les textes, les textes seuls, ou les deux. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ShaderEffectTarget { + #[default] + Image, + Text, + Both, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub struct ShaderParamDecl { + pub name: String, + #[serde(rename = "type")] + pub ty: ShaderParamType, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ShaderParamType { + Number, + Toggle, + Color, + Point, +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(untagged)] +pub enum ShaderParamValue { + Scalar(f32), + Vector(Vec), +} + /// Rendu du curseur. #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] @@ -798,6 +865,9 @@ pub struct Scene { /// Effet d'arrière-plan de la webcam. Absent = aucun effet. #[serde(default)] pub webcam_effect: Option, + /// Effets shader écrits par un agent (`shader_fx`). Absent = aucun. + #[serde(default)] + pub shader_effects: Vec, /// État de rendu interne, positionné par `for_clip_window` (jamais envoyé par l'app). #[serde(skip)] pub(crate) active_clip_index: usize, @@ -845,6 +915,9 @@ impl Scene { scene.annotations.retain(|annotation| { belongs(annotation.clip_index, annotation.start_sec, annotation.end_sec) }); + scene.shader_effects.retain(|effect| { + belongs(effect.clip_index, effect.start_sec, effect.end_sec) + }); // Le layout dépend de la FORME de la source du clip (dimensions natives × crop), qui // varie d'un clip à l'autre. On installe donc celui du clip composé dans les champs // scalaires : `compose_frame` continue de lire un seul `layout`, sans jamais avoir à diff --git a/crates/compositor/src/shader_fx.rs b/crates/compositor/src/shader_fx.rs new file mode 100644 index 0000000000..08299edc82 --- /dev/null +++ b/crates/compositor/src/shader_fx.rs @@ -0,0 +1,1312 @@ +//! Effets shader écrits par un agent (`docs/shader-effects.md`) : du WGSL qui définit +//! `fn effect(uv: vec2) -> vec4`, appliqué en une passe plein cadre par `compose_frame`, +//! donc à l'identique en preview et à l'export. +//! +//! Ce module porte ce qui ne dépend d'aucun backend : le contrat (le prélude et la struct des +//! paramètres, générée), la validation, la borne des boucles, l'estimation du coût en paliers, la +//! sélection des régions actives et le remplissage des constant buffers. Chaque backend traduit le +//! module par naga et le compile une fois, hors de la boucle de rendu. +//! +//! L'effet est une fonction PURE de l'image composée, du temps, du curseur et de ses paramètres : +//! pas d'état d'une frame à l'autre. C'est ce qui garde la preview (qui saute d'une frame à +//! l'autre au seek) et l'export (qui les rend dans l'ordre) d'accord. + +use crate::frame_geometry::FootageQuad; +use crate::scene::{SceneShaderEffect, ShaderEffectTarget, ShaderParamDecl, ShaderParamType, ShaderParamValue}; +use std::hash::{Hash, Hasher}; + +/// La version du contrat que ce prélude implémente. Un prélude livré ne change plus : une entrée +/// nouvelle s'ajoute à la fin de `FxContext`, un changement de sens crée la version suivante. +pub const CONTRACT: u32 = 1; + +/// Les entrées que l'effet lit, en plus de ses paramètres. Placées APRÈS le code de l'agent (et +/// après `FxParams`) : en WGSL l'ordre des déclarations de module est libre, et les erreurs +/// pointent ainsi les lignes que l'agent a écrites. Voir la section C de la spec. +pub const PRELUDE_V1: &str = r#" +struct FxContext { + resolution: vec2, + time: f32, + aspect: f32, + region: vec4, + cursor: vec4, + clicks: array, 4>, + footage: vec4, + camera: vec4, + footage_corners: array, 2>, + footage_aspect: f32, + footage_warp: f32, +}; +@group(0) @binding(0) var fx_frame: texture_2d; +@group(0) @binding(1) var fx_sampler: sampler; +@group(0) @binding(2) var ctx: FxContext; + +fn frame(uv: vec2) -> vec4 { + return textureSampleLevel(fx_frame, fx_sampler, uv, 0.0); +} + +fn fx_px(n: f32) -> f32 { + return n / 1080.0; +} + +// The recording's surface, wherever the zoom, the layout and the 3D camera put it, through the +// same warp the compositor draws it with. A point of the surface is (0,0) at the recording's +// top-left corner and (1,1) at its bottom-right, extended past its edges. Mirror of +// `quad_forward` in shaders.hlsl. +fn fx_from_footage(p: vec2) -> vec2 { + let r = ctx.resolution; + let c00 = ctx.footage_corners[0].xy * r; + let c10 = ctx.footage_corners[0].zw * r; + let c11 = ctx.footage_corners[1].xy * r; + let c01 = ctx.footage_corners[1].zw * r; + if (ctx.footage_warp > 0.5) { + let p1 = c10 - c00; + let p2 = c11 - c00; + let p3 = c01 - c00; + let d1 = p1 - p2; + let d2 = p3 - p2; + let d3 = p2 - p1 - p3; + let den = d1.x * d2.y - d2.x * d1.y; + let g = (d3.x * d2.y - d2.x * d3.y) / den; + let h = (d1.x * d3.y - d3.x * d1.y) / den; + return (c00 + (p1 * (1.0 + g) * p.x + p3 * (1.0 + h) * p.y) / (g * p.x + h * p.y + 1.0)) / r; + } + return (c00 + p.x * (c10 - c00) + p.y * (c01 - c00) + p.x * p.y * (c00 - c10 - c01 + c11)) / r; +} + +fn fx_footage_root(t: f32, e: vec2, f: vec2, g: vec2, h: vec2) -> vec3 { + let dx = e.x + g.x * t; + let dy = e.y + g.y * t; + var s = (h.y - f.y * t) / dy; + if (abs(dx) > abs(dy)) { + s = (h.x - f.x * t) / dx; + } + let inside = s >= -0.02 && s <= 1.02 && t >= -0.02 && t <= 1.02; + return vec3(s, t, select(0.0, 1.0, inside)); +} + +// The point of the recording's surface under `uv`: fx_from_footage backwards. Where no point of +// the surface lands (past its horizon), a point far outside 0..1. Mirror of `quad_inverse` in +// shaders.hlsl. +fn fx_to_footage(uv: vec2) -> vec2 { + let none = vec2(-1.0e4); + let r = ctx.resolution; + let c00 = ctx.footage_corners[0].xy * r; + let c10 = ctx.footage_corners[0].zw * r; + let c11 = ctx.footage_corners[1].xy * r; + let c01 = ctx.footage_corners[1].zw * r; + if (ctx.footage_warp > 0.5) { + let p1 = c10 - c00; + let p2 = c11 - c00; + let p3 = c01 - c00; + let d1 = p1 - p2; + let d2 = p3 - p2; + let d3 = p2 - p1 - p3; + let den = d1.x * d2.y - d2.x * d1.y; + let g = (d3.x * d2.y - d2.x * d3.y) / den; + let h = (d1.x * d3.y - d3.x * d1.y) / den; + let q = uv * r - c00; + let m00 = p1.x * (1.0 + g) - g * q.x; + let m01 = p3.x * (1.0 + h) - h * q.x; + let m10 = p1.y * (1.0 + g) - g * q.y; + let m11 = p3.y * (1.0 + h) - h * q.y; + let det = m00 * m11 - m01 * m10; + let st = vec2(q.x * m11 - m01 * q.y, m00 * q.y - q.x * m10) / det; + // Behind the camera: the homography reaches the point, the picture never shows it. + if (g * st.x + h * st.y + 1.0 <= 0.0) { + return none; + } + return st; + } + let e = c10 - c00; + let f = c01 - c00; + let g = c00 - c10 - c01 + c11; + let h = uv * r - c00; + let k2 = g.x * f.y - g.y * f.x; + let k1 = e.x * f.y - e.y * f.x + h.x * g.y - h.y * g.x; + let k0 = h.x * e.y - h.y * e.x; + if (abs(k2) < 1e-5 * abs(k1)) { + if (abs(k1) < 1e-6) { + return none; + } + return fx_footage_root(-k0 / k1, e, f, g, h).xy; + } + let disc = k1 * k1 - 4.0 * k2 * k0; + if (disc < 0.0) { + return none; + } + let q = -0.5 * (k1 + select(-1.0, 1.0, k1 >= 0.0) * sqrt(disc)); + let r0 = fx_footage_root(q / k2, e, f, g, h); + let r1 = fx_footage_root(select(q / k2, k0 / q, abs(q) > 0.0), e, f, g, h); + return select(r1.xy, r0.xy, r0.z > 0.5); +} + +fn fx_pcg(v: u32) -> u32 { + let state = v * 747796405u + 2891336453u; + let word = ((state >> ((state >> 28u) + 4u)) ^ state) * 277803737u; + return (word >> 22u) ^ word; +} + +fn fx_hash(p: vec2) -> f32 { + return f32(fx_pcg(p.x ^ fx_pcg(p.y)) >> 8u) / 16777216.0; +} + +struct FxVsOut { + @builtin(position) pos: vec4, + @location(0) uv: vec2, +}; + +@vertex +fn fx_vs(@builtin(vertex_index) i: u32) -> FxVsOut { + let p = vec2(f32((i << 1u) & 2u), f32(i & 2u)); + var o: FxVsOut; + o.pos = vec4(p.x * 2.0 - 1.0, 1.0 - p.y * 2.0, 0.0, 1.0); + o.uv = p; + return o; +} + +@fragment +fn fx_fs(v: FxVsOut) -> @location(0) vec4 { + return effect(v.uv); +} +"#; + +/// Combien de clics récents l'effet voit (`ctx.clicks`), du plus récent au plus ancien. +pub const MAX_CLICKS: usize = 4; +/// Jusqu'où un clic reste visible des effets : au-delà, son entrée est vide (`w` = 0). +pub const CLICK_WINDOW_S: f32 = 2.0; +/// Le plafond de sécurité du travail par pixel, toutes boucles imbriquées comprises. Ce n'est pas +/// un budget : c'est l'ordre de grandeur où un GPU intégré atteint le TDR en 1080p. +pub const MAX_ITERATIONS_PER_PIXEL: u64 = 4096; +/// Taille du constant buffer des paramètres : le plafond technique (32) de `vec4`. +pub const PARAMS_BUFFER_BYTES: usize = 32 * 16; + +/// Le constant buffer `ctx`, au layout de `FxContext` (192 octets). +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub struct FxContext { + pub resolution: [f32; 2], + pub time: f32, + pub aspect: f32, + pub region: [f32; 4], + pub cursor: [f32; 4], + pub clicks: [[f32; 4]; MAX_CLICKS], + pub footage: [f32; 4], + pub camera: [f32; 4], + /// Les coins du métrage, TL et TR puis BR et BL, en uv de sortie : `fx_to_footage`. + pub footage_corners: [[f32; 4]; 2], + /// Largeur sur hauteur du métrage dans son propre plan, avant toute inclinaison. + pub footage_aspect: f32, + /// 1 : l'homographie des coins (la caméra réelle), 0 : leur warp bilinéaire (`TiltedQuad`). + pub footage_warp: f32, + pub _pad: [f32; 2], +} + +/// Ce que le plan du curseur et la géométrie de la frame donnent aux effets, en uv de sortie. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct FxFrameInputs { + pub cursor: Option<[f32; 2]>, + /// Les clics récents, du plus récent au plus ancien : point et âge en secondes. + pub clicks: Vec<([f32; 2], f32)>, + pub footage: FootageQuad, + pub footage_aspect: f32, + pub camera: Option<[f32; 4]>, +} + +/// Le palier de coût d'un effet (section C.7 de la spec) : indicatif, jamais un refus. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Tier { + Light, + Medium, + Heavy, + VeryHeavy, +} + +impl Tier { + /// Les seuils, en lectures de `frame()` par pixel. Sur la RTX 4070 Ti, une lecture coûte + /// ~4 µs par frame en 1080p : léger ≤ 0,06 ms, moyen ≤ 0,14 ms, lourd ≤ 0,54 ms + /// (`shader_fx_tier_calibration`, rendering-performance.md). + pub fn of(frame_reads: u64) -> Tier { + match frame_reads { + 0..=8 => Tier::Light, + 9..=32 => Tier::Medium, + 33..=128 => Tier::Heavy, + _ => Tier::VeryHeavy, + } + } + + pub fn name(self) -> &'static str { + match self { + Tier::Light => "light", + Tier::Medium => "medium", + Tier::Heavy => "heavy", + Tier::VeryHeavy => "very-heavy", + } + } +} + +/// Ce que la validation sait d'un effet accepté. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EffectReport { + /// Lectures de texture par pixel, boucles multipliées par leur borne (le pire des branches). + pub frame_reads: u64, + /// Itérations de boucle par pixel, toutes boucles imbriquées comprises. + pub iterations: u64, + pub tier: Tier, +} + +impl ShaderParamType { + fn wgsl(self) -> &'static str { + match self { + ShaderParamType::Number | ShaderParamType::Toggle => "f32", + ShaderParamType::Color => "vec4", + ShaderParamType::Point => "vec2", + } + } + + /// (taille, alignement) en octets, règles WGSL de l'espace `uniform`. + fn size_align(self) -> (usize, usize) { + match self { + ShaderParamType::Number | ShaderParamType::Toggle => (4, 4), + ShaderParamType::Color => (16, 16), + ShaderParamType::Point => (8, 8), + } + } +} + +const RESERVED_PARAM_NAMES: &[&str] = &["unused"]; + +/// La struct `FxParams` et son binding, générés depuis les déclarations. WGSL refuse une struct +/// vide : sans paramètre, un champ `unused` en tient lieu. +fn params_source(params: &[ShaderParamDecl]) -> String { + let mut out = String::from("struct FxParams {\n"); + if params.is_empty() { + out.push_str(" unused: f32,\n"); + } + for p in params { + out.push_str(&format!(" {}: {},\n", p.name, p.ty.wgsl())); + } + out.push_str("};\n@group(0) @binding(3) var params: FxParams;\n"); + out +} + +/// Le module complet que naga compile : le code de l'agent d'abord, pour que ses lignes gardent +/// leur numéro dans les erreurs. +pub fn module_source(user: &str, params: &[ShaderParamDecl]) -> String { + format!("{user}\n{}{PRELUDE_V1}", params_source(params)) +} + +fn check_params(params: &[ShaderParamDecl]) -> Result<(), String> { + let mut seen = std::collections::HashSet::new(); + for p in params { + let valid = p.name.len() <= 32 + && p.name.starts_with(|c: char| c.is_ascii_lowercase()) + && p.name.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'); + if !valid { + return Err(format!("parameter `{}`: names are snake_case, a-z first, at most 32 characters", p.name)); + } + if RESERVED_PARAM_NAMES.contains(&p.name.as_str()) || !seen.insert(p.name.as_str()) { + return Err(format!("parameter `{}` is declared twice or reserved", p.name)); + } + } + Ok(()) +} + +/// Le module parsé, validé, et sa première fonction d'entrée de fragment. +pub struct Parsed { + pub module: naga::Module, + pub info: naga::valid::ModuleInfo, + pub report: EffectReport, +} + +/// Valide un effet comme l'agent doit le lire : paramètres, syntaxe et types (naga), bornes des +/// boucles, puis estimation du coût. L'erreur porte la ligne et la colonne du code de l'agent. +pub fn analyze(user: &str, params: &[ShaderParamDecl]) -> Result { + check_params(params)?; + let src = module_source(user, params); + let module = naga::front::wgsl::parse_str(&src).map_err(|e| e.emit_to_string(&src))?; + let info = naga::valid::Validator::new( + naga::valid::ValidationFlags::all(), + naga::valid::Capabilities::empty(), + ) + .validate(&module) + .map_err(|e| e.emit_to_string(&src))?; + // Les bindings sont ceux du prélude, et eux seuls : chaque backend lie ces quatre-là. Linux + // valide le pipeline contre ce layout, et une erreur de validation wgpu y fait paniquer. + for (handle, var) in module.global_variables.iter() { + let name = var.name.as_deref().unwrap_or("?"); + if var.binding.is_some() && !["fx_frame", "fx_sampler", "ctx", "params"].contains(&name) { + let line = module.global_variables.get_span(handle).location(&src).line_number; + return Err(format!( + "`{name}` at line {line}: an effect declares no binding of its own, it reads frame(), ctx and params" + )); + } + } + let entry = module + .entry_points + .iter() + .find(|e| e.name == "fx_fs") + .ok_or("the module has no fx_fs entry point")?; + let mut costs = CostWalk { module: &module, src: &src, by_function: Default::default() }; + let cost = costs.block(&entry.function, &entry.function.body)?; + if cost.iterations > MAX_ITERATIONS_PER_PIXEL { + return Err(format!( + "the loops run {} iterations per pixel, over the safety ceiling of {MAX_ITERATIONS_PER_PIXEL}: \ + past it a GPU can stall long enough for the system to reset it", + cost.iterations + )); + } + let report = EffectReport { frame_reads: cost.reads, iterations: cost.iterations, tier: Tier::of(cost.reads) }; + Ok(Parsed { module, info, report }) +} + +/// Ce que coûte un bloc, par pixel. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct Cost { + reads: u64, + iterations: u64, +} + +impl Cost { + fn add(self, o: Cost) -> Cost { + Cost { reads: self.reads.saturating_add(o.reads), iterations: self.iterations.saturating_add(o.iterations) } + } + fn max(self, o: Cost) -> Cost { + Cost { reads: self.reads.max(o.reads), iterations: self.iterations.max(o.iterations) } + } + fn times(self, n: u64) -> Cost { + Cost { reads: self.reads.saturating_mul(n), iterations: self.iterations.saturating_mul(n) } + } +} + +/// Parcourt l'IR de naga depuis `fx_fs` : les fonctions appelées sont dépliées, une branche +/// compte pour la plus chère, une boucle multiplie son corps par sa borne. +struct CostWalk<'a> { + module: &'a naga::Module, + src: &'a str, + by_function: std::collections::HashMap, Cost>, +} + +impl CostWalk<'_> { + fn block(&mut self, func: &naga::Function, block: &naga::Block) -> Result { + let mut total = Cost::default(); + for (at, (stmt, span)) in block.span_iter().enumerate() { + let c = match stmt { + naga::Statement::Emit(range) => { + let reads = range + .clone() + .filter(|&h| { + matches!( + func.expressions[h], + naga::Expression::ImageSample { .. } | naga::Expression::ImageLoad { .. } + ) + }) + .count() as u64; + Cost { reads, iterations: 0 } + } + naga::Statement::Block(b) => self.block(func, b)?, + naga::Statement::If { accept, reject, .. } => { + self.block(func, accept)?.max(self.block(func, reject)?) + } + naga::Statement::Switch { cases, .. } => { + let mut worst = Cost::default(); + for case in cases { + worst = worst.max(self.block(func, &case.body)?); + } + worst + } + naga::Statement::Loop { body, continuing, break_if } => { + let trips = self.trip_count(func, (block, at), body, continuing, *break_if, *span)?; + let inner = self.block(func, body)?.add(self.block(func, continuing)?); + Cost { reads: inner.reads, iterations: inner.iterations.saturating_add(1) }.times(trips) + } + naga::Statement::Call { function, .. } => self.function(*function)?, + _ => Cost::default(), + }; + total = total.add(c); + } + Ok(total) + } + + fn function(&mut self, h: naga::Handle) -> Result { + if let Some(c) = self.by_function.get(&h) { + return Ok(*c); + } + let module = self.module; + let f = &module.functions[h]; + let c = self.block(f, &f.body)?; + self.by_function.insert(h, c); + Ok(c) + } + + fn refuse(&self, span: naga::Span, why: &str) -> String { + let line = span.location(self.src).line_number; + format!( + "the loop at line {line} {why}. Write it as `for (var i = 0; i < 16; i++)`: a counter \ + initialised with a literal, compared to a literal, stepped by a literal in the loop header" + ) + } + + /// Le nombre de tours d'une boucle `for` à bornes littérales. Toute autre forme est refusée : + /// une boucle qui ne finit pas bloque le GPU jusqu'au TDR. `(parent, at)` : le bloc de la + /// boucle et son rang dans ce bloc. + fn trip_count( + &self, + func: &naga::Function, + (parent, at): (&naga::Block, usize), + body: &naga::Block, + continuing: &naga::Block, + break_if: Option>, + span: naga::Span, + ) -> Result { + use naga::{BinaryOperator as B, Expression as E, Statement as S}; + if break_if.is_some() { + return Err(self.refuse(span, "ends on `break if`")); + } + // Le `for` de WGSL devient `loop { if (cond) {} else { break; } … continuing { pas } }`. + let guard = body.iter().find(|s| !matches!(s, S::Emit(_))); + let Some(S::If { condition, accept, reject }) = guard else { + return Err(self.refuse(span, "has no bound in its header")); + }; + if !accept.is_empty() || reject.len() != 1 || !matches!(reject.iter().next(), Some(S::Break)) { + return Err(self.refuse(span, "has no bound in its header")); + } + let E::Binary { op, left, right } = func.expressions[*condition] else { + return Err(self.refuse(span, "is not bounded by a comparison")); + }; + let (var, bound, op) = match (self.counter(func, left), self.literal(func, right)) { + (Some(v), Some(k)) => (v, k, op), + _ => match (self.literal(func, left), self.counter(func, right)) { + // `N > i` se lit `i < N`. + (Some(k), Some(v)) => (v, k, match op { + B::Greater => B::Less, + B::GreaterEqual => B::LessEqual, + B::Less => B::Greater, + B::LessEqual => B::GreaterEqual, + other => other, + }), + _ => return Err(self.refuse(span, "is not bounded by a literal")), + }, + }; + let step = self + .step(func, continuing, var) + .ok_or_else(|| self.refuse(span, "does not step its counter by a literal in its header"))?; + if Self::writes(func, body, var) > 0 { + return Err(self.refuse(span, "changes its counter inside its body")); + } + // La valeur à l'entrée : le dernier `Store` qui précède la boucle dans son bloc, sinon + // l'initialisation. Elle n'est sûre que si rien d'autre n'écrit le compteur : une écriture + // dans un autre bloc, après la boucle quand son bloc se répète, ou par un pointeur, la + // changerait sans que le compte le voie, et le plafond de sécurité serait faux. + let is_var = |h: naga::Handle| { + matches!(func.expressions[h], E::LocalVariable(v) if v == var) + }; + let before: Vec<_> = parent + .iter() + .take(at) + .filter_map(|s| match s { + S::Store { pointer, value } if is_var(*pointer) => Some(*value), + _ => None, + }) + .collect(); + if Self::writes(func, &func.body, var) != before.len() + 1 { + return Err(self.refuse(span, "writes its counter outside its header")); + } + let init = match before.last() { + Some(&value) => self.literal(func, value), + None => func.local_variables[var].init.and_then(|h| self.literal(func, h)), + } + .ok_or_else(|| self.refuse(span, "starts its counter from a value that is not a literal"))?; + let trips = match op { + B::Less if step > 0.0 => ((bound - init) / step).ceil(), + B::LessEqual if step > 0.0 => ((bound - init) / step).floor() + 1.0, + B::Greater if step < 0.0 => ((init - bound) / -step).ceil(), + B::GreaterEqual if step < 0.0 => ((init - bound) / -step).floor() + 1.0, + _ => return Err(self.refuse(span, "never reaches its bound")), + }; + Ok(trips.max(0.0) as u64) + } + + /// `Load(LocalVariable(v))` : le compteur d'une condition. + fn counter(&self, func: &naga::Function, h: naga::Handle) -> Option> { + match func.expressions[h] { + naga::Expression::Load { pointer } => match func.expressions[pointer] { + naga::Expression::LocalVariable(v) => Some(v), + _ => None, + }, + _ => None, + } + } + + /// Un littéral, directement ou par une constante de module. + fn literal(&self, func: &naga::Function, h: naga::Handle) -> Option { + let lit = |e: &naga::Expression| match *e { + naga::Expression::Literal(l) => match l { + naga::Literal::I32(v) => Some(v as f64), + naga::Literal::U32(v) => Some(v as f64), + naga::Literal::F32(v) => Some(v as f64), + naga::Literal::I64(v) => Some(v as f64), + naga::Literal::U64(v) => Some(v as f64), + naga::Literal::F64(v) => Some(v), + naga::Literal::AbstractInt(v) => Some(v as f64), + naga::Literal::AbstractFloat(v) => Some(v), + _ => None, + }, + _ => None, + }; + match &func.expressions[h] { + naga::Expression::Constant(c) => lit(&self.module.global_expressions[self.module.constants[*c].init]), + e => lit(e), + } + } + + /// Le pas du compteur dans `continuing` : `i = i + k` ou `i = i - k`, `k` littéral. + fn step(&self, func: &naga::Function, continuing: &naga::Block, var: naga::Handle) -> Option { + continuing.iter().find_map(|s| match s { + naga::Statement::Store { pointer, value } + if matches!(func.expressions[*pointer], naga::Expression::LocalVariable(v) if v == var) => + { + match func.expressions[*value] { + naga::Expression::Binary { op, left, right } if self.counter(func, left) == Some(var) => { + let k = self.literal(func, right)?; + match op { + naga::BinaryOperator::Add => Some(k), + naga::BinaryOperator::Subtract => Some(-k), + _ => None, + } + } + _ => None, + } + } + _ => None, + }) + } + + /// Combien d'instructions du bloc écrivent le compteur, directement ou en passant son pointeur + /// à une fonction, blocs imbriqués compris. + fn writes(func: &naga::Function, block: &naga::Block, var: naga::Handle) -> usize { + let is_var = |h: naga::Handle| { + matches!(func.expressions[h], naga::Expression::LocalVariable(v) if v == var) + }; + block + .iter() + .map(|s| match s { + naga::Statement::Store { pointer, .. } => usize::from(is_var(*pointer)), + naga::Statement::Call { arguments, .. } => usize::from(arguments.iter().any(|&a| is_var(a))), + naga::Statement::Block(b) => Self::writes(func, b, var), + naga::Statement::If { accept, reject, .. } => { + Self::writes(func, accept, var) + Self::writes(func, reject, var) + } + naga::Statement::Switch { cases, .. } => cases.iter().map(|c| Self::writes(func, &c.body, var)).sum(), + naga::Statement::Loop { body, continuing, .. } => { + Self::writes(func, body, var) + Self::writes(func, continuing, var) + } + _ => 0, + }) + .sum() + } +} + +/// HLSL SM 5.0 pour D3D11/FXC. naga 24 est la dernière version dont la sortie HLSL passe FXC : +/// à partir de la 25 elle déclare les samplers dans un tas D3D12 (`register(s0, space0)`), que +/// `ps_5_0` refuse (X3721). Ne pas monter naga sans faire passer ce module. +pub fn to_hlsl(parsed: &Parsed) -> Result { + let options = naga::back::hlsl::Options { + shader_model: naga::back::hlsl::ShaderModel::V5_0, + ..Default::default() + }; + let mut out = String::new(); + naga::back::hlsl::Writer::new(&mut out, &options) + .write(&parsed.module, &parsed.info, None) + .map_err(|e| e.to_string())?; + Ok(out) +} + +/// Les index de buffer Metal de `ctx` et de `params`, ceux que `to_msl` donne au module et que +/// `compositor_macos::apply_shader_effect` passe à `set_fragment_bytes`. Les mêmes numéros que les +/// registres HLSL (b2, b3). +#[cfg(target_os = "macos")] +pub const MSL_CTX_BUFFER: u64 = 2; +#[cfg(target_os = "macos")] +pub const MSL_PARAMS_BUFFER: u64 = 3; + +/// Le module traduit en MSL, et les noms que naga a donnés à ses deux étages. +#[cfg(target_os = "macos")] +pub struct Msl { + pub source: String, + pub vs: String, + pub fs: String, +} + +/// MSL pour Metal. Les bindings du prélude sont liés explicitement : la texture en 0, le sampler +/// en `constexpr` (linéaire, bords répétés, comme `sampNV` de `shaders.metal` : le dépôt n'a aucun +/// `MTLSamplerState`), `ctx` et `params` en buffers. Sans cette carte, naga invente les bindings +/// (`fake_missing_bindings`) et émet `[[user(fake0)]]`, qui n'est pas du MSL. +#[cfg(target_os = "macos")] +pub fn to_msl(parsed: &Parsed) -> Result { + use naga::back::msl; + use naga::proc::BoundsCheckPolicy::ReadZeroSkipWrite; + let at = |binding| naga::ResourceBinding { group: 0, binding }; + let resources = msl::BindingMap::from([ + (at(0), msl::BindTarget { texture: Some(0), ..Default::default() }), + (at(1), msl::BindTarget { sampler: Some(msl::BindSamplerTarget::Inline(0)), ..Default::default() }), + (at(2), msl::BindTarget { buffer: Some(MSL_CTX_BUFFER as u8), ..Default::default() }), + (at(3), msl::BindTarget { buffer: Some(MSL_PARAMS_BUFFER as u8), ..Default::default() }), + ]); + let options = msl::Options { + lang_version: (2, 0), + per_entry_point_map: ["fx_vs", "fx_fs"] + .into_iter() + .map(|name| { + (name.to_string(), msl::EntryPointResources { resources: resources.clone(), ..Default::default() }) + }) + .collect(), + inline_samplers: vec![msl::sampler::InlineSampler { + address: [msl::sampler::Address::ClampToEdge; 3], + mag_filter: msl::sampler::Filter::Linear, + min_filter: msl::sampler::Filter::Linear, + ..Default::default() + }], + fake_missing_bindings: false, + // naga borne sinon chaque boucle par une lecture `volatile` à chaque tour, contre un + // compilateur qui supposerait qu'elle finit. `analyze` l'a déjà prouvé (bornes + // littérales, 4 096 itérations au plus) : la garde coûtait 7 à 10 % de la passe sur M1. + force_loop_bounding: false, + // Un index hors bornes lit 0, comme sous D3D11, au lieu de lire au-delà du buffer. + bounds_check_policies: naga::proc::BoundsCheckPolicies { + index: ReadZeroSkipWrite, + buffer: ReadZeroSkipWrite, + image_load: ReadZeroSkipWrite, + binding_array: ReadZeroSkipWrite, + }, + ..Default::default() + }; + let (source, info) = + msl::write_string(&parsed.module, &parsed.info, &options, &msl::PipelineOptions::default()) + .map_err(|e| format!("MSL: {e}"))?; + let name = |entry: &str| -> Result { + let i = parsed + .module + .entry_points + .iter() + .position(|e| e.name == entry) + .ok_or_else(|| format!("the module has no {entry} entry point"))?; + info.entry_point_names[i].clone().map_err(|e| format!("MSL {entry}: {e}")) + }; + Ok(Msl { vs: name("fx_vs")?, fs: name("fx_fs")?, source }) +} + +/// La pipeline de l'effet, prête pour la passe : MSL compilé par Metal, puis le pipeline state +/// RGBA8 sans blending (l'effet rend le pixel final). +#[cfg(target_os = "macos")] +pub type CompiledFx = metal::RenderPipelineState; + +/// Compile pour Metal. Hors du thread de rendu, donc sous un pool d'autorelease à soi : un thread +/// lancé par Rust n'en a pas, et les objets autoreleased de Metal (l'erreur de compilation, les +/// fonctions de la bibliothèque) fuiraient. +#[cfg(target_os = "macos")] +pub fn compile_metal(device: &metal::Device, code: &str, params: &[ShaderParamDecl]) -> Result { + let msl = to_msl(&analyze(code, params)?)?; + objc::rc::autoreleasepool(|| { + let library = device + .new_library_with_source(&msl.source, &metal::CompileOptions::new()) + .map_err(|e| format!("Metal: {e}"))?; + crate::compositor_macos::make_pipeline( + device, + &library, + &msl.vs, + &msl.fs, + metal::MTLPixelFormat::RGBA8Unorm, + crate::compositor_macos::Blend::Replace, + ) + .map_err(|e| format!("Metal: {e:#}")) + }) +} + +/// Compile sur un thread à part ; le résultat arrive par le canal. Le pendant de la version +/// Windows : `Device`, `Library` et `RenderPipelineState` de metal-rs sont `Send` et `Sync`. +#[cfg(target_os = "macos")] +pub fn spawn_compile( + effect: &SceneShaderEffect, + device: &metal::Device, +) -> std::sync::mpsc::Receiver> { + let (tx, rx) = std::sync::mpsc::channel(); + let (code, params, device) = (effect.code.clone(), effect.params.clone(), device.clone()); + std::thread::Builder::new() + .name("shader-fx".into()) + .spawn(move || { + let _ = tx.send(compile_metal(&device, &code, ¶ms)); + }) + .expect("thread shader-fx"); + rx +} + +/// Clé du cache des shaders compilés : ce qui change le binaire, et rien d'autre. +pub fn source_hash(effect: &SceneShaderEffect) -> u64 { + let mut h = std::collections::hash_map::DefaultHasher::new(); + effect.contract.hash(&mut h); + effect.code.hash(&mut h); + for p in &effect.params { + p.name.hash(&mut h); + p.ty.hash(&mut h); + } + h.finish() +} + +/// L'entrée à évincer d'un cache plein, parmi `(clé, dernier usage)` : la moins récemment utilisée +/// de celles que la scène n'emploie pas. Aucune si elle les emploie toutes : le cache dépasse alors +/// sa taille plutôt que de recompiler à chaque frame un effet qu'elle montre. +pub fn eviction_victim(entries: impl Iterator, scene: &[SceneShaderEffect]) -> Option { + let shown: std::collections::HashSet = scene.iter().map(source_hash).collect(); + entries.filter(|(key, _)| !shown.contains(key)).min_by_key(|&(_, used)| used).map(|(key, _)| key) +} + +/// Les régions actives au temps source `t`. Le document n'en tient qu'une à la fois ; une scène +/// écrite à la main qui en superpose les applique dans son ordre. +pub fn active_effects(effects: &[SceneShaderEffect], t: f32) -> Vec<&SceneShaderEffect> { + // En f32 comme `t` et comme les annotations : `0.7_f32 as f64` tombe sous `0.7_f64`, et la + // frame posée sur le début de la région la manquerait. + effects.iter().filter(|e| t >= e.start_sec as f32 && t < e.end_sec as f32).collect() +} + +/// La boîte englobante (x, y, w, h) d'un quad, en uv de sortie : `ctx.footage` sous un tilt 3D. +pub fn bounds(corners: &[[f32; 2]; 4]) -> [f32; 4] { + let (mut x0, mut y0, mut x1, mut y1) = (f32::MAX, f32::MAX, f32::MIN, f32::MIN); + for [x, y] in corners { + (x0, y0, x1, y1) = (x0.min(*x), y0.min(*y), x1.max(*x), y1.max(*y)); + } + [x0, y0, x1 - x0, y1 - y0] +} + +/// `ctx.region` : progression sur la pilule entière en temps programme quand l'app la donne, sur +/// le fragment en temps source sinon. +fn region(effect: &SceneShaderEffect, source_t: f32, programme_t: f32) -> [f32; 4] { + let (start, end, now) = match (effect.programme_start_sec, effect.programme_end_sec) { + (Some(s), Some(e)) => (s, e, programme_t as f64), + _ => (effect.start_sec, effect.end_sec, source_t as f64), + }; + let span = (end - start).max(1e-6); + let since = (now - start).max(0.0); + [(since / span).clamp(0.0, 1.0) as f32, span as f32, since as f32, 0.0] +} + +pub fn context( + effect: &SceneShaderEffect, + source_t: f32, + programme_t: f32, + resolution: [f32; 2], + inputs: &FxFrameInputs, +) -> FxContext { + let [tl, tr, br, bl] = inputs.footage.corners; + let mut clicks = [[0.0; 4]; MAX_CLICKS]; + for (slot, ([x, y], age)) in clicks.iter_mut().zip(&inputs.clicks) { + *slot = [*x, *y, *age, 1.0]; + } + FxContext { + resolution, + time: programme_t, + aspect: resolution[0] / resolution[1].max(1.0), + region: region(effect, source_t, programme_t), + cursor: inputs.cursor.map_or([0.0; 4], |[x, y]| [x, y, 1.0, 0.0]), + clicks, + footage: bounds(&inputs.footage.corners), + camera: inputs.camera.unwrap_or([0.0; 4]), + footage_corners: [[tl[0], tl[1], tr[0], tr[1]], [br[0], br[1], bl[0], bl[1]]], + footage_aspect: inputs.footage_aspect, + footage_warp: if inputs.footage.projective { 1.0 } else { 0.0 }, + _pad: [0.0; 2], + } +} + +/// Les paramètres de la région, rangés comme la struct `FxParams` générée (règles d'alignement de +/// WGSL), en mots de 4 octets. Une valeur absente vaut 0 : l'app envoie déjà les défauts. +pub fn pack_params(effect: &SceneShaderEffect) -> Vec { + let mut words: Vec = Vec::new(); + let mut offset = 0usize; + for p in &effect.params { + let (size, align) = p.ty.size_align(); + offset = offset.div_ceil(align) * align; + let end = offset + size; + words.resize(end / 4, 0.0); + let value: Vec = match effect.values.get(&p.name) { + Some(ShaderParamValue::Scalar(v)) => vec![*v], + Some(ShaderParamValue::Vector(v)) => v.clone(), + None => Vec::new(), + }; + for (i, v) in value.into_iter().take(size / 4).enumerate() { + words[offset / 4 + i] = v; + } + offset = end; + } + words.resize(offset.div_ceil(16).max(1) * 4, 0.0); + words +} + +/// Le bytecode DXBC des deux étages, prêt pour `CreateVertexShader` / `CreatePixelShader`. +#[cfg(windows)] +pub struct CompiledFx { + pub vs: Vec, + pub ps: Vec, +} + +#[cfg(windows)] +pub fn compile_dxbc(code: &str, params: &[ShaderParamDecl]) -> Result { + let parsed = analyze(code, params)?; + let hlsl = to_hlsl(&parsed)?; + Ok(CompiledFx { + vs: d3d_compile(&hlsl, "fx_vs", "vs_5_0")?, + ps: d3d_compile(&hlsl, "fx_fs", "ps_5_0")?, + }) +} + +#[cfg(windows)] +fn d3d_compile(hlsl: &str, entry: &str, profile: &str) -> Result, String> { + use windows::core::PCSTR; + use windows::Win32::Graphics::Direct3D::Fxc::{D3DCompile, D3DCOMPILE_OPTIMIZATION_LEVEL3}; + use windows::Win32::Graphics::Direct3D::ID3DBlob; + + fn bytes(blob: &ID3DBlob) -> Vec { + unsafe { + std::slice::from_raw_parts(blob.GetBufferPointer() as *const u8, blob.GetBufferSize()) + .to_vec() + } + } + let (entry_z, profile_z) = (format!("{entry}\0"), format!("{profile}\0")); + let (mut code, mut errors): (Option, Option) = (None, None); + let compiled = unsafe { + D3DCompile( + hlsl.as_ptr().cast(), + hlsl.len(), + PCSTR(b"effect.hlsl\0".as_ptr()), + None, + None, + PCSTR(entry_z.as_ptr()), + PCSTR(profile_z.as_ptr()), + D3DCOMPILE_OPTIMIZATION_LEVEL3, + 0, + &mut code, + Some(&mut errors), + ) + }; + match (compiled, code) { + (Ok(()), Some(code)) => Ok(bytes(&code)), + (result, _) => { + let log = errors.as_ref().map(bytes).unwrap_or_default(); + Err(format!("{entry}: {result:?}\n{}", String::from_utf8_lossy(&log))) + } + } +} + +/// Compile sur un thread à part ; le résultat arrive par le canal. naga + FXC prennent ~10 ms +/// pour un effet de 60 lignes, soit une frame de preview : jamais sur le thread de rendu. +#[cfg(windows)] +pub fn spawn_compile(effect: &SceneShaderEffect) -> std::sync::mpsc::Receiver> { + let (tx, rx) = std::sync::mpsc::channel(); + let (code, params) = (effect.code.clone(), effect.params.clone()); + std::thread::Builder::new() + .name("shader-fx".into()) + .spawn(move || { + let _ = tx.send(compile_dxbc(&code, ¶ms)); + }) + .expect("thread shader-fx"); + rx +} + +/// Le SPIR-V que wgpu tirera du même module pour Vulkan. Le compositeur Linux passe le WGSL à +/// wgpu, qui refait cette traduction lui-même : celle-ci ne sert qu'à la vérifier d'avance. +#[cfg(target_os = "linux")] +fn to_spirv(parsed: &Parsed) -> Result, String> { + naga::back::spv::write_vec(&parsed.module, &parsed.info, &naga::back::spv::Options::default(), None) + .map_err(|e| e.to_string()) +} + +/// Ce que l'outil de l'agent demande avant d'accepter un code : la validation, la compilation +/// réelle pour le backend de la machine quand elle existe, et le palier. +pub fn check_for_agent(code: &str, params: &[ShaderParamDecl]) -> Result { + let parsed = analyze(code, params)?; + #[cfg(windows)] + compile_dxbc(code, params)?; + #[cfg(target_os = "linux")] + to_spirv(&parsed)?; + #[cfg(target_os = "macos")] + compile_metal(&metal::Device::system_default().ok_or("no Metal device")?, code, params)?; + Ok(parsed.report) +} + +/// `check_for_agent`, les paramètres reçus en JSON (`[{ "name", "type" }, …]`) depuis l'addon. +pub fn check_for_agent_json(code: &str, params_json: &str) -> Result { + let params: Vec = + serde_json::from_str(params_json).map_err(|e| format!("params: {e}"))?; + check_for_agent(code, ¶ms) +} + +/// Un effet tel que les tests et les bancs le construisent. +#[cfg(test)] +pub(crate) fn test_effect(code: &str, params: Vec, values: &[(&str, ShaderParamValue)]) -> SceneShaderEffect { + SceneShaderEffect { + clip_index: Some(0), + start_sec: 1.0, + end_sec: 3.0, + programme_start_sec: None, + programme_end_sec: None, + target: ShaderEffectTarget::Image, + contract: CONTRACT, + code: code.into(), + params, + values: values.iter().map(|(k, v)| (k.to_string(), v.clone())).collect(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn decl(name: &str, ty: ShaderParamType) -> ShaderParamDecl { + ShaderParamDecl { name: name.into(), ty } + } + + const VIGNETTE: &str = "fn effect(uv: vec2) -> vec4 {\n let c = frame(uv);\n let d = distance(uv, vec2(0.5));\n return vec4(c.rgb * (1.0 - d * params.strength * ctx.region.x), c.a);\n}\n"; + + #[test] + fn a_valid_effect_becomes_hlsl_that_d3d11_accepts() { + let parsed = analyze(VIGNETTE, &[decl("strength", ShaderParamType::Number)]).expect("analyze"); + let hlsl = to_hlsl(&parsed).expect("translate"); + assert!(hlsl.contains("fx_vs") && hlsl.contains("fx_fs")); + // Les registres que les backends lient : t0, s1, b2 (ctx), b3 (params), sans `space`. + for reg in ["register(t0)", "register(s1)", "register(b2)", "register(b3)"] { + assert!(hlsl.contains(reg), "{reg} missing:\n{hlsl}"); + } + assert!(!hlsl.contains("space"), "FXC ps_5_0 refuse les espaces de registres"); + } + + #[test] + fn an_error_points_at_the_line_the_agent_wrote() { + let broken = "fn effect(uv: vec2) -> vec4 {\n return frame(uv) * nope;\n}\n"; + let err = analyze(broken, &[]).err().expect("must fail"); + assert!(err.contains("nope") && err.contains(":2:"), "{err}"); + } + + #[test] + fn an_effect_without_the_entry_function_is_refused() { + assert!(analyze("fn other() {}", &[]).is_err()); + } + + #[test] + fn an_effect_declares_no_binding_of_its_own() { + let code = "@group(0) @binding(4) var extra: texture_2d;\n\ + fn effect(uv: vec2) -> vec4 { return textureLoad(extra, vec2(0), 0); }\n"; + let err = analyze(code, &[]).err().expect("refusé"); + assert!(err.contains("`extra` at line 1"), "{err}"); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_valid_effect_becomes_spirv() { + let parsed = analyze(VIGNETTE, &[decl("strength", ShaderParamType::Number)]).expect("analyze"); + assert!(!to_spirv(&parsed).expect("spirv").is_empty()); + } + + #[test] + fn parameter_names_are_checked() { + for bad in ["Strength", "2x", "unused", "a-b"] { + assert!(analyze(VIGNETTE, &[decl(bad, ShaderParamType::Number)]).is_err(), "{bad}"); + } + let twice = [decl("a", ShaderParamType::Number), decl("a", ShaderParamType::Color)]; + assert!(analyze(VIGNETTE, &twice).is_err()); + } + + /// Le packer et la struct générée doivent placer chaque paramètre au même octet : un écart lirait + /// une couleur là où l'app a écrit une force. + #[test] + fn the_packer_matches_the_offsets_naga_gives_the_generated_struct() { + let params = vec![ + decl("a", ShaderParamType::Number), + decl("tint", ShaderParamType::Color), + decl("on", ShaderParamType::Toggle), + decl("at", ShaderParamType::Point), + decl("b", ShaderParamType::Number), + ]; + let code = "fn effect(uv: vec2) -> vec4 { return frame(uv) * params.a + params.tint * params.on + vec4(params.at, params.b, 0.0); }"; + let parsed = analyze(code, ¶ms).expect("analyze"); + let ty = parsed + .module + .types + .iter() + .find(|(_, t)| t.name.as_deref() == Some("FxParams")) + .expect("FxParams") + .1; + let naga::TypeInner::Struct { members, span } = &ty.inner else { panic!("struct") }; + let values = [ + ("a", ShaderParamValue::Scalar(1.0)), + ("tint", ShaderParamValue::Vector(vec![2.0, 3.0, 4.0, 5.0])), + ("on", ShaderParamValue::Scalar(6.0)), + ("at", ShaderParamValue::Vector(vec![7.0, 8.0])), + ("b", ShaderParamValue::Scalar(9.0)), + ]; + let words = pack_params(&test_effect(code, params, &values)); + assert_eq!(words.len() * 4, (*span as usize).div_ceil(16) * 16); + let first = |name: &str| members.iter().find(|m| m.name.as_deref() == Some(name)).unwrap().offset as usize / 4; + assert_eq!(words[first("a")], 1.0); + assert_eq!(&words[first("tint")..first("tint") + 4], &[2.0, 3.0, 4.0, 5.0]); + assert_eq!(words[first("on")], 6.0); + assert_eq!(&words[first("at")..first("at") + 2], &[7.0, 8.0]); + assert_eq!(words[first("b")], 9.0); + } + + #[test] + fn the_context_matches_its_wgsl_struct() { + let parsed = analyze(VIGNETTE, &[decl("strength", ShaderParamType::Number)]).unwrap(); + let ty = parsed.module.types.iter().find(|(_, t)| t.name.as_deref() == Some("FxContext")).unwrap().1; + let naga::TypeInner::Struct { members, span } = &ty.inner else { panic!("struct") }; + assert_eq!(std::mem::size_of::(), *span as usize); + let offset = |name: &str| members.iter().find(|m| m.name.as_deref() == Some(name)).unwrap().offset as usize; + let c = FxContext::default(); + let base = &c as *const FxContext as usize; + assert_eq!(offset("region"), &c.region as *const _ as usize - base); + assert_eq!(offset("clicks"), &c.clicks as *const _ as usize - base); + assert_eq!(offset("camera"), &c.camera as *const _ as usize - base); + assert_eq!(offset("footage_corners"), &c.footage_corners as *const _ as usize - base); + assert_eq!(offset("footage_warp"), &c.footage_warp as *const _ as usize - base); + } + + #[test] + fn a_for_loop_with_literal_bounds_is_counted() { + let code = "fn effect(uv: vec2) -> vec4 {\n var acc = vec4(0.0);\n for (var i = 0; i < 16; i++) {\n acc += frame(uv + vec2(f32(i) * 0.001, 0.0));\n }\n return acc / 16.0;\n}\n"; + let r = analyze(code, &[]).expect("analyze").report; + assert_eq!((r.frame_reads, r.iterations, r.tier), (16, 16, Tier::Medium)); + } + + #[test] + fn nested_loops_multiply_and_called_functions_are_unfolded() { + let code = "fn tap(uv: vec2) -> vec4 { return frame(uv) + frame(uv * 0.5); }\nfn effect(uv: vec2) -> vec4 {\n var acc = vec4(0.0);\n for (var y = 0u; y <= 7u; y += 1u) {\n for (var x = 8; x > 0; x--) {\n acc += tap(uv);\n }\n }\n return acc;\n}\n"; + let r = analyze(code, &[]).expect("analyze").report; + assert_eq!(r.frame_reads, 8 * 8 * 2); + assert_eq!(r.iterations, 8 * (8 + 1)); + assert_eq!(r.tier, Tier::Heavy); + } + + #[test] + fn a_branch_counts_for_the_dearer_side() { + let code = "fn effect(uv: vec2) -> vec4 {\n if (uv.x > 0.5) { return frame(uv) + frame(uv) + frame(uv); }\n return frame(uv);\n}\n"; + assert_eq!(analyze(code, &[]).unwrap().report.frame_reads, 4, "branche chère + suite"); + } + + #[test] + fn loops_without_a_literal_bound_are_refused_with_their_line() { + let refused = [ + ("fn effect(uv: vec2) -> vec4 {\n var i = 0;\n loop {\n i++;\n if (i > 4) { break; }\n }\n return frame(uv);\n}\n", 3), + ("fn effect(uv: vec2) -> vec4 {\n var i = 0;\n while (i < 4) {\n i++;\n }\n return frame(uv);\n}\n", 3), + ("fn effect(uv: vec2) -> vec4 {\n let n = i32(uv.x * 100.0);\n for (var i = 0; i < n; i++) {}\n return frame(uv);\n}\n", 3), + ("fn effect(uv: vec2) -> vec4 {\n for (var i = 0; i < 4; i++) { i = 0; }\n return frame(uv);\n}\n", 2), + ("fn effect(uv: vec2) -> vec4 {\n for (var i = 0; i < 4; i--) {}\n return frame(uv);\n}\n", 2), + ]; + for (code, line) in refused { + let err = analyze(code, &[]).err().unwrap_or_else(|| panic!("accepted: {code}")); + assert!(err.contains(&format!("line {line}")) && err.contains("for (var i = 0"), "{err}"); + } + } + + /// Le compteur part de la valeur qu'il a à l'entrée de SA boucle : le dernier `Store` qui la + /// précède dans son bloc, pas une écriture plus loin. Une écriture ailleurs le rendrait + /// inconnu : refusée, pour que le plafond de sécurité ne compte jamais trop peu de tours. + #[test] + fn the_counter_starts_from_its_value_at_the_loop() { + let wrap = |body: &str| format!("fn effect(uv: vec2) -> vec4 {{\n var i = 0;\n{body}\n return frame(uv);\n}}\n"); + let reused = wrap(" i = 2;\n for (; i < 6; i++) {}"); + assert_eq!(analyze(&reused, &[]).expect("analyze").report.iterations, 4); + let refused = [ + // Écrit après la boucle : si son bloc se répète, la valeur d'entrée n'est plus 0. + wrap(" for (; i < 4; i++) {}\n i = -100000;"), + // Écrit dans un autre bloc avant la boucle : 100 004 tours, pas 4. + wrap(" if (uv.x > 0.5) { i = -100000; }\n for (; i < 4; i++) {}"), + // Deux boucles sur le même compteur : chacune voit le pas de l'autre. + wrap(" for (; i < 4; i++) { }\n for (; i < 8; i++) { }"), + ]; + for code in refused { + let err = analyze(&code, &[]).err().unwrap_or_else(|| panic!("accepted: {code}")); + assert!(err.contains("writes its counter outside its header"), "{err}"); + } + } + + #[test] + fn a_module_constant_is_a_literal_bound() { + let code = "const TAPS = 12;\nfn effect(uv: vec2) -> vec4 {\n var acc = vec4(0.0);\n for (var i = 0; i < TAPS; i += 3) { acc += frame(uv); }\n return acc;\n}\n"; + assert_eq!(analyze(code, &[]).expect("analyze").report.iterations, 4); + } + + #[test] + fn the_safety_ceiling_refuses_runaway_work() { + let code = "fn effect(uv: vec2) -> vec4 {\n var a = 0.0;\n for (var i = 0; i < 100; i++) { for (var j = 0; j < 100; j++) { a += 1.0; } }\n return frame(uv) * a;\n}\n"; + let err = analyze(code, &[]).err().expect("must fail"); + assert!(err.contains("10100") && err.contains("4096"), "{err}"); + } + + /// Le ripple de l'exemple C.6 de la spec, et ses paramètres. + fn spec_ripple() -> (&'static str, Vec) { + let code = include_str!("../../../docs/shader-effects.md") + .split("```wgsl\n") + .nth(2) + .and_then(|s| s.split("```").next()) + .expect("l'exemple C.6"); + let params = vec![ + decl("strength", ShaderParamType::Number), + decl("radius", ShaderParamType::Number), + decl("duration", ShaderParamType::Number), + ]; + (code, params) + } + + #[test] + fn the_ripple_of_the_spec_is_light() { + let (code, params) = spec_ripple(); + let r = analyze(code, ¶ms).expect("l'exemple de la spec compile").report; + assert_eq!((r.frame_reads, r.tier), (1, Tier::Light)); + } + + /// The examples agents read in the MCP resource (`electron/mcp/effects-contract.ts`) compile + /// against this prelude, at the tier the resource promises. + #[test] + fn the_examples_of_the_mcp_contract_compile() { + let [(ripple, ripple_params), (vignette, vignette_params)] = mcp_examples(); + assert_eq!(analyze(&ripple, &ripple_params).expect("ripple").report.tier, Tier::Light); + assert_eq!(analyze(&vignette, &vignette_params).expect("vignette").report.tier, Tier::Light); + } + + /// Les deux exemples de la ressource MCP (`electron/mcp/effects-contract.ts`), le ripple et la + /// vignette, avec leurs paramètres. + fn mcp_examples() -> [(String, Vec); 2] { + let src = include_str!("../../../electron/mcp/effects-contract.ts"); + let examples: Vec = src + .split("## Example") + .skip(1) + .map(|section| { + section + .lines() + .filter_map(|l| l.strip_prefix(" ")) + .collect::>() + .join("\n") + }) + .collect(); + let [ripple, vignette]: [String; 2] = examples.try_into().expect("le ripple et la vignette"); + [ + ( + ripple, + vec![ + decl("strength", ShaderParamType::Number), + decl("radius", ShaderParamType::Number), + decl("duration", ShaderParamType::Number), + ], + ), + (vignette, vec![decl("depth", ShaderParamType::Number), decl("tint", ShaderParamType::Color)]), + ] + } + + #[test] + fn the_active_regions_and_their_progress() { + let mut a = test_effect(VIGNETTE, vec![], &[]); + (a.start_sec, a.end_sec) = (3.0, 4.0); + let b = test_effect(VIGNETTE, vec![], &[]); + let effects = [a, b]; + assert!(active_effects(&effects, 0.5).is_empty()); + let during = active_effects(&effects, 2.0); + assert!(during.len() == 1 && std::ptr::eq(during[0], &effects[1])); + let after = active_effects(&effects, 3.0); + assert!(after.len() == 1 && std::ptr::eq(after[0], &effects[0]), "fin exclusive"); + let mut c = test_effect(VIGNETTE, vec![], &[]); + (c.start_sec, c.end_sec) = (0.7, 1.3); + let fractional = [c]; + assert_eq!(active_effects(&fractional, 0.7).len(), 1, "début fractionnaire inclus"); + assert!(active_effects(&fractional, 1.3).is_empty(), "fin fractionnaire exclue"); + + let ctx = context(&effects[1], 2.0, 10.0, [1920.0, 1080.0], &FxFrameInputs::default()); + assert_eq!(ctx.region, [0.5, 2.0, 1.0, 0.0], "fragment, temps source"); + let mut whole = effects[1].clone(); + (whole.programme_start_sec, whole.programme_end_sec) = (Some(8.0), Some(12.0)); + let ctx = context(&whole, 2.0, 10.0, [1920.0, 1080.0], &FxFrameInputs::default()); + assert_eq!(ctx.region, [0.5, 4.0, 2.0, 0.0], "pilule entière, temps programme"); + assert_eq!((ctx.cursor[2], ctx.clicks[0][3], ctx.camera), (0.0, 0.0, [0.0; 4])); + + let inputs = FxFrameInputs { + cursor: Some([0.25, 0.5]), + clicks: vec![([0.3, 0.6], 0.2), ([0.1, 0.1], 1.5)], + footage: FootageQuad { + corners: [[0.25, 0.5], [0.75, 0.25], [0.625, 0.75], [0.25, 0.625]], + projective: true, + }, + footage_aspect: 16.0 / 9.0, + camera: Some([0.7, 0.7, 0.2, 0.2]), + }; + let ctx = context(&whole, 2.0, 10.0, [1920.0, 1080.0], &inputs); + assert_eq!(ctx.cursor, [0.25, 0.5, 1.0, 0.0]); + assert_eq!(ctx.clicks[..3], [[0.3, 0.6, 0.2, 1.0], [0.1, 0.1, 1.5, 1.0], [0.0; 4]]); + assert_eq!((ctx.footage, ctx.camera), ([0.25, 0.25, 0.5, 0.5], [0.7, 0.7, 0.2, 0.2])); + assert_eq!(ctx.footage_corners, [[0.25, 0.5, 0.75, 0.25], [0.625, 0.75, 0.25, 0.625]]); + assert_eq!((ctx.footage_aspect, ctx.footage_warp), (16.0 / 9.0, 1.0)); + } + + #[test] + fn eviction_spares_the_effects_the_scene_shows() { + let scene = [test_effect(VIGNETTE, vec![], &[]), test_effect("fn effect(uv: vec2) -> vec4 { return frame(uv); }", vec![], &[])]; + let (a, b) = (source_hash(&scene[0]), source_hash(&scene[1])); + let entries = || [(a, 1), (b, 2), (7, 5), (9, 3)].into_iter(); + assert_eq!(eviction_victim(entries(), &scene), Some(9), "la plus ancienne hors de la scène"); + assert_eq!(eviction_victim(entries(), &[]), Some(a), "sans scène, la plus ancienne"); + assert_eq!(eviction_victim([(a, 1), (b, 2)].into_iter(), &scene), None, "tout est montré"); + } + + #[test] + fn the_cache_key_follows_code_and_layout_not_values() { + let p = vec![decl("strength", ShaderParamType::Number)]; + let a = test_effect(VIGNETTE, p.clone(), &[("strength", ShaderParamValue::Scalar(1.0))]); + let mut b = test_effect(VIGNETTE, p, &[("strength", ShaderParamValue::Scalar(2.0))]); + assert_eq!(source_hash(&a), source_hash(&b), "une valeur ne recompile rien"); + b.params[0].ty = ShaderParamType::Color; + assert_ne!(source_hash(&a), source_hash(&b), "un layout différent est un autre binaire"); + } + + #[cfg(windows)] + #[test] + fn fxc_compiles_both_stages() { + let c = compile_dxbc(VIGNETTE, &[decl("strength", ShaderParamType::Number)]).expect("compile"); + assert!(!c.vs.is_empty() && !c.ps.is_empty()); + } + + /// Les bindings que `compositor_macos` lie sont ceux du MSL : la texture en 0, `ctx` et + /// `params` aux buffers 2 et 3, le sampler en `constexpr`, et aucun binding inventé. + #[cfg(target_os = "macos")] + #[test] + fn a_valid_effect_becomes_msl_with_the_bindings_metal_binds() { + let parsed = analyze(VIGNETTE, &[decl("strength", ShaderParamType::Number)]).expect("analyze"); + let msl = to_msl(&parsed).expect("translate"); + assert!(msl.source.contains(&msl.vs) && msl.source.contains(&msl.fs)); + for binding in ["[[texture(0)]]", "[[buffer(2)]]", "[[buffer(3)]]", "constexpr metal::sampler"] { + assert!(msl.source.contains(binding), "{binding} missing:\n{}", msl.source); + } + assert!(!msl.source.contains("fake"), "binding inventé :\n{}", msl.source); + } + + /// « Accepté » veut dire « se dessinera » : le compilateur de Metal et le pipeline state + /// acceptent l'exemple de la spec, les deux de la ressource MCP et un effet qui lit chaque + /// type de paramètre. + #[cfg(target_os = "macos")] + #[test] + fn metal_compiles_the_examples() { + let Some(device) = metal::Device::system_default() else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let (ripple, ripple_params) = spec_ripple(); + compile_metal(&device, ripple, &ripple_params).expect("le ripple de la spec"); + for (code, params) in mcp_examples() { + compile_metal(&device, &code, ¶ms).expect("un exemple de la ressource MCP"); + } + let every_type = [ + decl("a", ShaderParamType::Number), + decl("tint", ShaderParamType::Color), + decl("on", ShaderParamType::Toggle), + decl("at", ShaderParamType::Point), + ]; + let code = "fn effect(uv: vec2) -> vec4 {\n let n = fx_hash(vec2(uv * ctx.resolution));\n let d = distance(uv, params.at) * ctx.aspect + fx_px(4.0) + n * 0.0;\n return mix(frame(uv), params.tint, params.on * params.a * ctx.clicks[0].w * d);\n}\n"; + compile_metal(&device, code, &every_type).expect("chaque type de paramètre"); + check_for_agent(VIGNETTE, &[decl("strength", ShaderParamType::Number)]).expect("check_for_agent"); + } +} diff --git a/crates/compositor/src/timeline_walk.rs b/crates/compositor/src/timeline_walk.rs index 0953656ce2..5556e577b7 100644 --- a/crates/compositor/src/timeline_walk.rs +++ b/crates/compositor/src/timeline_walk.rs @@ -157,6 +157,119 @@ pub(crate) unsafe fn walk_composited_timeline( webcam_decs: &mut HashMap, on_frame: &mut dyn FnMut(u64) -> Result<()>, on_clip_end: &mut dyn FnMut(usize, f64, u64, &[SpeedSegment]) -> Result<()>, +) -> Result { + walk_timeline(clips, gpu, comp, cfg, out_fps, scene, screen_decs, webcam_decs, None, on_frame, on_clip_end) +} + +/// Une frame de sortie relue par `render_frames`. +pub struct RenderedFrame { + /// Son indice sur la grille de sortie (`out_fps`). + pub index: u64, + pub width: u32, + pub height: u32, + /// RGBA8, `width * height * 4` octets. + pub rgba: Vec, +} + +impl RenderedFrame { + /// En JPEG (qualité 85), réduite à `max_width` pixels de large au plus : la forme d'une image + /// de `getFrames`. Rend (largeur, hauteur, octets). + pub fn to_jpeg(self, max_width: u32) -> Result<(u32, u32, Vec)> { + let (index, width, height) = (self.index, self.width, self.height); + let img = image::RgbaImage::from_raw(width, height, self.rgba) + .ok_or_else(|| anyhow::anyhow!("frame {index}: the readback does not hold {width}x{height} pixels"))?; + let img = if max_width > 0 && width > max_width { + let h = ((height as u64 * max_width as u64) / width as u64).max(1) as u32; + image::imageops::resize(&img, max_width, h, image::imageops::FilterType::Triangle) + } else { + img + }; + let rgb = image::DynamicImage::ImageRgba8(img).to_rgb8(); + let mut jpeg = Vec::new(); + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 85).encode_image(&rgb)?; + Ok((rgb.width(), rgb.height(), jpeg)) + } +} + +/// Les frames de sortie `wanted` (indices sur la grille `out_fps`), composées comme l'export les +/// compose et relues en RGBA, par indice croissant. C'est la marche de l'export : mêmes clips, +/// mêmes vitesses, même frame source à chaque instant. Seules les autres frames ne sont ni +/// décodées ni composées. Un indice au-delà de la timeline ne rend rien. +pub unsafe fn render_frames( + clips: &[ClipSource], + gpu: &Gpu, + comp: &Compositor, + cfg: &Cfg, + out_fps: i32, + wanted: &[u64], +) -> Result> { + let mut only = wanted.to_vec(); + only.sort_unstable(); + only.dedup(); + let scene = comp.scene_snapshot(); + let mut out = Vec::new(); + walk_timeline( + clips, + gpu, + comp, + cfg, + out_fps, + &scene, + &mut HashMap::new(), + &mut HashMap::new(), + Some(&only), + &mut |index| { + let (width, height, rgba) = comp.readback_direct()?; + out.push(RenderedFrame { index, width, height, rgba }); + Ok(()) + }, + &mut |_, _, _, _| Ok(()), + )?; + Ok(out) +} + +/// Combien de secondes de vidéo un décodeur déroule, plutôt que de repartir d'une image clé, pour +/// rattraper les frames qu'une marche partielle (`only`) a sautées. +const CATCH_UP_DECODE_SEC: f64 = 2.0; + +/// Rattrape un décodeur laissé derrière `target` par les frames sautées, sans changer la frame que +/// la sémantique de hold y montrera. `seek_to` rend la première frame à ou après le point demandé : +/// on recule ce point jusqu'à tomber au plus sur `target`, ou sur le début du clip, là où la marche +/// s'est placée. `advance_decoder_to` adopte ensuite la dernière frame dont le pts est ≤ `target`, +/// celle qu'une marche sans saut aurait montrée. `target` et `clip_start` sont dans le temps du +/// décodeur (la webcam retranche son offset). +unsafe fn catch_up(dec: &mut Decoder, target: f64, clip_start: f64) -> Result<()> { + if target - dec.cur_time_sec() <= CATCH_UP_DECODE_SEC { + return Ok(()); + } + let mut back = CATCH_UP_DECODE_SEC; + loop { + let from = (target - back).max(clip_start); + let frame = dec.seek_to(from)?; + if from <= clip_start || (!frame.is_null() && dec.cur_time_sec() <= target) { + return Ok(()); + } + back *= 2.0; + } +} + +/// `walk_composited_timeline`, qui ne compose que les frames de sortie `only` (triées) quand il y +/// en a : les autres avancent le compteur sans décoder ni composer, et la marche s'arrête après +/// la dernière. Un flux sans pts fiable avance d'une frame par frame de sortie : sauter le rend +/// inexact, le cas est assez rare pour ne pas le traiter. +#[allow(clippy::too_many_arguments)] +unsafe fn walk_timeline( + clips: &[ClipSource], + gpu: &Gpu, + comp: &Compositor, + cfg: &Cfg, + out_fps: i32, + scene: &Option, + screen_decs: &mut HashMap, + webcam_decs: &mut HashMap, + only: Option<&[u64]>, + on_frame: &mut dyn FnMut(u64) -> Result<()>, + on_clip_end: &mut dyn FnMut(usize, f64, u64, &[SpeedSegment]) -> Result<()>, ) -> Result { // La piste se charge même curseur MASQUÉ : le focus auto des zooms la suit, comme en preview // (qui la charge toujours). Sans elle, un zoom auto suivait en preview et restait figé à @@ -180,7 +293,7 @@ pub(crate) unsafe fn walk_composited_timeline( // donc avec le vrai arrière-plan de la webcam gravé dans le fichier. comp.set_segmentation_deterministic(true); - for (clip_index, clip) in clips.iter().enumerate() { + 'clips: for (clip_index, clip) in clips.iter().enumerate() { // Le preset de layout est GLOBAL (un seul panneau pour toute la timeline) mais la // caméra est PAR CLIP : un projet mélange sans problème un enregistrement avec webcam // et un import qui n'en a pas. Le preset ne doit donc s'appliquer qu'aux clips qui ont @@ -313,6 +426,21 @@ pub(crate) unsafe fn walk_composited_timeline( 'clip_frames: for segment in &speed_segments { for segment_frame in 0..segment.frame_count { let target_source_time = segment.source_time(segment_frame, out_fps as f64); + if let Some(only) = only { + if only.last().map_or(true, |&last| frames > last) { + break 'clips; + } + if only.binary_search(&frames).is_err() { + frames += 1; + continue; + } + catch_up(sdec, target_source_time, clip.source_start_sec)?; + catch_up( + wdec, + target_source_time - clip.webcam_offset_sec, + (clip.source_start_sec - clip.webcam_offset_sec).max(0.0), + )?; + } { let _p = crate::export_probe::scope(crate::export_probe::Stage::DecodeScreen); if !advance_decoder_to(sdec, target_source_time, 0.0)? { @@ -374,7 +502,7 @@ pub(crate) unsafe fn walk_composited_timeline( #[cfg(test)] mod tests { - use super::{frame_step, FrameStep, NextFrameTime}; + use super::{frame_step, FrameStep, NextFrameTime, RenderedFrame}; /// La cadence de lecture, en une phrase : à 24 fps, une seconde réelle doit adopter 24 /// frames et pas une de plus. Le bug d'origine (un pas fixe de 1/60 s, une frame par @@ -464,4 +592,18 @@ mod tests { FrameStep::CommitAndStop ); } + + /// La frame rendue devient l'image de getFrames : un JPEG réduit à la largeur demandée, au + /// ratio de la sortie, et seulement réduit. + #[test] + fn a_rendered_frame_becomes_a_jpeg_no_wider_than_asked() { + let frame = |w: u32, h: u32| RenderedFrame { index: 3, width: w, height: h, rgba: vec![200; (w * h * 4) as usize] }; + let (w, h, jpeg) = frame(64, 32).to_jpeg(16).expect("jpeg"); + assert_eq!((w, h), (16, 8)); + let decoded = image::load_from_memory(&jpeg).expect("decode"); + assert_eq!((decoded.width(), decoded.height()), (16, 8)); + assert_eq!(frame(8, 4).to_jpeg(16).expect("jpeg").0, 8, "jamais agrandie"); + let short = RenderedFrame { index: 0, width: 8, height: 4, rgba: vec![0; 10] }; + assert!(short.to_jpeg(16).is_err(), "une relecture incomplète est une erreur"); + } } diff --git a/crates/compositor/tests/render_frames_match_export.rs b/crates/compositor/tests/render_frames_match_export.rs new file mode 100644 index 0000000000..40a805e497 --- /dev/null +++ b/crates/compositor/tests/render_frames_match_export.rs @@ -0,0 +1,117 @@ +//! Une frame rendue seule (`timeline_walk::render_frames`, ce que getFrames montre sans export) +//! est celle de l'export, au pixel près. La marche est la même ; seules les frames sautées +//! diffèrent, et `catch_up` rattrape les décodeurs sans changer la frame source montrée. Comparé +//! ici à la marche complète, où toutes les frames sont demandées et aucune n'est sautée. +//! +//! La scène met deux coupes du même fichier, une région 2× dans la seconde, un effet shader qui +//! lit le temps programme, et le curseur de la prise s'il a sa piste à côté. +//! +//! La sortie est à 24 i/s : sur une source à 30 i/s, ses instants tombent entre deux frames +//! source, là où un rattrapage qui viserait la cible au lieu de la précéder montrerait la frame +//! suivante. Encore faut-il que deux frames voisines diffèrent : une prise d'écran immobile ne +//! prouve rien, d'où une mire dont chaque frame change. +//! +//! La CI ne le joue pas : il faut un device (lavapipe convient) et la vidéo, d'au moins 20 s. +//! +//! ```powershell +//! ffmpeg -f lavfi -i testsrc2=size=640x360:rate=30 -t 25 -c:v libopenh264 -g 60 counter30.mp4 +//! $env:OPENSCREEN_RENDER_SOURCE = "...\counter30.mp4" +//! cargo test -p openscreen-compositor --test render_frames_match_export -- --nocapture +//! ``` + +use openscreen_compositor::compositor::{live_params_from_scene, Compositor}; +use openscreen_compositor::config; +use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::pipeline::ClipSource; +use openscreen_compositor::scene::Scene; +use openscreen_compositor::timeline_walk::{render_frames, RenderedFrame}; + +const FPS: i32 = 24; +const CLIPS: [(f64, f64); 2] = [(1.0, 4.0), (6.0, 20.0)]; + +fn scene_json(source: &str) -> String { + let clips: Vec = CLIPS + .iter() + .map(|(start, end)| { + format!( + r#"{{"screenPath":"{source}","webcamPath":"","sourceStartSec":{start},"sourceEndSec":{end},"webcamOffsetSec":0,"hasAudio":false}}"# + ) + }) + .collect(); + format!( + r##"{{ + "clips": [{clips}], + "layout": {{"preset":"no-webcam","webcamSize":1.0,"webcamShape":"rounded","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, + "effects": {{"padding":0.1,"blur":false,"shadow":0.0,"roundnessFrac":0,"motionBlur":0.0}}, + "background": {{"kind":"color","color":"#303030"}}, + "zoomRegions": [], + "speedRegions": [{{"clipIndex":1,"startSec":8,"endSec":10,"speed":2}}], + "cursor": {{"show":true,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}}, + "cropByClip": [null, null], + "output": {{"width":640,"height":360,"fps":null}}, + "shaderEffects": [{{ + "clipIndex": 1, "startSec": 6.0, "endSec": 20.0, "target": "image", "contract": 1, + "code": "fn effect(uv: vec2) -> vec4 {{ let c = frame(uv); return vec4(c.rgb * (0.6 + 0.4 * sin(ctx.time * 3.0)), 1.0); }}", + "params": [], "values": {{}} + }}] + }}"##, + clips = clips.join(",") + ) +} + +#[test] +fn a_frame_rendered_alone_is_the_frame_of_the_full_walk() { + let Some(source) = std::env::var_os("OPENSCREEN_RENDER_SOURCE") else { + eprintln!("OPENSCREEN_RENDER_SOURCE absent — test sauté"); + return; + }; + let source = source.to_string_lossy().replace('\\', "/"); + let gpu = Gpu::create_auto(false).expect("device"); + let clips: Vec = CLIPS + .iter() + .map(|&(start, end)| ClipSource { + screen: source.clone(), + webcam: String::new(), + source_start_sec: start, + source_end_sec: end, + webcam_offset_sec: 0.0, + has_audio: false, + }) + .collect(); + // La config de l'export (`export_cfg_and_scene` dans l'addon). + let mut cfg = config::all().pop().expect("C8"); + cfg.zoom = false; + cfg.layout_anim = false; + cfg.mblur_n = 1; + cfg.bg_blur = 0.0; + cfg.cursor = true; + let render = |wanted: &[u64]| -> Vec { + let scene = Scene::from_json(&scene_json(&source)).expect("scène"); + let comp = Compositor::new_sized(&gpu, 640, 360).expect("compositeur"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + unsafe { render_frames(&clips, &gpu, &comp, &cfg, FPS, wanted) }.expect("render_frames") + }; + + let all: Vec = (0..2_000).collect(); + let dense = render(&all); + let total = dense.len() as u64; + eprintln!("marche complète : {total} frames"); + // 3 s, puis 2 s + 1 s (2×) + 10 s : 16 s de sortie. + assert_eq!(total, 16 * FPS as u64, "la marche complète ne couvre pas la timeline"); + assert!(dense.iter().enumerate().all(|(i, f)| f.index == i as u64)); + + // Des sauts de plus de `CATCH_UP_DECODE_SEC`, dans les deux clips, dont un dans la région 2×. + let sparse_at = [2, 70, 130, 140, 260, total - 1, total + 50]; + let sparse = render(&sparse_at); + assert_eq!( + sparse.iter().map(|f| f.index).collect::>(), + &sparse_at[..6], + "une frame demandée manque, ou une au-delà de la timeline a été rendue" + ); + for frame in &sparse { + let reference = &dense[frame.index as usize]; + let differing = frame.rgba.iter().zip(&reference.rgba).filter(|(a, b)| a != b).count(); + assert_eq!(differing, 0, "frame {} : {differing} octets diffèrent de la marche complète", frame.index); + } +} diff --git a/docs/shader-effects.md b/docs/shader-effects.md new file mode 100644 index 0000000000..ab4a96b5f9 --- /dev/null +++ b/docs/shader-effects.md @@ -0,0 +1,868 @@ +# Effets shader écrits par les agents + +Un agent écrit un effet visuel en WGSL. Le compositeur l'applique en une passe sur l'image +composée, en preview comme à l'export. L'effet vit dans le document comme une région de la +timeline : on peut le relire, le régler, le déplacer et le supprimer, que l'on soit l'utilisateur +ou un agent. + +Ce document fixe le modèle, le contrat, le DSL, le rendu et l'interface, puis découpe les PR. +Il part du spike Windows (branche `worktree-custom-shader-fx-spike`, commits `9cb30018d` et +`42728e335`). + +**Ce que le spike a établi** + +- **Coût.** +0,04 ms par frame en 1080p, +0,12 ms en 4K, pour un effet lourd. +- **Export.** 520 ms sans effet, 523 ms avec, pour 180 frames. +- **Preview.** Le rendu tient la cadence de la source, avec ou sans effet. +- **SSOT.** Entre la vue live et le MP4, l'écart reste au niveau du bruit du codec. +- **Boucle d'agent.** À partir du prompt « effet ripple lors du clic », un agent a écrit un + effet correct en deux itérations, par les seuls outils MCP. + +--- + +## A. Les principes + +1. **Le compositeur reste la seule source de pixels.** Un effet est une passe de + `compose_frame`, que la preview et l'export appellent tous les deux. Il n'existe pas de + chemin de rendu propre aux effets. +2. **Le document ne stocke que la source.** Le WGSL, son nom, sa description, ses paramètres et + les valeurs posées. Tout binaire compilé en est dérivé, et n'est jamais écrit dans le projet. +3. **Un effet est une fonction pure** de l'image composée, du temps, du curseur et de ses + paramètres. Pas d'état d'une frame à l'autre. +4. **Le rendu ne compile jamais.** Le code est compilé à l'écriture, puis à la pose de la scène, + sur un thread de fond. +5. **Ce que seul le compositeur sait entre dans le contrat.** La place de l'écran, de la webcam + et du curseur dans la sortie, après zoom, layout et tilt : un effet ne peut pas la recalculer. + Le prompt du ripple l'a montré, il ne pouvait pas s'écrire sans. +6. **L'agent passe par le DSL.** Il appelle des outils de document. Il n'écrit jamais le JSON du + projet. +7. **Pour tout le monde.** Aucun réglage, aucune condition ne réserve la fonctionnalité à + certains utilisateurs. + +--- + +## B. Le modèle de données + +### B.1 Deux objets : la définition et la région + +- **La définition** (`shaderEffects`) porte le code, son nom, sa description et ses paramètres + déclarés. Elle dit ce qu'est l'effet. +- **La région** (`shaderEffectRanges`) pose une définition sur la timeline, avec ses valeurs de + paramètres. Elle dit où l'effet s'applique et comment il est réglé. + +La séparation sert trois choses : +- réutiliser un effet à plusieurs endroits sans dupliquer son code ; +- régler une région (`values`) sans réécrire de code ; +- garder léger le snapshot de l'agent, où le code n'apparaît qu'une fois par définition. + +### B.2 Le schéma + +Deux tableaux de premier rang dans `documentSchemaShape` +(`src/lib/ai-edition/schema/index.ts`), à côté de `zoomRanges` : + +```ts +const paramName = z.string().regex(/^[a-z][a-z0-9_]{0,31}$/); + +const shaderEffectParamSchema = z.discriminatedUnion("type", [ + z.object({ type: z.literal("number"), name: paramName, label: z.string().max(40), + default: z.number(), min: z.number(), max: z.number() }), + z.object({ type: z.literal("toggle"), name: paramName, label: z.string().max(40), + default: z.boolean() }), + z.object({ type: z.literal("color"), name: paramName, label: z.string().max(40), + default: hexColor }), + z.object({ type: z.literal("point"), name: paramName, label: z.string().max(40), + default: z.object({ x: z.number().min(0).max(1), y: z.number().min(0).max(1) }) }), +]); + +export const shaderEffectSchema = z.object({ + id: z.string().min(1), // "fx_…" + name: z.string().min(1).max(60), // ce que la pilule affiche + description: z.string().max(500).default(""), // pour l'agent suivant : ce que l'effet fait + contract: z.number().int().positive(), // pas un littéral : voir C.5 + code: z.string().min(1).max(65_536), // plafond technique, pas un budget (C.7) + params: z.array(shaderEffectParamSchema).max(32).default([]), + origin: z.enum(["agent", "user"]).default("agent"), + cost: z.object({ tier, frameReads }).optional(), // estimation du compositeur, en cache +}).superRefine(/* noms uniques, min ≤ default ≤ max */); + +export const shaderEffectRangeSchema = endGteStart(z.object({ + id: z.string().min(1), // "fxr_…", un par fragment + startMs: z.number().nonnegative(), // cache dérivé de l'ancre, comme les zooms + endMs: z.number().nonnegative(), + ...clipAnchorShape, + effectId: z.string().min(1), + values: z.record(z.string(), shaderEffectValueSchema).default({}), + target: z.enum(["image", "text", "both"]).default("image"), +}), "endMs", "startMs"); + +// documentSchemaShape +shaderEffects: z.array(shaderEffectSchema).optional(), +shaderEffectRanges: z.array(shaderEffectRangeSchema).optional(), +``` + +- **`values`** ne porte que ce qui diffère du `default` de la définition. Une valeur hors bornes + est ramenée dans ses bornes à l'écriture (les outils) et à la lecture (la scène), sans erreur. + Le parse ne le peut pas : les bornes sont dans la définition, pas dans la région. +- **Les `label`** sont écrits par l'agent, dans la langue de l'utilisateur, et affichés tels + quels. Ils ne passent pas par l'i18n : c'est du contenu, pas de l'interface. +- **Une pilule, ce sont les fragments identiques et contigus** (`regionIdentityKey`), comme pour + les zooms. Il n'y a pas de clé de groupe : une région coupée par un trim ou une frontière de + clip reste une seule pilule, et sa progression (`ctx.region`) court sur la pilule entière (B.5). +- **`cost`** est l'estimation du compositeur (C.7), mise en cache quand le code a été vérifié. Elle + est dérivée, jamais une entrée : un code modifié la recalcule. +- **Une définition que plus aucune région n'utilise est retirée** par l'édition qui a retiré sa + dernière région (`removeRegion`, `mapAllRegionCollections`). Le document ne garde pas de code + mort. +- **Les `max` du schéma sont des plafonds techniques, pas des budgets.** Ils protègent le document + (64 Ko de code, 32 paramètres) et sont loin au-dessus de tout effet raisonnable. Ce que + l'agent doit viser, ce sont les paliers de C.7. + +### B.3 Ancrée sur le clip, comme les autres régions + +Une région d'effet est une région de plus, pas un cas à part. Elle passe par tout le pipeline des +modificateurs ([timeline-model.md](../technical-documentation/architecture/timeline-model.md)), +et suit donc les clips quand on les déplace, les coupe ou les supprime. Le spike la stockait en +ms brutes, sans ancre : elle restait en place pendant que les clips bougeaient. + +- **`RegionKind`** gagne `"shaderEffect"`. Les sites qui switchent dessus sont listés dans + [editor-shell.md, « Adding a region kind »](../technical-documentation/architecture/editor-shell.md) + (ses numéros de ligne sont périmés). Les principaux : + - **le crochet des éditions de clips :** `mapAllRegionCollections` (`document/timeline.ts`), + par lequel passent moveClip, duplicateClip, setClipSourceRange, removeClip, replaceTimeline + et l'insertion ; + - **les listes codées en dur :** `anchoredRegionsOf`, et `removeRegion`/`EDIT_REGIONS`, dont + les types rendent un oubli bloquant à la compilation ; + - **côté agent :** `ModifierKind`, `modifierIds`, `modifierIdsOf` et la recherche d'id de + `removeModifier` (`agent-tools.ts`) ; + - **côté éditeur :** `removeRegions` (`useTimeline.ts`), `regionClipboard.ts`, le + copier-coller et la touche Suppr de `NewEditorShell.tsx`. +- **L'identité des pilules** (`regionIdentityKey`) compte tous les champs sauf la position et la + provenance. Deux régions contiguës du même effet, aux mêmes `values`, se fondent donc en une + pilule, comme deux zooms identiques. +- **Un effet à la fois** (K.6). Deux régions d'effet ne se chevauchent pas : les outils refusent + un chevauchement, avec la règle des autres régions (`refuseIfBlocked`). Pour combiner deux + rendus au même instant, l'agent les écrit dans un seul shader. +- **Après une édition de clip** qui ferait se chevaucher deux régions, le rendu applique les + deux, dans l'ordre du tableau. Pas de réparation automatique : deux effets superposés ne + cassent rien. + +### B.4 Persistance + +- **Champs additifs et optionnels, sans montée de `schemaVersion`.** C'est la convention de + `vocabulary` : `.optional()`, et `z.object` qui ignore les clés inconnues. Un document écrit + avant cette fonctionnalité s'ouvre tel quel, un document sans effet n'en porte aucune trace, et + tout lecteur prend une liste absente pour vide. +- **Le prix de ce choix.** Une build antérieure qui ouvre puis sauvegarde un projet perd ses + effets en silence. Voir K, question 1. +- **Pas dans les style presets.** Un preset porte le look du projet, jamais son contenu + (`stylePresets.ts`), et un effet est posé sur des moments précis du contenu. +- **`openscreen info`** compte les effets avec les zooms et les trims (`documentInfoFields`, + `projectCommands.ts`). + +### B.5 Ce que la scène native reçoit + +`buildSceneDescription` projette les régions comme les zooms (`projectRegionsToSource`). Il +ajoute à chaque fragment ce que le compositeur ne peut pas recalculer : + +```json +"shaderEffects": [{ + "clipIndex": 0, "startSec": 1.0, "endSec": 5.0, + "programmeStartSec": 1.0, "programmeEndSec": 5.0, + "target": "image", + "contract": 1, "code": "fn effect(uv: vec2) -> vec4 { … }", + "params": [ { "name": "strength", "type": "number" } ], + "values": { "strength": 1.0 } +}] +``` + +- **`programmeStart/EndSec`** est l'étendue de la pilule entière en temps programme. Le + compositeur en tire `ctx.region` sans rien savoir des trims. +- **`values` est déjà normalisé :** une couleur devient 4 floats, un toggle 0 ou 1, un point + 2 floats, et une valeur absente prend son `default`. +- **Le packing dans l'uniform** se fait côté Rust, au même endroit que la struct WGSL générée. + Le layout n'a qu'une source. +- **Les fragments sous un trim** (`underTrim`) restent dans la scène, comme pour les autres + régions. Ils se dessinent quand la tête de lecture est posée sur le trim, et l'export ne les + rencontre jamais. + +--- + +## C. Le contrat WGSL (version 1) + +### C.1 Ce que l'agent écrit + +Une seule fonction, appelée pour chaque pixel de la sortie : + +```wgsl +fn effect(uv: vec2) -> vec4 +``` + +`uv` va de (0, 0) en haut à gauche à (1, 1) en bas à droite de la sortie. La fonction rend la +couleur finale du pixel. Le reste du module (bindings, entrées, `@vertex`, `@fragment`) est écrit +par le compositeur et placé APRÈS le code de l'agent : en WGSL, l'ordre des déclarations est +libre, et les erreurs pointent ainsi les lignes que l'agent a écrites. + +### C.2 Ce qui est fourni + +| Nom | Type | Sens | +|---|---|---| +| `frame(uv)` | `vec4` | L'image composée en ce point (D.1 dit ce qu'elle contient). Échantillonnage bilinéaire, bords répétés. | +| `ctx.resolution` | `vec2` | Taille de la sortie, en px. | +| `ctx.aspect` | `f32` | Largeur sur hauteur de la sortie, pour les distances circulaires en uv. | +| `ctx.time` | `f32` | Secondes de programme, continues à travers coupes, clips et vitesses. | +| `ctx.region` | `vec4` | x : progression de 0 à 1 sur la pilule entière. y : sa durée en s. z : secondes depuis son début. | +| `ctx.cursor` | `vec4` | xy : la pointe en uv de sortie. z : 1 si elle est à l'écran, même quand le curseur est masqué. | +| `ctx.clicks[i]` | `vec4` | Les 4 derniers clics des 2 dernières secondes, du plus récent au plus ancien. xy : le point du clic en uv de sortie. z : secondes écoulées depuis. w : 1 si l'entrée est occupée. | +| `ctx.footage` | `vec4` | Le rect de l'enregistrement en uv de sortie (x, y, w, h), après zoom et layout. Sous un tilt 3D, c'est sa boîte englobante. | +| `ctx.camera` | `vec4` | Le rect de la webcam en uv de sortie. Taille nulle sans caméra. | +| `ctx.footage_aspect` | `f32` | Largeur sur hauteur de l'enregistrement dans son propre plan, pour les distances circulaires sur lui. | +| `ctx.footage_corners`, `ctx.footage_warp` | | Ce que lisent les deux fonctions suivantes : les coins TL, TR, BR, BL du métrage en uv de sortie, et son warp (1 homographie, 0 bilinéaire). | +| `fx_to_footage(uv)` | `vec2` | Le point du plan de l'enregistrement sous `uv` : (0, 0) son coin haut-gauche, (1, 1) le bas-droit, prolongé au-delà de ses bords. Loin hors de 0..1 là où aucun point du plan ne tombe. | +| `fx_from_footage(p)` | `vec2` | L'inverse : où tombe en uv de sortie le point `p` du plan. | +| `params.` | selon le type | Les paramètres déclarés par l'effet (C.3). | +| `fx_px(n)` | `f32` | `n` pixels d'une sortie 1080p, en fraction de la hauteur. | +| `fx_hash(p)` | `f32` | Hasard déterministe dans [0, 1) pour un `vec2`, identique sur tous les GPU (PCG). | + +- **Tout ce qui est en uv de sortie** est déjà passé par le zoom, le layout, le cadre et le + tilt. Ces valeurs viennent de `plan_frame` et `effect_pointer` (`frame_geometry.rs`), que les + trois backends partagent. +- **Le plan de l'enregistrement** suit le warp que le compositeur dessine : bilinéaire sous un + angle fixe, homographie sous la caméra réelle (`FootageQuad`, `quad_inverse` des shaders). + L'effet choisit son repère : l'image de sortie, ou ce plan. Le contrat ne décide pas lequel + convient à quel effet. +- **Le curseur masqué** (« Afficher le curseur » coupé, l'auto-hide, le `hideCursor` d'un zoom) + n'est plus dessiné, mais `ctx.cursor` et `ctx.clicks` restent ceux de l'enregistrement : un + halo suit toujours la souris, une onde part toujours du clic, et un effet peut remplacer le + curseur par son propre dessin. `ctx.cursor.z` ne vaut 0 que si l'enregistrement a caché la + pointe ou si elle sort du cadre (un zoom serré) ; les clics d'avant restent. +- **Le point d'un clic** est la pointe à l'instant du clic, au même endroit que l'impact intégré + (mode 16). +- **Le curseur dessiné** passe par le lissage et peut traîner un instant derrière ce point quand + la souris arrive pile au moment du clic. Ce cas est rare sur une vraie prise. +- **Le spike exposait un seul clic** (`ctx.click`). Avec quatre, deux clics rapprochés gardent + chacun leur onde. +- **Rien d'autre n'est lié.** Une variable `@group`/`@binding` dans le code de l'agent est + refusée, avec sa ligne : aucun backend ne la lierait. + +### C.3 Les paramètres sont nommés et typés + +Un effet déclare ses paramètres. Le compositeur en génère une struct `FxParams` liée en uniform, +et le code lit `params.radius`, jamais `param(2)`. Un agent qui relit l'effet six mois plus tard +comprend ce qu'il lit, et l'inspecteur sait quel contrôle afficher. + +| `type` | WGSL | Dans le document | Contrôle de l'inspecteur | +|---|---|---|---| +| `number` | `f32` | nombre, entre `min` et `max` | curseur | +| `toggle` | `f32` (0 ou 1) | booléen | interrupteur | +| `color` | `vec4` (RGBA, 0 à 1) | `#rrggbb` ou `#rrggbbaa` | sélecteur de couleur | +| `point` | `vec2` (uv de sortie) | `{ x, y }` | deux curseurs | + +- **Huit paramètres au plus, recommandé.** Au-delà, l'inspecteur devient une liste que personne + ne règle. Ce n'est pas un refus : le plafond technique est à 32. +- **Les noms** sont des identifiants WGSL en `snake_case`, uniques, hors mots réservés. +- **Le layout uniform est calculé côté Rust** avec les règles d'alignement de WGSL : `f32` sur 4, + `vec2` sur 8, `vec4` sur 16. Il produit le même ordre et les mêmes trous que la struct générée, + et un test compare les offsets de naga à ceux du packer. +- **Un paramètre doit se voir.** La description de l'outil exige qu'il change visiblement + l'effet sur toute sa plage. Un contrôle qui ne fait rien à l'écran est exactement ce que + l'inspecteur doit éviter. + +### C.4 Les règles + +- **Pure.** Le résultat ne dépend que de `uv`, de `frame()`, de `ctx` et de `params`. Pas d'état + d'une frame à l'autre, pas d'horloge, pas de compteur de frames. La preview saute d'une frame à + l'autre au seek, l'export les rend dans l'ordre, et les deux doivent donner le même pixel. +- **Indépendante de la résolution.** La preview rend plus petit que l'export. Distances en uv ou + en `fx_px`, jamais en px bruts. +- **Hasard par `fx_hash`.** Le classique `fract(sin(x) * 43758.5453)` donne des valeurs + différentes d'un GPU à l'autre. +- **Boucles à borne littérale.** Toute boucle s'arrête sur un compteur comparé à un littéral + (`for (var i = 0; i < 16; i++)`). Une boucle dont la borne n'est pas un littéral est refusée. + C'est la seule règle ferme du contrat, et ce n'est pas un budget : une boucle qui ne finit pas + bloque le GPU, le TDR de Windows le réinitialise, et l'app perd son device sans retour (G.2). + La valeur de la borne, elle, est libre : elle compte dans le palier de coût (C.7). +- **Un plafond de sécurité sur le travail par pixel :** 4 096 itérations, toutes boucles + imbriquées comprises. L'ordre de grandeur visé est le seuil du TDR (2 s par frame) sur un GPU + intégré en 1080p ; c'est une estimation, que la PR 1 mesure et ajuste. Un flou de 64 taps en + fait 64, un raymarching de 64 pas sur 8 objets 512. +- **Alpha.** Pour les cibles `image` et `both`, la valeur rendue remplace le pixel ; les + encodeurs ignorent l'alpha, et rendre `frame(uv).a` suffit. Pour la cible `text`, `frame()` + rend le calque des textes en alpha prémultiplié, transparent hors du texte, et l'effet doit + rendre de l'alpha prémultiplié lui aussi (D.1). + +### C.5 Le contrat est versionné + +Chaque définition porte `contract: 1`. +- **Le prélude d'une version livrée ne change plus.** Une entrée nouvelle s'ajoute à la fin de + `FxContext` : le code existant ne la lit pas et compile à l'identique. +- **Un changement de sens crée la version 2,** et le compositeur garde le prélude de la 1. +- **Un document qui porte une version inconnue** (écrit par une build plus récente) garde ses + régions, sans les appliquer. + +### C.6 Exemple + +Le ripple du spike, réécrit pour le contrat v1 : + +```wgsl +// Une onde part de chaque clic, réfracte l'image et laisse deux ou trois vagues derrière elle. +fn ripple(uv: vec2, click: vec4) -> vec2 { + if (click.w < 0.5 || click.z > params.duration) { return vec2(0.0); } + let t = click.z / params.duration; + let aspect = ctx.resolution.x / ctx.resolution.y; + let d = (uv - click.xy) * vec2(aspect, 1.0); + let r = length(d); + let x = params.radius * (1.0 - pow(1.0 - t, 3.0)) - r; + let env = smoothstep(-0.008, 0.0, x) * exp(-max(x, 0.0) / 0.07); + let wave = sin(x / 0.03 * 6.2831853) * env * (1.0 - t) * (1.0 - t) * params.strength; + let dir = select(vec2(0.0), d / r, r > 1e-5); + return dir * wave * 0.01 / vec2(aspect, 1.0); +} + +fn effect(uv: vec2) -> vec4 { + var offset = vec2(0.0); + for (var i = 0; i < 4; i++) { offset += ripple(uv, ctx.clicks[i]); } + return frame(uv - offset); +} +``` + +Avec `params` : `strength` (number, 0 à 2, 1), `radius` (number, 0,05 à 0,5, 0,2), `duration` +(number, 0,3 à 2 s, 0,9). + +### C.7 Le coût : des paliers indicatifs, pas des limites + +Le coût d'un effet est estimé depuis son code, sans le faire tourner. Il ne dépend donc pas de la +machine : le même code reçoit le même palier partout, et un effet haut de gamme n'est jamais +refusé parce que la machine qui l'écrit est modeste. + +- **L'estimation se fait sur l'IR de naga.** Elle compte les lectures de `frame()` par pixel, + multipliées par la borne des boucles qui les contiennent ; les fonctions appelées sont + dépliées. Les lectures de texture dominent le coût mesuré dans le spike : c'est la bande + passante qui paie. +- **Les paliers**, en lectures par pixel : + +| Palier | Lectures | Exemples | Recommandation | +|---|---|---|---| +| Léger | ≤ 8 | vignette, étalonnage, ripple à 4 clics | Partout, y compris sur toute la vidéo. | +| Moyen | ≤ 32 | aberration chromatique échantillonnée, petit flou, glitch | Sur des régions de quelques dizaines de secondes. | +| Lourd | ≤ 128 | gros flou, bokeh | Sur des régions courtes : une transition, un moment fort. | +| Très lourd | au-delà | raymarching, simulations | Quelques secondes. L'export reste juste, mais plus lent ; la preview peut ralentir sur un petit GPU. | + +- **Mesuré sur la RTX 4070 Ti** (`shader_fx_tier_calibration`) : environ 4 µs par lecture en + 1080p et 17 µs en 4K, au-dessus d'un fixe de 0,03 ms. Un effet léger coûte au plus 0,06 ms par + frame en 1080p, un moyen 0,14 ms, un lourd 0,54 ms, 512 lectures 2,1 ms. Le détail est dans + [rendering-performance.md](../technical-documentation/engineering/rendering-performance.md). +- **Mesuré sur lavapipe** (le repli CPU de Linux, 16 threads) : environ 1,4 ms par lecture en + 1080p, 350 fois la RTX. Un effet léger y ajoute jusqu'à 11 ms à une frame qui en coûte 7,5 ; + un lourd, 234 ms. L'ordre des paliers tient, mais sans GPU même un effet léger double le + temps d'export. +- **Mesuré sur un Apple M1** (GPU intégré, Metal, même banc) : environ 0,12 ms par lecture en + 1080p et 0,5 ms en 4K, au-dessus d'un fixe de 0,3 ms. Un effet léger coûte au plus 1,3 ms par + frame en 1080p, un moyen 4,3 ms, un lourd 16 ms, 512 lectures 49 ms : trente fois la RTX. Les + lectures y dominent aussi : sans les `sin`/`cos` du banc, 128 lectures coûtent encore 10,5 ms. +- **Le Radeon 610M** des mesures Linux reste à mesurer. La description de l'outil donnera les + paliers en ms sur la RTX et sur le M1, pour que l'agent raisonne en temps et non en lectures. +- **Rien n'est refusé au coût.** Le palier et sa recommandation reviennent à l'agent (E.3), qui + décide, et ils restent lisibles dans le snapshot (E.2). + +--- + +## D. Le rendu + +### D.1 Où la passe s'insère + +`compose_frame` dessine, dans l'ordre : +1. le fond ; +2. le métrage et le cadre ; +3. les flous de confidentialité ; +4. le curseur ; +5. la webcam ; +6. les annotations et les sous-titres. + +Une région choisit sa cible (`target`). Sans demande explicite, c'est `image`. + +- **`image`** : l'image sans les textes. La passe s'insère avant les annotations et les + sous-titres. Un effet déforme l'écran, la webcam, le curseur et le fond, jamais le texte que + l'utilisateur a posé pour être lu. +- **`text`** : le calque posé sur l'image, c'est-à-dire les annotations de tout type (textes, + images, flèches) et les sous-titres. Un effet `text` touche donc aussi une flèche ou un logo : + les séparer casserait leur ordre de superposition (`zIndex`). Ce calque est dessiné à + part, transparent, l'effet s'y applique, puis le calque est composé par-dessus l'image + (« over » prémultiplié). L'image dessous reste intacte. Le calque n'existe que si une région + `text` couvre la frame. +- **`both`** : l'image finale, textes compris, par exemple pour un fondu au noir qui doit couvrir + les sous-titres. La passe vient après tout. + +Les flous de confidentialité restent sûrs. L'effet ne voit que des pixels déjà floutés et ne peut +rien y retrouver : il n'existe aucun chemin vers l'image d'avant le flou. + +### D.2 Un effet à la fois + +- **Le document n'en tient qu'un à un instant donné** (B.3). Une scène qui en superposerait les + appliquerait dans l'ordre du tableau. +- **Le coût.** Un effet fait une copie du RT dans `ann_copy` et une passe plein cadre. +- **Les cibles s'enchaînent dans l'ordre du dessin :** les effets `image`, puis les textes et + leurs effets `text`, puis les effets `both`. Un effet ne voit que ce qui est dessiné avant lui. + +### D.3 Les chemins de rendu + +- **Un seul `compose_frame`.** La preview (`live.rs`), l'export MP4 et l'export GIF + (`walk_composited_timeline`, `timeline_walk.rs`) l'appellent tous. Les effets apparaissent + donc partout, sans code propre à chaque chemin. +- **Une passe par frame de sortie.** `compose_frame_mb` n'a aucun appelant, et le flou de + mouvement est calculé par calque (`mb_taps`). + +### D.4 La compilation : une fois, hors du rendu + +Il y a deux compilations, à deux moments : + +1. **À l'écriture.** Quand un outil pose ou change un code (`addEffect`, + `setEffectDefinition`), le processus principal appelle `checkShaderEffect(code, params)` de l'addon. + - **La compilation est réelle,** pour le backend de la machine, en plus de la validation + naga. « Accepté » veut donc dire « se dessinera ». + - **Elle tourne sur le pool de libuv** (une `AsyncTask`) : la compilation Metal d'un code neuf + prend 120 ms, et jusqu'à 375 ms pour la première du processus. `runDocumentTool` attend la + réponse, puis la passe à l'exécuteur des outils, qui reste synchrone. + - **Le coût est estimé** depuis le code (C.7), sans rendu ni mesure sur la machine. + - **L'erreur revient à l'agent** avec la ligne et la colonne de SON code. +2. **À la pose de la scène** (`Compositor::set_scene`). Chaque code nouveau part compiler sur un + thread de fond. + - **La preview** saute l'effet le temps de la compilation, puis recompose la frame en pause + (`shader_fx_landed`). Environ 10 ms avec FXC ; avec Metal, 120 ms pour une source neuve et + jusqu'à 375 ms pour la première du processus (H.3). + - **L'export** attend le binaire, et ne rend jamais une frame sans son effet. C'est le mode + par défaut du compositeur : seule la vue live le coupe. + +Le cache est en mémoire, par compositeur. +- **La clé** est le hash de (contrat, layout des paramètres, code). +- **L'éviction** est LRU au-delà de 64 entrées : un agent qui itère en crée une par version. +- **Pas de cache disque en v1.** 10 ms par effet ne le justifient pas, et Metal comme Mesa gardent + déjà leur propre cache de shaders : sur le M1, une source déjà vue se recompile en 0,4 ms dans le + même processus, en 3 ms dans un autre. L'app ne passe d'ailleurs aujourd'hui aucun dossier + inscriptible au natif : il faudrait une variable d'environnement sur le modèle de + `OPENSCREEN_FONTS_DIR`. + +### D.5 Par plateforme + +Une seule source WGSL, traduite par naga au moment de compiler. + +| | Windows | macOS | Linux | +|---|---|---|---| +| Traduction | naga vers HLSL SM 5.0 | naga vers MSL | aucune : WGSL tel quel | +| Compilation | `D3DCompile` (FXC) vers DXBC, puis `CreateVertexShader`/`CreatePixelShader` | `new_library_with_source`, puis `make_pipeline(…, Blend::Replace)` | `create_shader_module`, puis `create_render_pipeline` | +| Thread de fond | naga et FXC (fait) | naga, bibliothèque et pipeline, sous `autoreleasepool` (fait) | module et pipeline (fait) | +| Copie du RT | `CopySubresourceRegion` vers `ann_copy` (fait) | blit vers `ann_copy` dans le command buffer de la frame (fait) | `copy_texture_to_texture` vers le niveau 0 d'`ann_copy` (fait) | +| Insertion | fin de `compose_frame` (fait) | fin de `compose_frame`, avant `submit` (fait) | `image` entre la passe caméra et l'ann-pass, `text` et `both` après (fait) | +| Uniforms | `Map(WRITE_DISCARD)` (fait) | `set_fragment_bytes` (fait) | `create_buffer_init(UNIFORM)` par passe (fait) | + +**macOS en détail** +- **Activer `msl-out`.** Le dépôt l'obtient aujourd'hui par unification des features de wgpu ; le + déclarer explicitement sous `cfg(target_os = "macos")`. +- **Pas d'objet sampler.** Les shaders Metal du dépôt utilisent des `constexpr sampler` : + passer le sampler en `inline_samplers` de naga (`BindSamplerTarget::Inline`). +- **Lier les bindings explicitement.** Remplir `per_entry_point_map` pour `fx_fs` : texture, + sampler, buffer. Sinon `fake_missing_bindings` émet `[[user(fake0)]]`, qui n'est pas du MSL + valide. +- **Lire les noms de fonctions** dans `TranslationInfo.entry_point_names`. +- **Le thread de fond.** `Device`, `Library` et `RenderPipelineState` de metal-rs sont `Send` et + `Sync`, et `live.rs` déplace déjà un `Gpu` vers un thread de préchargement. +- **Pas de garde de boucle.** naga borne sinon chaque boucle par une lecture `volatile` à chaque + tour (`force_loop_bounding`). La validation a déjà prouvé que toute boucle finit (C.4) : la garde + coûtait 7 à 10 % de la passe sur le M1. +- **Le calque des textes.** `draw_annotations` ouvre sa propre passe : elle prend désormais sa + cible, le RT ou le calque. Le calque est effacé dans une passe à part, puisque + `draw_annotations` n'en ouvre aucune quand rien n'est visible, puis posé sur le RT par + `pipeline_fs_tex` (« over » prémultiplié). +- **La passe charge sa cible** au lieu de l'ignorer : un effet peut faire `discard`, et le pixel + doit alors rester celui de la composition. + +**Linux en détail** +- **Un bind group layout explicite,** jamais `layout: None`. Sinon un effet qui n'appelle pas + `frame()`, ou ne lit pas `ctx`, perd ces bindings, et le bind group échoue à la validation. +- **Le thread de fond.** `wgpu::Device` est `Send`, `Sync` et `Clone` : le pipeline peut se + construire hors du thread de rendu. +- **Valider avant wgpu.** Aucun `on_uncaptured_error` n'est posé, donc une erreur de validation + wgpu fait paniquer. La pré-validation (`shader_fx::analyze`) en protège : naga 24, celui de + wgpu 24, et le refus de tout binding que le prélude ne déclare pas. Si wgpu panique malgré tout, + seul le thread de compilation meurt, et l'effet ne se dessine pas (D.6). +- **« Accepté » sous Linux.** `check_for_agent` traduit l'effet en SPIR-V par naga, comme wgpu le + fera. La compilation par le pilote Vulkan n'a lieu qu'à `set_scene`, faute de device dans + l'outil. +- **Le calque des textes** est une texture RGBA8 transparente, posée sur le RT par la passe qui + recopie déjà `accum` (`pipeline_copy`, « over » prémultiplié). + +**naga reste en 24.** À partir de la 25, sa sortie HLSL déclare les samplers dans un tas D3D12 +(`register(s0, space0)`), que FXC refuse en `ps_5_0` (erreur X3721). Le test +`a_valid_effect_becomes_hlsl_that_d3d11_accepts` garde cette frontière : il doit passer avant tout +bump de naga ou de wgpu. + +### D.6 Un effet qui ne compile pas au rendu + +Ce cas arrive avec un document écrit ailleurs, une version de contrat inconnue ou un +compilateur qui diffère d'une machine à l'autre. +- **La région ne se dessine pas,** et la frame reste celle de la composition. +- **L'erreur est écrite une fois** dans le journal natif (`[shader-fx]`) et remonte à + l'inspecteur (F.2). +- **L'export ne s'arrête pas.** Un effet manquant vaut mieux qu'un export perdu, et l'agent ou + l'utilisateur a déjà vu l'erreur à l'écriture. + +--- + +## E. Le DSL : ce que l'agent appelle et relit + +Les effets passent par les outils de document, comme le reste : `TOOL_ARG_SCHEMAS`, +`TOOL_DESCRIPTIONS`, `executeAgentTool`. Le serveur MCP les sert sans rien de plus, et l'agent +intégré les reçoit par le même tableau. + +### E.1 Les outils + +| Outil | Arguments | Effet | +|---|---|---| +| `addEffect` | `startSec`, `endSec` (secondes de règle), `effect` **ou** `effectId`, `values?`, `target?` | Pose une région. Avec `effect` (`name`, `description`, `code`, `params?`), crée d'abord la définition. Avec `effectId`, réutilise une définition du projet. | +| `setEffect` | `effectRegionId`, `startSec?`, `endSec?`, `values?`, `target?` | Change une région. Seuls les champs passés changent. | +| `setEffectDefinition` | `effectId`, `name?`, `description?`, `code?`, `params?` | Change une définition, donc toutes les régions qui l'utilisent. Un code neuf est compilé avant d'être accepté. Les `values` d'un paramètre retiré tombent, un paramètre nouveau prend son `default`. | +| `getEffect` | `effectId` | Lecture : la définition entière, code compris. | +| `removeModifier` | `id` | Retire une région d'effet, comme un zoom. | + +- **Un seul `add`.** Écrire puis poser un effet est le cas courant : un appel, pas deux. + `effectId` sert à réutiliser. +- **Pas de `removeEffect`.** La définition part avec sa dernière région (B.2). +- **Les noms suivent le DSL :** `add*`, puis `set*` qui ne touche que les champs passés, puis + `removeModifier` pour retirer. Pas de neutralisation (force à 0, durée nulle) à la place d'un + retrait. +- **Le placement suit les autres régions :** `anchorForAgent` / `landingOf`. Le modèle est + `addCameraFullscreen`. Une étendue qui ne couvre aucun clip est refusée (`coversNoClip`), et + une région qui en heurterait une autre aussi : un effet à la fois (`refuseIfBlocked`). + +### E.2 Ce que l'agent relit + +`getCurrentDocument` ajoute deux listes, sans le code, qui coûterait cher en tokens à chaque +lecture : + +```json +"effects": [ + { "id": "fxr_…", "effectId": "fx_…", "name": "Ripple au clic", + "startSec": 0, "endSec": 6, "target": "image", "values": { "strength": 1 } } +], +"effectDefinitions": [ + { "effectId": "fx_…", "name": "Ripple au clic", + "description": "Une onde part de chaque clic et réfracte l'image.", + "contract": 1, "lines": 22, "regions": 1, + "cost": { "tier": "light", "frameReads": 4 }, + "params": [ { "name": "strength", "type": "number", "label": "Force", + "default": 1, "min": 0, "max": 2 } ] } +] +``` + +- **Les pilules sont coalescées** comme les autres régions (`coalesceForAgent`) : un id par + pilule, celui de son premier fragment. +- **Le code se lit par `getEffect`,** quand l'agent veut le modifier. +- **`description` est écrite par l'agent** et lue par l'agent suivant. La description de l'outil + l'exige : ce que l'effet fait, en une phrase, pas comment il le fait. +- **`cost`** est l'estimation mise en cache dans la définition. `null` signifie que rien n'a pu + vérifier le code quand il a été écrit. +- **`timeBaseNote`** nomme les effets parmi les régions en secondes de règle. + +### E.3 Les retours + +- **Accepté :** l'id, l'étendue posée (`landingReport`, comme les autres régions : fragments, + bornes rognées), et le coût : le palier de l'effet, et la recommandation du palier quand la région + la dépasse. Par exemple : « lourd sur 45 s : + pour une région longue, vise moyen ». +- **Refusé à la compilation :** l'enveloppe `{ error }`, avec l'erreur de naga ou du compilateur + du backend, lignes et colonnes de SON code. Le document ne change pas. +- **Refusé pour une boucle sans borne littérale,** ou au-delà du plafond de sécurité de C.4, + avec la ligne de la boucle. C'est le seul refus qui ne vient pas du compilateur. +- **Sans validateur branché** (un harnais de test), le code est stocké et `validated: false` le + dit. + +### E.4 Voir le résultat sans exporter + +Avant, `getFrames` (`media-tools.ts`) ne montrait que le dernier export : vérifier un effet +coûtait un export complet. Sur un projet de dix minutes, la boucle de l'agent devenait trop +lente. + +**`getFrames` rend désormais depuis le document quand l'export manque ou est périmé** (PR 5). +- **Avec un export à jour,** l'outil lit toujours ce fichier. +- **Sinon,** le processus principal construit les clips et la scène comme l'export + (`buildNativeClipList`, `buildSceneDescription`). Il passe les secondes de règle en indices de + frame, sur la grille à 60 i/s de `exportVideo`, et demande ces frames à l'addon + (`renderFrames`). +- **L'addon rend par la marche de l'export,** `timeline_walk::render_frames` : mêmes clips, mêmes + vitesses, même frame source à chaque instant. Les frames non demandées ne sont pas composées. + Le décodeur ne lit que ce qui mène à chaque frame demandée : depuis la précédente quand elle + est à moins de 2 s, sinon depuis une image clé. +- **Un saut de plus de 2 s repart d'une image clé** (`catch_up`). `seek_to` rend la première + frame à ou après le point visé ; on recule donc ce point jusqu'à tomber avant l'instant voulu, + puis on avance comme l'export : la frame montrée est la dernière dont le pts le précède. +- **Le compositeur attend la compilation des effets,** comme celui de l'export. +- **Les images sont des JPEG** de 1 280 px de large au plus, comme les frames d'un export. +- **Prouvé au pixel** par `tests/render_frames_match_export.rs` : une frame rendue seule est + identique, octet pour octet, à la même frame de la marche complète. La source est une mire dont + chaque frame change, sur une grille à 24 i/s qui tombe entre ses frames. Un seek droit sur + l'instant visé échoue à ce test. +- **Ce n'est pas la vue live** que le spike pilotait (`presentTime`, `readFrame`) : elle saute un + effet qui compile encore, et il faut la sonder en boucle. + +### E.5 Ce que l'agent lit avant d'écrire + +- **`TOOL_DESCRIPTIONS.addEffect`** porte le contrat court : la signature, les entrées de C.2, + les règles de C.4, les paliers de C.7 en ms, les trois cibles et le défaut `image`, l'exigence + d'un paramètre visible, et la vérification avec `getFrames`. +- **La ressource MCP `openscreen://effects/contract`** porte le contrat complet et deux + exemples : le ripple au clic et une vignette qui respire. + - **Ce serait la première ressource du serveur,** qui n'en a aucune aujourd'hui. + - **Pourquoi une ressource :** le contrat est trop long pour une description d'outil, et il + évolue avec `contract`. +- **Le prompt de base** (`BASE_SYSTEM_PROMPT`) gagne deux choses : + - une ligne dans la carte des intentions : un effet visuel que les réglages n'offrent pas se + fait avec `addEffect` ; + - les effets dans la ligne des bases de temps. +- **Le skill `openscreen-demo`** gagne une entrée « effets » à son étape 6. + +### E.6 L'agent intégré + +Il a les mêmes outils et le même validateur que le serveur MCP : `handlers.ts` passe +`checkShaderEffect: (code, params) => loudnessService.checkShaderEffect(code, params)` à +`runChat`, qui le transmet par `ChatRunEnv` et `InvokeArgs` jusqu'à `buildTools`. + +### E.7 Le banc (workbench) + +Un scénario, `effect-ripple-on-click`, sur une mire avec trois clics : + +- **Attendu :** un `addEffect` dont le code lit `ctx.clicks`, une région qui couvre chaque clic et + la demi-seconde qui suit, et la cible `image`, puisque rien ne demande les textes. +- **Pas exigé :** lire la piste du curseur. Le compositeur donne déjà les clics placés sur la + sortie, c'est tout l'intérêt du contrat (C.2). + +`documentInvariants` valide les deux nouvelles listes : chaque région pointe une définition +existante, et ses `values` sont dans les bornes. Hors ligne, rien ne compile le code : le résultat +dit `validated: false`, et les checks lisent le code comme du texte. + +--- + +## F. L'interface + +### F.1 Une piste « Effets » dans la timeline + +- **Où.** Une seule rangée « Effets », sous la piste Full Camera : les effets ne se chevauchent + pas (K.6). Elle n'apparaît que si le projet a un effet, puisqu'aucun bouton n'en ajoute, et + la timeline grandit alors de sa hauteur (`EFFECT_LANE_EXPANSION_PX`). +- **La pilule** porte le nom de la définition. La glisser, l'étirer, la sélectionner et la + supprimer se font comme pour un zoom (`selectPill`, `startPillDrag`, `removeRegions`). +- **L'annulation** passe par l'historique habituel (`saveDocument({ history: true })`). Une + édition d'agent reste un seul pas d'annulation par tour (`agentDocumentApply.ts`). +- **Pas de bouton « ajouter un effet » en v1.** Un effet naît d'un agent : le chat intégré ou un + client MCP. Un bouton qui ouvrirait un panneau vide n'aurait rien à proposer. La bibliothèque + d'effets (J) lui donnera un contenu. +- **Le copier-coller** (`regionClipboard`) copie la région avec sa définition. Collée dans un + autre projet, la définition y est recréée. +- **Le site** (`website/scripts/gen-recreation.mjs`) recopie `LANE_ORDER` à la main. La nouvelle + piste n'y casse rien, et la recréation ne la montrera pas tant qu'on ne l'y ajoute pas. + +### F.2 L'inspecteur + +`SelectionPane` (`FloatingInspector.tsx`) gagne une branche `shaderEffect`. Elle contient, dans +l'ordre : + +- **le nom de l'effet,** sans phrase d'explication sous les contrôles ; +- **un contrôle par paramètre déclaré,** généré depuis `params` (curseur, interrupteur, couleur + ou point), qui écrit `values` sur la région ; +- **la cible :** « Image », « Textes » ou « Les deux » (`target`) ; +- **le palier de coût,** en un mot (léger, moyen, lourd, très lourd), sans phrase ; +- **le code,** replié par défaut, en lecture seule, avec un bouton copier ; l'édition du code + passe par l'agent en v1 ; +- **supprimer,** comme pour les autres régions. + +**Un effet qui ne compile pas** affiche l'erreur du compilateur dans l'inspecteur. C'est +l'exception admise aux phrases sous les réglages : sans elle, la pilule existerait sans rien +faire, et ce serait une vraie surprise. + +### F.3 Textes de l'interface + +Les libellés fixes vont dans `editor.json`, pour les 15 locales : `inspector.effect*` (cible, +coût et ses quatre paliers, code, erreur) et `regionClipboard.pasteEffectOverlap`. La pilule +et l'en-tête de l'inspecteur portent le nom de l'effet, que l'agent écrit : la piste n'a pas de +libellé propre. +`localeParity.test.ts` et `npm run i18n:check` vérifient leur parité. Les noms et libellés +écrits par l'agent n'y passent pas. + +--- + +## G. SSOT, déterminisme, sécurité + +### G.1 Ce qui garantit preview = export + +- **Un seul chemin de pixels.** La preview et l'export appellent le même `compose_frame`, avec la + même scène : `buildSceneDescription` sert aux deux. +- **Une seule horloge.** `ctx.time` est le temps programme, et `ctx.region` vient de l'étendue + de la pilule en temps programme. Aucun des deux ne lit le compteur de frames. +- **Un effet pur.** C'est le contrat C.4. La validation fait respecter ce qui se vérifie + statiquement (boucles, taille), et la description de l'outil donne le reste à l'agent comme + règle d'écriture. +- **Jamais de placeholder à l'export.** Le mode bloquant est le défaut du compositeur. +- **À résolution égale.** La preview rend plus petit que l'export. Un effet qui suit C.4 donne la + même image, mais pas les mêmes pixels, puisqu'il n'y en a pas le même nombre. C'est déjà vrai + de tout le reste du rendu. + +**Mesure du spike.** Écart entre la frame de la vue live et celle du MP4 exporté, décodé en +BT.709 : 2,2 à 2,6 / 255, dans la région d'effet comme hors d'elle. C'est le bruit du codec. + +### G.2 Un GPU qui ne répond plus + +Le seul vrai risque d'un code écrit par un agent est une boucle qui ne finit pas. Le TDR de +Windows réinitialise alors le GPU, et l'app perd son device. Rien ne gère ce cas dans le dépôt : +le thread de rendu s'arrête sur son erreur (`fatal`, `live.rs`) et ne redémarre pas. + +La protection est statique, posée à la validation : la borne littérale de toute boucle et le +plafond de 4 096 itérations par pixel (C.4). Ce n'est pas un budget de coût, qui n'existe qu'en +paliers indicatifs (C.7). Récupérer un device perdu est un chantier à part : il concerne toute +l'app, pas les effets. + +### G.3 Un projet partagé + +Un projet porte le code de ses effets. Ouvrir le projet de quelqu'un d'autre compile donc du code +qu'on n'a pas écrit. +- **Ce code ne tourne que sur le GPU,** sans accès aux fichiers, au réseau ni à la mémoire du + processus. +- **Les garde-fous de G.2 s'appliquent aussi à l'ouverture.** Un effet refusé reste dans le + document, et sa région ne se dessine pas. + +--- + +## H. Performance + +### H.1 Pas de budget ferme + +- **Une région inactive ne coûte rien.** La passe n'existe que si une région couvre la frame. +- **Le coût d'un effet se lit en paliers** (C.7), estimés depuis le code et identiques sur toutes + les machines. L'agent les connaît et suit leurs recommandations ; rien n'est refusé au coût. +- **Une cible `text`** ajoute un calque transparent et sa composition, seulement aux frames + qu'elle couvre. + +### H.2 Mesuré par le spike (RTX 4070 Ti, D3D11) + +| Mesure | Résultat | +|---|---| +| Compilation (naga + FXC) d'un effet de 60 lignes | 10 ms, sur un thread de fond | +| Passe d'un effet lourd, 1080p | +0,04 ms par frame | +| Passe d'un effet lourd, 4K | +0,12 ms par frame | +| Export de 180 frames en 1752×1080 : sans effet, avec effet, avec effet lourd | 520, 523 et 521 ms | +| Preview en lecture, source à 30 i/s | 30 i/s avec et sans effet | + +Mesuré à la passe Linux (lavapipe sous WSL, 16 threads) : effet prêt en 18 ms après `set_scene`, +module et pipeline compris ; les paliers en C.7. + +À mesurer à l'implémentation : +- le Radeon 610M des mesures Linux. + +### H.3 Mesuré sur macOS (Apple M1, Metal) + +| Mesure | Résultat | +|---|---| +| Compilation (naga + Metal + pipeline) d'une source neuve, le ripple de C.6 | 120 ms en médiane, 160 à 375 ms pour la première du processus, sur le thread de fond | +| Recompilation d'une source que Metal a déjà vue | 0,4 ms dans le même processus, 3 ms dans un autre | +| `addEffect` accepté, compilation réelle comprise (CLI `mcp`) | 152 ms ; refusé à la ligne de l'agent en 6 ms | +| Passe d'un effet de 1 lecture, 1080p et 4K | +0,41 ms et +1,2 ms par frame | +| Passe d'un effet de 128 lectures, 1080p et 4K | +16 ms et +63 ms par frame | +| Export de 180 frames en 1752×1080 : sans effet, ripple, effet lourd du spike | 2 469, 2 428 et 2 375 ms : dans le bruit, l'export est limité par l'encodeur | +| Preview en lecture, source à 30 i/s | 29,7 i/s avec et sans effet | +| Vue live contre MP4 (BT.709), image entière | 3,1 / 255 avec effet, 3,0 sans | +| Vue live contre MP4, sur les pixels où l'effet agit | 13 à 16 / 255 avec effet, 14 à 16 sans effet sur les mêmes pixels : le bruit du codec sur des bords nets | + +La compilation, dix fois plus lente qu'avec FXC, reste hors du rendu. Elle tourne aussi à +l'écriture (`checkShaderEffect`), d'où le pool de libuv plutôt que le thread principal (D.4). + +Le détail est dans +[rendering-performance.md](../technical-documentation/engineering/rendering-performance.md). + +--- + +## I. Les tests + +| Niveau | Ce qu'il couvre | +|---|---| +| Rust, `shader_fx.rs` | Prélude et struct générée ; packer de paramètres contre les offsets de naga ; borne littérale et plafond d'itérations ; estimation du coût et paliers (boucles imbriquées, fonctions dépliées) ; erreurs aux lignes de l'agent ; sélection des régions (fenêtre, `underTrim`) ; `ctx.region` sur une pilule coupée. | +| Rust, par backend | Une passe repeint la frame dans sa région et nulle part ailleurs ; les trois cibles (`text` laisse l'image intacte, `image` laisse les textes intacts) ; preview non bloquante puis `shader_fx_landed` ; effet cassé = frame intacte. Windows (fait), macOS (fait, sur le runner CI, avec en plus une vraie annotation entre une cible `image` et une cible `text`), Linux sur lavapipe. | +| Rust, `frame_geometry.rs` | `ctx.clicks`, `ctx.footage`, `ctx.camera` en uv de sortie, sous zoom et sous tilt. | +| TS, schéma | Parse, bornes des `values`, définition orpheline retirée, document ancien inchangé. | +| TS, timeline | La région suit `moveClip`, `removeClip`, `replaceTimeline` et les trims (`timelineMap.test.ts`, `timeline.test.ts`). | +| TS, outils | `addEffect` avec `effect` puis avec `effectId` ; `setEffect` ; `setEffectDefinition` ; `getEffect` ; `removeModifier` ; les refus (compilation, boucle sans borne, chevauchement, aucun clip) ; le palier et sa recommandation dans le retour ; le snapshot. | +| TS, scène | Projection, `programmeStart/EndSec` d'une pilule coupée, normalisation des `values`. | +| Banc | Les deux scénarios de E.7. | +| Bout en bout | La boucle du spike par la CLI `mcp` : prompt, `addEffect`, `getFrames`, puis comparaison de la vue live avec l'export. Une ligne dans le [journal e2e](../technical-documentation/testing/manual-e2e-checklist.md). | + +--- + +## J. Hors de la v1 + +- **Effets par calque** (l'écran seul, la webcam seule). La passe agit sur l'image composée. +- **Effets temporels** (traînées, persistance). Ils prendraient les frames précédentes comme + entrées explicites, jamais un état gardé ; ce n'est pas fait. +- **Transitions entre deux clips.** Elles demandent deux images en entrée. +- **L'édition du code dans l'interface.** En v1, le code se lit, se copie, et s'écrit par + l'agent. +- **La bibliothèque d'effets** partagée entre projets, avec un bouton « ajouter » qui la propose. +- **Une poignée sur la preview** pour les paramètres `point`. +- **Le cache disque** des binaires compilés. +- **La récupération d'un device GPU perdu,** un chantier qui concerne toute l'app. + +--- + +## K. Décisions (14/10/2026) + +1. **Champs additifs, sans montée de `schemaVersion`.** Comme `audioTracks`. Une build antérieure + qui sauvegarde le projet perd ses effets : c'est accepté. +2. **Pas de budget ferme, des paliers indicatifs** (C.7). Ils sont estimés depuis le code, donc + indépendants de la machine. L'agent les connaît, avec leurs recommandations. Un budget lié à + la machine aurait refusé des effets haut de gamme sur une machine modeste, alors que l'export, + lui, les rend très bien. +3. **Pas de limites fermes non plus.** Paramètres et code ont des recommandations, et des + plafonds techniques loin au-dessus (B.2). Seule reste ferme la borne littérale des boucles, + avec son plafond de 4 096 itérations par pixel : une protection contre le blocage du GPU, pas + une limite de créativité. +4. **Pas de bouton « ajouter un effet » en v1.** Les effets naissent des agents. +5. **Trois cibles : `image`, `text`, `both`.** Sans demande explicite, l'effet vise `image`, sous + les textes. +6. **Un effet à la fois** (décidé après les premières PR). Un effet est écrit sur mesure pour ce + que l'utilisateur demande : deux rendus au même instant s'écrivent dans un seul shader, au lieu + de s'empiler. Les lanes 0 à 7 des premières PR sont retirées du document, des outils et du + compositeur, et la timeline n'a qu'une rangée d'effets. + +## L. Découpage en PR + +Le spike n'est pas fusionné tel quel : les PR 1 et 2 reprennent son code, et il est stocké +ailleurs (`legacyEditor`, sans ancre). + +1. **Contrat v1 et passe Windows (Rust).** + - `shader_fx.rs` : struct `FxParams` générée et packer ; `ctx.region`, `ctx.clicks`, + `ctx.footage`, `ctx.camera`, `fx_px`, `fx_hash` ; borne littérale des boucles et plafond + d'itérations ; estimation du coût et paliers, calibrés en ms sur le GPU intégré de + référence. + - `check_shader_effect` compile pour le backend et rend le palier. + - `scene.rs` lit la nouvelle forme ; la passe Windows gère les trois cibles (calque de textes + à part pour `text`) et l'éviction LRU. +2. **Modèle de données et DSL (TS).** + - Schéma (B.2) et `RegionKind` dans tout le pipeline (B.3) ; + - outils (E.1), snapshot (E.2), projection de scène (B.5) ; + - validateur branché sur l'agent intégré (E.6) ; + - descriptions, prompt de base, ressource MCP du contrat, skill, `openscreen info`, banc. +3. **Passe macOS.** naga `msl-out`, samplers inline, bindings explicites, thread de fond, tests + sur le runner CI. +4. **Passe Linux.** WGSL direct, bind group layout explicite, thread de fond, test lavapipe. +5. **`getFrames` depuis le document.** Rendu de frames sans export quand celui-ci manque ou est + périmé (E.4). +6. **Interface.** Piste, inspecteur, copier-coller, i18n. + +La PR 2 dépend de la 1. Les PR 3, 4 et 5 dépendent de la 1 et peuvent avancer en parallèle. La +PR 6 dépend de la 2. + +Chaque PR met à jour la documentation qu'elle rend fausse, dans le même changement : +- `native-compositor.md` ; +- `document-model.md` et `timeline-model.md` ; +- `editor-shell.md`, dont la checklist « Adding a region kind » a ses numéros de ligne à + rafraîchir ; +- `ai-agent.md`, dont le tableau d'outils annonce encore « 26 tools » ; +- `mcp-server.md` ; +- une ligne de [decisions.md](../technical-documentation/architecture/decisions.md) : WGSL et + naga 24, compilation hors du rendu, effet pur ; +- et, à la PR 6, une page de `website/docs/`. diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index 9cfc65449b..c105d2e4b2 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -13,6 +13,8 @@ import { executeAgentTool, isMutatingTool, MUTATING_TOOL_NAMES, + shaderEffectCheckLookup, + shaderEffectToCheck, } from "./agent-tools"; import { measureTextInShippedFonts } from "./text-measure"; @@ -192,6 +194,7 @@ describe("the mutating-tool table", () => { "addAnnotation", "addAudio", "addCameraFullscreen", + "addEffect", "addSpeed", "addTrim", "addTrims", @@ -211,6 +214,8 @@ describe("the mutating-tool table", () => { "setAudio", "setCameraFullscreen", "setClipRange", + "setEffect", + "setEffectDefinition", "setEditorSettings", "setSpeed", "setTrim", @@ -3734,3 +3739,244 @@ describe("clip structure (#1180)", () => { }); }); }); + +describe("shader effects", () => { + const CODE = "fn effect(uv: vec2) -> vec4 { return frame(uv) * params.level; }"; + const params = [ + { type: "number", name: "level", label: "Level", default: 1, min: 0, max: 2 }, + { type: "color", name: "tint", label: "Tint", default: "#000000" }, + ]; + // Stands in for the compositor's check: refuses `nope`, prices every effect at 4 reads. + const checkShaderEffect = (code: string) => + code.includes("nope") + ? { error: "error: no definition in scope for identifier: `nope`" } + : { tier: "light" as const, frameReads: 4, iterations: 0 }; + const options = { checkShaderEffect }; + const run = (document: AxcutDocument, name: string, args: unknown) => + executeAgentTool(document, name, JSON.stringify(args), options); + const add = (document: AxcutDocument, args: Record = {}) => + run(document, "addEffect", { + startSec: 2, + endSec: 6, + effect: { name: "Dim", description: "Dims the picture.", code: CODE, params }, + ...args, + }); + + it("writes the effect and poses it, anchored, with only the values that differ", () => { + const added = add(fixtureDocument(), { values: { level: 0.5, tint: "#000000" } }); + expect(added.ok).toBe(true); + const doc = added.document as AxcutDocument; + const [effect] = doc.shaderEffects ?? []; + const [range] = doc.shaderEffectRanges ?? []; + expect(effect).toMatchObject({ + name: "Dim", + contract: 1, + origin: "agent", + cost: { tier: "light", frameReads: 4 }, + }); + expect(range).toMatchObject({ + effectId: effect.id, + startMs: 2000, + endMs: 6000, + clipId: "clip_1", + values: { level: 0.5 }, + target: "image", + }); + expect(JSON.parse(added.resultJson)).toMatchObject({ + effectRegionId: range.id, + effectId: effect.id, + validated: true, + cost: { tier: "light", frameReads: 4 }, + }); + expect(() => documentSchema.parse(doc)).not.toThrow(); + }); + + it("refuses code that does not compile, with the compiler's error, and changes nothing", () => { + const refused = add(fixtureDocument(), { + effect: { + name: "Broken", + description: "", + code: "fn effect(uv: vec2) -> vec4 { return nope; }", + }, + }); + expect(refused.ok).toBe(false); + expect(refused.document).toBeUndefined(); + expect(JSON.parse(refused.resultJson).error).toMatch(/does not compile[\s\S]*nope/); + }); + + it("takes either a new effect or one of the project's, not both or neither", () => { + const doc = add(fixtureDocument()).document as AxcutDocument; + const effectId = doc.shaderEffects?.[0].id; + expect(run(doc, "addEffect", { startSec: 8, endSec: 9 }).ok).toBe(false); + const reused = run(doc, "addEffect", { startSec: 8, endSec: 9, effectId }); + expect(reused.ok).toBe(true); + expect(reused.document?.shaderEffects).toHaveLength(1); + expect(reused.document?.shaderEffectRanges).toHaveLength(2); + }); + + it("runs one effect at a time, and says to combine looks in one shader", () => { + const doc = add(fixtureDocument()).document as AxcutDocument; + const blocked = add(doc, { startSec: 4, endSec: 8, target: "text" }); + expect(blocked.ok).toBe(false); + expect(JSON.parse(blocked.resultJson).error).toMatch( + /one effect runs at a time[\s\S]*one shader/, + ); + expect(add(doc, { startSec: 6, endSec: 8 }).ok).toBe(true); + }); + + it("recommends a lighter tier when a heavy effect runs long, and never refuses it", () => { + const heavy = { + ...options, + checkShaderEffect: () => ({ tier: "heavy" as const, frameReads: 100 }), + }; + const added = executeAgentTool( + fixtureDocument(), + "addEffect", + JSON.stringify({ + startSec: 0, + endSec: 40, + effect: { name: "Blur", description: "", code: CODE, params }, + }), + heavy, + ); + expect(added.ok).toBe(true); + expect(JSON.parse(added.resultJson).cost.advice).toMatch(/heavy over 40\.0 s/); + }); + + it("re-tunes, retargets and moves a region; a value back at its default is dropped", () => { + const doc = add(fixtureDocument(), { values: { level: 0.5 } }).document as AxcutDocument; + const regionId = doc.shaderEffectRanges?.[0].id; + const tuned = run(doc, "setEffect", { + effectRegionId: regionId, + values: { level: 1, tint: "#ff0000" }, + target: "both", + startSec: 10, + endSec: 12, + }); + expect(tuned.ok).toBe(true); + expect(tuned.document?.shaderEffectRanges?.[0]).toMatchObject({ + startMs: 10000, + endMs: 12000, + target: "both", + values: { tint: "#ff0000" }, + }); + }); + + it("moves only its own pill when a re-tune makes it match the neighbour it touches", () => { + const first = add(fixtureDocument(), { values: { level: 0.5 } }).document as AxcutDocument; + const effectId = first.shaderEffects?.[0].id; + const doc = run(first, "addEffect", { startSec: 6, endSec: 8, effectId }) + .document as AxcutDocument; + const regionId = doc.shaderEffectRanges?.find((r) => r.startMs === 2000)?.id; + const moved = run(doc, "setEffect", { + effectRegionId: regionId, + values: { level: 1 }, + endSec: 5, + }); + expect(moved.ok).toBe(true); + expect( + [...(moved.document?.shaderEffectRanges ?? [])] + .sort((a, b) => a.startMs - b.startMs) + .map((r) => [r.startMs, r.endMs, r.values]), + ).toEqual([ + [2000, 5000, {}], + [6000, 8000, {}], + ]); + }); + + it("refuses to move an effect onto another one", () => { + const first = add(fixtureDocument()).document as AxcutDocument; + const doc = add(first, { startSec: 8, endSec: 9 }).document as AxcutDocument; + const second = doc.shaderEffectRanges?.find((r) => r.startMs === 8000); + const moved = run(doc, "setEffect", { effectRegionId: second?.id, startSec: 4 }); + expect(moved.ok).toBe(false); + expect(JSON.parse(moved.resultJson).error).toMatch(/one effect runs at a time/); + }); + + it("changes the definition for every region, recompiles, and drops a removed parameter's values", () => { + const doc = add(fixtureDocument(), { values: { tint: "#ff0000" } }).document as AxcutDocument; + const effectId = doc.shaderEffects?.[0].id; + const refused = run(doc, "setEffectDefinition", { + effectId, + code: "fn effect(uv: vec2) -> vec4 { return nope; }", + }); + expect(refused.ok).toBe(false); + const changed = run(doc, "setEffectDefinition", { effectId, params: [params[0]] }); + expect(changed.ok).toBe(true); + expect(changed.document?.shaderEffectRanges?.[0].values).toEqual({}); + expect(JSON.parse(changed.resultJson)).toMatchObject({ regions: 1, validated: true }); + }); + + it("reads an effect's code by getEffect, and lists effects without it in the snapshot", () => { + const doc = add(fixtureDocument()).document as AxcutDocument; + const effectId = doc.shaderEffects?.[0].id; + const read = JSON.parse(run(doc, "getEffect", { effectId }).resultJson); + expect(read).toMatchObject({ + code: CODE, + regions: [{ startSec: 2, endSec: 6, target: "image" }], + }); + const snapshot = JSON.parse(run(doc, "getCurrentDocument", {}).resultJson); + expect(snapshot.effects).toEqual([ + expect.objectContaining({ effectId, name: "Dim", startSec: 2, endSec: 6 }), + ]); + expect(snapshot.effectDefinitions[0]).toMatchObject({ effectId, lines: 1, regions: 1 }); + expect(snapshot.effectDefinitions[0].code).toBeUndefined(); + expect(snapshot.timeBaseNote).toMatch(/effects/); + }); + + it("removes a region by removeModifier, and its definition with the last one", () => { + const doc = add(fixtureDocument()).document as AxcutDocument; + const removed = run(doc, "removeModifier", { id: doc.shaderEffectRanges?.[0].id }); + expect(removed.ok).toBe(true); + expect(JSON.parse(removed.resultJson).kind).toBe("shaderEffect"); + expect(removed.document?.shaderEffectRanges).toEqual([]); + expect(removed.document?.shaderEffects).toEqual([]); + }); + + it("says when nothing could check the code", () => { + const added = executeAgentTool( + fixtureDocument(), + "addEffect", + JSON.stringify({ + startSec: 1, + endSec: 2, + effect: { name: "Dim", description: "", code: CODE, params }, + }), + ); + expect(JSON.parse(added.resultJson)).toMatchObject({ validated: false, cost: null }); + }); +}); + +describe("the executor's view of a check asked beforehand", () => { + const source = { + code: "fn effect(uv: vec2) -> vec4 { return frame(uv); }", + params: [{ name: "level", type: "number" as const }], + }; + const check = { tier: "light" as const, frameReads: 1, iterations: 0 }; + + it("is the answer for the same code and layout, and nothing for any other", () => { + const lookup = shaderEffectCheckLookup(source, check); + expect(lookup(source.code, [{ name: "level", type: "number" }])).toBe(check); + expect(lookup(`${source.code}\n`, source.params)).toBeNull(); + expect(lookup(source.code, [{ name: "level", type: "color" }])).toBeNull(); + expect(lookup(source.code, [])).toBeNull(); + }); + + it("is asked only by the two tools that compile", () => { + const document = fixtureDocument(); + expect( + shaderEffectToCheck(document, "addEffect", { startSec: 0, endSec: 1, effectId: "fx_1" }), + ).toBeNull(); + expect(shaderEffectToCheck(document, "addTrim", { startSec: 0, endSec: 1 })).toBeNull(); + expect( + shaderEffectToCheck(document, "addEffect", { + startSec: 0, + endSec: 1, + effect: { name: "A", description: "", code: source.code }, + }), + ).toEqual({ code: source.code, params: [] }); + expect( + shaderEffectToCheck(document, "setEffectDefinition", { effectId: "fx_missing", code: "x" }), + ).toBeNull(); + }); +}); diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 3a60dd38d8..50bb5de41c 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -47,10 +47,23 @@ import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcrip import { type AxcutAnnotationRegion, type AxcutDocument, + type AxcutShaderEffect, + type AxcutShaderEffectRange, blurDataSchema, clipCropRegionSchema, figureDataSchema, + shaderEffectParamSchema, + shaderEffectSchema, + shaderEffectValueSchema, } from "../../src/lib/ai-edition/schema"; +import { + findShaderEffect, + SHADER_EFFECT_CONTRACT, + type ShaderEffectCheck, + type ShaderEffectCheckLookup, + shaderEffectCostAdvice, + storedShaderEffectValues, +} from "../../src/lib/ai-edition/shaderEffects"; import { getEditorSettings } from "../../src/lib/ai-edition/store/editorSettings"; import { appendAutoZoomSuggestions } from "../../src/lib/ai-edition/timeline/apply-auto-zooms"; import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera"; @@ -135,6 +148,8 @@ function modifierIds(document: AxcutDocument, kind: ModifierKind): string[] { // `removeRegion` applies the same grouping by routing this kind through // `removeAudioTrack`. return [...new Set(document.audioTracks.map(trackGroupId))]; + case "shaderEffect": + return (document.shaderEffectRanges ?? []).map((region) => region.id); } } @@ -590,6 +605,7 @@ function modifierIdsOf(document: AxcutDocument): string[] { ...document.annotations.map((r) => r.id), ...speedRegions.map((r) => r.id), ...cameraFullscreenRegions.map((r) => r.id), + ...(document.shaderEffectRanges ?? []).map((r) => r.id), ]; } @@ -987,6 +1003,45 @@ export const addCameraFullscreenArgs = z.object({ endSec: secondsSchema, }); +const effectTargetSchema = z.enum(["image", "text", "both"]); +const effectValuesSchema = z.record(z.string(), shaderEffectValueSchema); + +export const addEffectArgs = z.object({ + startSec: secondsSchema, + endSec: secondsSchema, + effectId: z.string().min(1).optional(), + effect: z + .object({ + name: z.string().min(1).max(60), + description: z.string().max(500), + code: z.string().min(1).max(65_536), + params: z.array(shaderEffectParamSchema).max(32).optional(), + }) + .optional(), + values: effectValuesSchema.optional(), + target: effectTargetSchema.optional(), +}); + +export const setEffectArgs = z.object({ + effectRegionId: z.string().min(1), + startSec: secondsSchema.optional(), + endSec: secondsSchema.optional(), + values: effectValuesSchema.optional(), + target: effectTargetSchema.optional(), +}); + +export const setEffectDefinitionArgs = z.object({ + effectId: z.string().min(1), + name: z.string().min(1).max(60).optional(), + description: z.string().max(500).optional(), + code: z.string().min(1).max(65_536).optional(), + params: z.array(shaderEffectParamSchema).max(32).optional(), +}); + +export const getEffectArgs = z.object({ + effectId: z.string().min(1), +}); + export const setCameraFullscreenArgs = z.object({ cameraFullscreenId: z.string().min(1), startSec: secondsSchema.optional(), @@ -1062,6 +1117,10 @@ export const OPENSCREEN_TOOL_NAMES = [ "setAnnotation", "addCameraFullscreen", "setCameraFullscreen", + "addEffect", + "setEffect", + "setEffectDefinition", + "getEffect", "addAudio", "setAudio", "removeTrim", @@ -1140,6 +1199,9 @@ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ "setAnnotation", "addCameraFullscreen", "setCameraFullscreen", + "addEffect", + "setEffect", + "setEffectDefinition", "addAudio", "setAudio", "removeTrim", @@ -1281,7 +1343,7 @@ export function documentSnapshotForModel( const autoFocusAll = legacy?.autoFocusAll === true; return { timeBaseNote: - "clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions and audioTracks are in virtual (edited-timeline) seconds: the editor's ruler, where a trim leaves a hole instead of shortening it and a speed region stretches nothing. The exported video's clock skips those holes and plays sped-up spans faster, so it differs from the ruler once there are any.", + "clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions, effects and audioTracks are in virtual (edited-timeline) seconds: the editor's ruler, where a trim leaves a hole instead of shortening it and a speed region stretches nothing. The exported video's clock skips those holes and plays sped-up spans faster, so it differs from the ruler once there are any.", audioNote: "audioTracks are imported voiceover / music files laid over the recording. They are clip-anchored like every other region, so they travel with their clip through reorder and trim, and they play at 1x whatever a speed region does to the picture under them. addAudio places an EXISTING asset of kind 'audio': if the project has none, a file has to be imported first (importMedia, where it is offered), so never invent an id.", zoomNote: @@ -1408,6 +1470,28 @@ export function documentSnapshotForModel( startSec: roundSec(c.startMs), endSec: roundSec(c.endMs), })), + // Where each effect is posed, and how. The code is not here: getEffect reads it. + effects: coalesceForAgent(document.shaderEffectRanges ?? []).map((r) => ({ + id: r.id, + effectId: r.effectId, + name: findShaderEffect(document, r.effectId)?.name ?? null, + startSec: roundSec(r.startMs), + endSec: roundSec(r.endMs), + target: r.target, + values: r.values, + })), + effectDefinitions: (document.shaderEffects ?? []).map((e) => ({ + effectId: e.id, + name: e.name, + description: e.description, + contract: e.contract, + lines: e.code.split("\n").length, + regions: coalesceForAgent( + (document.shaderEffectRanges ?? []).filter((r) => r.effectId === e.id), + ).length, + cost: e.cost ?? null, + params: e.params, + })), // Imported audio, collapsed to the pills the ruler draws — a track ventilated // across a clip boundary is several fragments the user sees as one thing, and // the model has to name what the user sees. @@ -1432,6 +1516,109 @@ export function documentSnapshotForModel( }; } +/** A definition as it will be stored: schema-checked, then compiled by the compositor when one is + * wired, its cost cached. An error reads as the agent must act on it. */ +function checkedEffectDefinition( + candidate: unknown, + checker: ShaderEffectCheckLookup | undefined, +): { definition: AxcutShaderEffect; check: ShaderEffectCheck | null } | { error: string } { + const parsed = shaderEffectSchema.safeParse(candidate); + if (!parsed.success) { + return { + error: parsed.error.issues + .map((i) => `${i.path.join(".") || "effect"}: ${i.message}`) + .join("; "), + }; + } + const definition = parsed.data; + const check = checker?.(definition.code, definition.params) ?? null; + if (check?.error) return { error: `The effect does not compile:\n${check.error}` }; + const cost = + check?.tier && typeof check.frameReads === "number" + ? { tier: check.tier, frameReads: check.frameReads } + : undefined; + return { definition: { ...definition, cost }, check }; +} + +type ShaderEffectSource = { + code: string; + params: Array<{ name: string; type: AxcutShaderEffect["params"][number]["type"] }>; +}; + +/** The code and parameter layout `addEffect` or `setEffectDefinition` will have the compositor + * check, read from the same arguments the executor reads; `null` for a call that compiles nothing. + * The check is asynchronous (a Metal compile takes 120 ms and more) and the executor is not: the + * async wrapper asks first, then hands the answer over (`shaderEffectCheckLookup`). */ +export function shaderEffectToCheck( + document: AxcutDocument, + name: string, + args: unknown, +): ShaderEffectSource | null { + const layout = ( + params: ReadonlyArray>, + ) => params.map(({ name, type }) => ({ name, type })); + if (name === "addEffect") { + const parsed = addEffectArgs.safeParse(args); + if (!parsed.success || !parsed.data.effect) return null; + return { code: parsed.data.effect.code, params: layout(parsed.data.effect.params ?? []) }; + } + if (name === "setEffectDefinition") { + const parsed = setEffectDefinitionArgs.safeParse(args); + if (!parsed.success) return null; + const { effectId, code, params } = parsed.data; + const existing = findShaderEffect(document, effectId); + if (!existing || (code === undefined && params === undefined)) return null; + return { code: code ?? existing.code, params: layout(params ?? existing.params) }; + } + return null; +} + +/** The executor's view of an answer asked for `source`: that answer for the same code and + * layout, `null` (not checked) for anything else. */ +export function shaderEffectCheckLookup( + source: ShaderEffectSource, + check: ShaderEffectCheck | null, +): ShaderEffectCheckLookup { + return (code, params) => + code === source.code && + params.length === source.params.length && + params.every((p, i) => p.name === source.params[i].name && p.type === source.params[i].type) + ? check + : null; +} + +/** One effect at a time: the refusal names the one in the way, and the way out. */ +function effectBlocked( + ranges: AxcutShaderEffectRange[], + startMs: number, + endMs: number, + except: ReadonlySet = new Set(), +): string | null { + const wall = coalesceRegionsForRuler(ranges.filter((r) => !except.has(r.id))).find( + (pill) => pill.start * 1000 < endMs && pill.end * 1000 > startMs, + ); + if (!wall) return null; + return ( + `${formatSec(startMs / 1000)} – ${formatSec(endMs / 1000)} would run into effect ${wall.ids[0]} ` + + `at ${formatSec(wall.start)} – ${formatSec(wall.end)}, and one effect runs at a time. To combine ` + + "two looks, write them in one shader (setEffectDefinition); otherwise pick a span clear of it." + ); +} + +/** What an effect costs: its tier, and a recommendation when the tier runs longer than it is + * meant to. */ +function effectCostReport(definition: AxcutShaderEffect, startSec: number, endSec: number) { + if (!definition.cost) return { cost: null }; + const advice = shaderEffectCostAdvice(definition.cost.tier, endSec - startSec); + return { + cost: { + tier: definition.cost.tier, + frameReads: definition.cost.frameReads, + ...(advice ? { advice } : {}), + }, + }; +} + function failure(message: string): AgentToolExecution { return { ok: false, resultJson: JSON.stringify({ error: message }) }; } @@ -1578,6 +1765,10 @@ export interface AgentToolOptions { /** Whether this machine can cut out the camera background, asked only by the two tools that * report it. Absent where nothing can ask the compositor: reported as not checked. */ cameraSegmentation?: () => SegmentationSupport; + /** The compositor's check of the WGSL this call writes (`checkShaderEffect`), asked beforehand + * by the async wrapper (`shaderEffectToCheck`). Absent where nothing can: the effect is stored + * unchecked, and the result says `validated: false`. */ + checkShaderEffect?: ShaderEffectCheckLookup; /** `addAutoZooms`' plan, made by the async wrapper from the recorded clicks. Absent means * no cursor reader is wired here, which the tool reports as "could not look". */ autoZooms?: AutoZoomPlan; @@ -3169,6 +3360,205 @@ export function executeAgentTool( }; } + case "addEffect": { + const parsed = addEffectArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { effectId, effect, values = {}, target = "image" } = parsed.data; + if ((effectId === undefined) === (effect === undefined)) { + return failure( + "Pass either `effect` to write a new effect, or `effectId` to reuse one of this project's (getCurrentDocument lists them under effectDefinitions), not both.", + ); + } + let definition: AxcutShaderEffect; + let check: ShaderEffectCheck | null = null; + const created = effect !== undefined; + if (effect) { + const checked = checkedEffectDefinition( + { + id: createId("fx"), + ...effect, + contract: SHADER_EFFECT_CONTRACT, + params: effect.params ?? [], + origin: "agent", + }, + options?.checkShaderEffect, + ); + if ("error" in checked) return failure(checked.error); + ({ definition, check } = checked); + } else { + const existing = findShaderEffect(document, effectId as string); + if (!existing) return failure(`Unknown effect: ${effectId}.`); + definition = existing; + } + const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec)); + const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec)); + const blocked = effectBlocked(document.shaderEffectRanges ?? [], startMs, endMs); + if (blocked) return failure(`The effect was NOT placed: ${blocked}`); + const region = { + id: createId("fxr"), + startMs, + endMs, + effectId: definition.id, + values: storedShaderEffectValues(definition, values), + target, + }; + const placed = anchorForAgent(region, document, "fxr"); + const landing = landingOf(placed, document); + if (!landing.anchored) return coversNoClip("effect", startMs / 1000, endMs / 1000, document); + const next: AxcutDocument = { + ...document, + shaderEffects: created + ? [...(document.shaderEffects ?? []), definition] + : (document.shaderEffects ?? []), + shaderEffectRanges: [...(document.shaderEffectRanges ?? []), ...placed], + }; + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + effectRegionId: landing.ids[0], + effectId: definition.id, + ...landingReport(landing, startMs / 1000, endMs / 1000), + ...(created ? { validated: check !== null } : {}), + ...effectCostReport(definition, landing.startSec, landing.endSec), + }), + summary: + `effect "${definition.name}" ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + + landingSuffix(landing, startMs / 1000, endMs / 1000), + }; + } + + case "setEffect": { + const parsed = setEffectArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const prev = document.shaderEffectRanges ?? []; + const existing = prev.find((r) => r.id === parsed.data.effectRegionId); + if (!existing) return failure(`Unknown effect region: ${parsed.data.effectRegionId}`); + const definition = findShaderEffect(document, existing.effectId); + if (!definition) return failure(`Effect ${existing.effectId} is missing from the project.`); + const pillIds = new Set(resolvePillIds(prev, existing.id)); + const pill = coalesceRegionsForRuler(prev).find((p) => p.ids.includes(existing.id)); + const { values, target } = parsed.data; + const { startMs, endMs } = resolveSpanMs( + pill + ? { startMs: Math.round(pill.start * 1000), endMs: Math.round(pill.end * 1000) } + : existing, + parsed.data.startSec, + parsed.data.endSec, + ); + const blocked = effectBlocked(prev, startMs, endMs, pillIds); + if (blocked) return failure(`The effect was NOT changed: ${blocked}`); + const moved = parsed.data.startSec !== undefined || parsed.data.endSec !== undefined; + // The span is replaced before the patch: patched first, the pill could take the same + // values as a touching neighbour, coalesce with it, and move it along. + const spanned = moved + ? replacePillSpan(prev, existing.id, startMs, endMs, document.timeline.clips, () => + createId("fxr"), + ) + : prev; + const others = new Set(prev.filter((r) => !pillIds.has(r.id)).map((r) => r.id)); + const rebuilt = spanned.map((r) => + others.has(r.id) + ? r + : { + ...r, + values: + values === undefined + ? r.values + : storedShaderEffectValues(definition, { ...r.values, ...values }), + target: target ?? r.target, + }, + ); + const landing = landingAfterPillEdit(prev, rebuilt, pillIds, document); + if (!landing.anchored) return coversNoClip("effect", startMs / 1000, endMs / 1000, document); + const next: AxcutDocument = { ...document, shaderEffectRanges: rebuilt }; + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + effectRegionId: landing.ids[0] ?? existing.id, + effectId: definition.id, + ...landingReport(landing, startMs / 1000, endMs / 1000), + ...effectCostReport(definition, landing.startSec, landing.endSec), + }), + summary: + `updated effect "${definition.name}" ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + + landingSuffix(landing, startMs / 1000, endMs / 1000), + }; + } + + case "setEffectDefinition": { + const parsed = setEffectDefinitionArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { effectId, ...patch } = parsed.data; + const existing = findShaderEffect(document, effectId); + if (!existing) return failure(`Unknown effect: ${effectId}.`); + const recompile = patch.code !== undefined || patch.params !== undefined; + const candidate = { + ...existing, + ...Object.fromEntries(Object.entries(patch).filter(([, v]) => v !== undefined)), + cost: recompile ? undefined : existing.cost, + }; + let definition: AxcutShaderEffect; + let check: ShaderEffectCheck | null = null; + if (recompile) { + const checked = checkedEffectDefinition(candidate, options?.checkShaderEffect); + if ("error" in checked) return failure(checked.error); + ({ definition, check } = checked); + } else { + const checked = shaderEffectSchema.safeParse(candidate); + if (!checked.success) return failure(checked.error.message); + definition = checked.data; + } + // The regions follow the new parameters: a removed one's value drops, bounds re-clamp. + const next: AxcutDocument = { + ...document, + shaderEffects: (document.shaderEffects ?? []).map((e) => + e.id === effectId ? definition : e, + ), + shaderEffectRanges: (document.shaderEffectRanges ?? []).map((r) => + r.effectId === effectId + ? { ...r, values: storedShaderEffectValues(definition, r.values) } + : r, + ), + }; + const regions = coalesceForAgent( + (next.shaderEffectRanges ?? []).filter((r) => r.effectId === effectId), + ); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + effectId, + regions: regions.length, + ...(recompile ? { validated: check !== null } : {}), + ...(definition.cost ? { cost: definition.cost } : {}), + }), + summary: `updated effect "${definition.name}" (${regions.length} region${regions.length === 1 ? "" : "s"})`, + }; + } + + case "getEffect": { + const parsed = getEffectArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const effect = findShaderEffect(document, parsed.data.effectId); + if (!effect) return failure(`Unknown effect: ${parsed.data.effectId}.`); + const regions = coalesceForAgent( + (document.shaderEffectRanges ?? []).filter((r) => r.effectId === effect.id), + ).map((r) => ({ + id: r.id, + startSec: roundSec(r.startMs), + endSec: roundSec(r.endMs), + target: r.target, + values: r.values, + })); + return { + ok: true, + resultJson: JSON.stringify({ ...effect, regions }), + summary: `read effect "${effect.name}"`, + }; + } + case "setCameraFullscreen": { const parsed = setCameraFullscreenArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); @@ -3411,9 +3801,10 @@ export function executeAgentTool( else if (speedRegions.some((s) => s.id === id)) kind = "speed"; else if (cameraFullscreenRegions.some((c) => c.id === id)) kind = "cameraFullscreen"; else if (document.audioTracks.some((t) => trackGroupId(t) === id)) kind = "audio"; + else if ((document.shaderEffectRanges ?? []).some((r) => r.id === id)) kind = "shaderEffect"; if (!kind) { return failure( - `No zoom / speed / annotation / full-camera / audio modifier with id ${id}. ` + + `No zoom / speed / annotation / full-camera / audio / effect modifier with id ${id}. ` + `For a trim use removeTrim; for a clip use removeClip.`, ); } diff --git a/electron/ai-edition/chat-service.ts b/electron/ai-edition/chat-service.ts index e1e55c94c9..c222a8de44 100644 --- a/electron/ai-edition/chat-service.ts +++ b/electron/ai-edition/chat-service.ts @@ -16,6 +16,7 @@ import { createEmptyDocument, documentSchema, } from "../../src/lib/ai-edition/schema"; +import type { ShaderEffectChecker } from "../../src/lib/ai-edition/shaderEffects"; import type { AiEditionChatMessage, AiEditionChatResult, @@ -322,6 +323,8 @@ export interface ChatRunEnv { cursor?: CursorTelemetryReader; /** Whether this machine can cut out the camera background, asked of the compositor. */ cameraSegmentation?: () => SegmentationSupport; + /** The compositor's check of an agent-written effect's WGSL. */ + checkShaderEffect?: ShaderEffectChecker; /** Runtime-only model retry override used by bounded measurement harnesses. */ maxRetries?: number; } @@ -468,6 +471,7 @@ export async function runChat( editsAllowed, cursor: env.cursor, cameraSegmentation: env.cameraSegmentation, + checkShaderEffect: env.checkShaderEffect, }); // The project was deleted while the agent ran: its sessions are gone, so a diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 8fa62ea514..fffbeb4c53 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -83,6 +83,19 @@ const ARGS: Record = { setAnnotation: { annotationId: "ann_nope" }, addCameraFullscreen: { startSec: 1, endSec: 2 }, setCameraFullscreen: { cameraFullscreenId: "cam_nope" }, + // No checker in these tests: the effect is stored unchecked, and the result says so. + addEffect: { + startSec: 1, + endSec: 2, + effect: { + name: "Pass-through", + description: "Shows the frame unchanged.", + code: "fn effect(uv: vec2) -> vec4 { return frame(uv); }", + }, + }, + setEffect: { effectRegionId: "fxr_nope" }, + setEffectDefinition: { effectId: "fx_nope" }, + getEffect: { effectId: "fx_nope" }, // The fixture has no `kind: "audio"` asset, so these exercise the refusal branch — // the honest one to pin: the agent can place imported audio, never import it. addAudio: { assetId: "audio_nope", startSec: 1, endSec: 2 }, @@ -389,6 +402,7 @@ describe("one description of the tools, not two", () => { "getTranscript", "getTranscriptWords", "getCursorTrack", + "getEffect", "getCaptions", "getCaptionTranslationUnits", ]); @@ -766,3 +780,104 @@ describe("addAutoZooms reads the clicks through the runtime's cursor reader", () expect(result.resultJson).toContain("could not be read"); }); }); + +describe("an effect's code is checked off the main thread, before the executor reads it", () => { + const LEVEL = { type: "number", name: "level", label: "Level", default: 1, min: 0, max: 2 }; + const effect = (code: string) => ({ + name: "Dim", + description: "Dims the picture.", + code, + params: [LEVEL], + }); + // The compositor's check takes a while (a Metal compile is 120 ms and more): a promise that + // settles later, refusing `nope` and pricing the rest at 20 reads. + const slowChecker = (asked: Array<{ code: string; params: unknown }>) => ({ + checkShaderEffect: async ( + code: string, + params: ReadonlyArray<{ name: string; type: string }>, + ) => { + asked.push({ code, params }); + await new Promise((resolve) => setTimeout(resolve, 5)); + return code.includes("nope") + ? { error: "error: no definition in scope for identifier: `nope`" } + : { tier: "medium" as const, frameReads: 20, iterations: 20 }; + }, + }); + + it("waits for the compositor's answer, accepting or refusing on it", async () => { + const asked: Array<{ code: string; params: unknown }> = []; + const runtime = slowChecker(asked); + const accepted = await runDocumentTool( + fixtureDocument(), + "addEffect", + { + startSec: 1, + endSec: 2, + effect: effect("fn effect(uv: vec2) -> vec4 { return frame(uv); }"), + }, + true, + runtime, + ); + expect(accepted.ok).toBe(true); + const result = JSON.parse(accepted.resultJson); + expect(result.validated).toBe(true); + expect(result.cost).toMatchObject({ tier: "medium", frameReads: 20 }); + expect(asked[0].params).toEqual([{ name: "level", type: "number" }]); + + const refused = await runDocumentTool( + fixtureDocument(), + "addEffect", + { startSec: 1, endSec: 2, effect: effect("fn effect(uv: vec2) -> vec4 { nope }") }, + true, + runtime, + ); + expect(refused.ok).toBe(false); + expect(refused.resultJson).toContain("does not compile"); + }); + + it("checks a new code against the definition's parameters when only the code changes", async () => { + const asked: Array<{ code: string; params: unknown }> = []; + const runtime = slowChecker(asked); + const added = await runDocumentTool( + fixtureDocument(), + "addEffect", + { + startSec: 1, + endSec: 2, + effect: effect("fn effect(uv: vec2) -> vec4 { return frame(uv); }"), + }, + true, + runtime, + ); + const { effectId } = JSON.parse(added.resultJson); + const document = added.document as AxcutDocument; + const code = "fn effect(uv: vec2) -> vec4 { return frame(uv) * params.level; }"; + const changed = await runDocumentTool( + document, + "setEffectDefinition", + { effectId, code }, + true, + runtime, + ); + expect(changed.ok).toBe(true); + expect(asked.at(-1)).toEqual({ code, params: [{ name: "level", type: "number" }] }); + }); + + it("asks nothing when edits are off, or for a tool that compiles nothing", async () => { + const asked: Array<{ code: string; params: unknown }> = []; + const runtime = slowChecker(asked); + await runDocumentTool( + fixtureDocument(), + "addEffect", + { + startSec: 1, + endSec: 2, + effect: effect("fn effect(uv: vec2) -> vec4 { return frame(uv); }"), + }, + false, + runtime, + ); + await runDocumentTool(fixtureDocument(), "addTrim", { startSec: 1, endSec: 2 }, true, runtime); + expect(asked).toEqual([]); + }); +}); diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 44571c2dd9..69e0b85a8e 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -35,6 +35,7 @@ import { import { AUDIO_FADE_MAX_MS } from "../../../src/lib/ai-edition/document/audioTracks"; import { VOCABULARY_MAX_CHARS } from "../../../src/lib/ai-edition/document/transcript"; import type { AxcutDocument } from "../../../src/lib/ai-edition/schema"; +import type { ShaderEffectChecker } from "../../../src/lib/ai-edition/shaderEffects"; import { collectAutoZoomSuggestionsForDocument } from "../../../src/lib/ai-edition/timeline/apply-auto-zooms"; // ponytail: the legend is DERIVED from the depth→scale table, not retyped. Both // tool descriptions used to assert "depth 1–6 maps to 1.0×–3.5×" — a formula @@ -55,6 +56,7 @@ import { addAudioArgs, addAutoZoomsArgs, addCameraFullscreenArgs, + addEffectArgs, addSpeedArgs, addTrimArgs, addTrimsArgs, @@ -64,6 +66,7 @@ import { duplicateClipArgs, executeAgentTool, getCursorTrackArgs, + getEffectArgs, getTranscriptArgs, getTranscriptWordsArgs, insertClipArgs, @@ -80,10 +83,14 @@ import { setAudioArgs, setCameraFullscreenArgs, setClipRangeArgs, + setEffectArgs, + setEffectDefinitionArgs, setSpeedArgs, setTrimArgs, setWordTextArgs, setZoomArgs, + shaderEffectCheckLookup, + shaderEffectToCheck, } from "../agent-tools"; import { getCaptionsArgs, @@ -149,7 +156,7 @@ const BASE_SYSTEM_PROMPT = [ "You can call the tools below against the live document snapshot; the runtime executes each edit and feeds the result back into the loop.", "The AxcutDocument is the single source of truth. The timeline, the transcript editor, and the chat panel are all direct editors of the same document — when the user places a clip on the timeline, the document updates immediately, and when the timeline is empty, the document has no clips. Your edits operate on the live document, so preserve the user's placed clips.", "", - "Time-bases (do not mix them up): clips and trims are in SOURCE-time seconds of an asset; zooms, speed regions, annotations and camera-fullscreen regions are in VIRTUAL (edited-timeline) seconds — the position on the editor's ruler, where the clips sit end to end. A trim leaves a hole in that ruler rather than shortening it, and a speed region does not stretch it, so the exported video's own clock differs from it as soon as there is a trim or a speed region: always place effects in ruler seconds. getCurrentDocument returns all of them, clearly labelled.", + "Time-bases (do not mix them up): clips and trims are in SOURCE-time seconds of an asset; zooms, speed regions, annotations, camera-fullscreen regions and effects are in VIRTUAL (edited-timeline) seconds — the position on the editor's ruler, where the clips sit end to end. A trim leaves a hole in that ruler rather than shortening it, and a speed region does not stretch it, so the exported video's own clock differs from it as soon as there is a trim or a speed region: always place effects in ruler seconds. getCurrentDocument returns all of them, clearly labelled.", "", // ponytail: these describe what each tool is FOR, deliberately without quoting // user phrasings. A phrase→tool table reads as helpful and is not: it swaps the @@ -167,6 +174,7 @@ const BASE_SYSTEM_PROMPT = [ "- addAudio lays an imported voiceover or music file over a span. It plays an asset the project already has (kind 'audio'); importing or recording one is the editor's job, not a tool you have — so when the project has none, say so rather than naming an id that does not exist.", "- moveClip changes the order of placed clips, one call per clip that moves, preserving ids, source ranges, trims and anchored effects. replaceTimeline rebuilds the timeline from kept intervals and sorts them, so it cannot reorder anything.", "- Captions are the transcript drawn on the video, never typed in: setCaptionSettings turns them on, styles and places them, getCaptions reads them back. To caption in another language, read the phrases with getCaptionTranslationUnits, translate them yourself, store them with putCaptionTranslation, then show that language with setCaptionSettings. A translation never rewrites the transcript. For text the speaker did not say, use addAnnotation.", + "- A visual effect the settings do not offer (a ripple on clicks, a glitch on a cut, a color grade) is written as a shader with addEffect; setEffect re-tunes or moves it, setEffectDefinition changes its code, getEffect reads its code.", "- Deleting is a first-class action, not a workaround: removeTrim, removeModifier, removeClip. Never fake a deletion by re-adding an element or zeroing it out (span 0, speed 1×) — that leaves it in the document and misreports what you did.", "If nothing in the list does what was asked, say so; do not approximate it with a bigger tool.", "", @@ -212,7 +220,7 @@ const CUT_TRANSITIONS_RESULT = `When the result cuts a zoom's move in or out, it export const TOOL_DESCRIPTIONS: Record = { getCurrentDocument: - "Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation / camera-fullscreen / audio regions (virtual, edited-timeline time), each with the settings its writes take: a zoom's level, focus mode and hidden cursor, an annotation's type, box, look and arrow or blur data, an audio track's level and fades. It also gives the project's look under `settings`, in the shape setEditorSettings takes, with `settingsContext`: the formats, wallpapers and cursor themes on offer, and under `inactive` each setting the editor hides in the current state and why. Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.", + "Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation / camera-fullscreen / audio / effect regions (virtual, edited-timeline time), each with the settings its writes take: a zoom's level, focus mode and hidden cursor, an annotation's type, box, look and arrow or blur data, an audio track's level and fades. It also gives the project's look under `settings`, in the shape setEditorSettings takes, with `settingsContext`: the formats, wallpapers and cursor themes on offer, and under `inactive` each setting the editor hides in the current state and why. Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.", getTranscript: "Read the transcript segments (speech and silence, with start/end seconds and text) for an asset. Omit assetId to read the primary asset's transcript.", getCursorTrack: @@ -256,6 +264,14 @@ export const TOOL_DESCRIPTIONS: Record = { 'Change the project\'s look: pass only the settings to change, named as getCurrentDocument returns them under `settings`. Background: wallpaper (a bundled path from settingsContext.bundledWallpapers, a CSS colour or gradient, or a base64 PNG/JPEG data URL; never a file path or web URL; an embedded image reads back as a marker, leave the key out to keep it), wallpaperMotion (moves a gradient or image only), backgroundBlur 0–1. Frame: aspectRatio ("auto", a preset, or an originalAspectRatios token; a fixed format picked for the first time fills the frame with a window that follows the cursor, which formatFollowCursor turns off when settingsContext.formatFill is "available"), padding 0–100 %, borderRadius 0–64 px, shadowIntensity 0–1, frame (none, window, laptop, phone, monitor) and frameTheme (light, dark), motionBlurAmount 0–1, depthOfField (blurs 3D-tilted zooms only). Camera: webcamLayoutPreset, webcamMirrored, webcamReactiveZoom (shrinks on zoom), webcamMaskShape (square or rectangle, the camera\'s own proportions; an untouched roundness follows it), webcamRoundness 0–1, webcamSizePreset 15–60, webcamAnchor (a corner or an edge middle), webcamCropRegion (the framing: a square share of the picture, width = height ≥ 1/3) and webcamCropPan (where that window sits, 0–1 per axis, 0.5 centred), webcamBackgroundMode (none, transparent, blur, custom: needs settingsContext.cameraSegmentation ready), webcamBlurIntensity 0–1, webcamWallpaper (as wallpaper). Cursor, under `cursor`: show, autoHide, theme (settingsContext.cursorThemes), asArrow (the recorded cursor kinds drawn as the arrow instead), size 1.5–6, smoothing 0–1, motionBlur 0–1, clickBounce 0–2, clickImpact, clickSound (only heard when the take recorded clicks) and clickSoundGainDb −24–0, model3d. Also autoFocusAll (every zoom follows the cursor) and audioGainDb −12–12 dB, the master volume. A value out of range or a key not listed is refused with the reason, and nothing changes. The result lists what was applied, including what the editor changes with it, and `inactive` for any of those the current state leaves without effect.', addCameraFullscreen: "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.", + addEffect: + "Write a visual effect as a shader and pose it over a span of the edited timeline (virtual seconds), or pose one of this project's effects again (effectId). Use it for a look the settings do not offer. `effect.code` is WGSL that defines `fn effect(uv: vec2) -> vec4`, called for every pixel; uv is 0..1 with (0,0) top-left. Available inside it: `frame(uv)` samples the composed picture; `params.` reads each declared parameter; `ctx.time` is programme seconds; `ctx.region.x` goes 0 to 1 across the region (y its length in s, z seconds since it began); `ctx.resolution` and `ctx.aspect`; `ctx.cursor` (xy in uv, z = 1 when on screen); `ctx.clicks[0..3]`, the last four clicks, newest first (xy in uv where it happened, z seconds since, w = 1 when the slot holds one); `ctx.footage` and `ctx.camera`, the recording's and the webcam's rects (x, y, w, h in uv; camera zero-sized when absent) — all already placed through zoom and layout; `fx_to_footage(uv)` and `fx_from_footage(p)` map between output uv and the recording's own surface (0..1 across the recording) through the warp the compositor draws it with, zoom, layout and 3D camera included, and `ctx.footage_aspect` is its width over height: the effect works in whichever of the two spaces fits it; `fx_px(n)`, n pixels of a 1080p frame as a fraction of the height; `fx_hash(vec2)`, a random number in [0,1) identical on every GPU. Rules: the effect is a pure function of those inputs (no state between frames), so the preview and the export show the same pixels; reason in uv or fx_px, never raw pixels; every loop is a `for` with literal bounds (`for (var i = 0; i < 16; i++)`), at most 4096 iterations per pixel — the one hard rule. `params` declares up to 8 (recommended) parameters, each `{type: number|toggle|color|point, name, label, default}` (number also `min`, `max`); declare one only when moving it visibly changes the effect over its whole range. `target`: `image` (default: the picture under the annotations and captions), `text` (the annotations and captions alone, premultiplied alpha), `both` (everything); use text or both only when asked. One effect runs at a time: to combine two looks, write them in one shader. The result gives the cost tier, in frame() reads per pixel, with what a 1080p frame pays on a fast discrete GPU (RTX 4070 Ti) and on an integrated one (Apple M1): light ≤ 8, up to 0.06 ms and 1.3 ms; medium ≤ 32, 0.14 ms and 4.3 ms; heavy ≤ 128, 0.54 ms and 16 ms; very heavy beyond, 2 ms and 49 ms at 512 reads. It also gives a recommendation when the tier runs long: light anywhere, medium up to a minute, heavy up to 10 s, very heavy a few seconds. It is a recommendation, never a refusal. Code that does not compile is refused with the compiler's error at the line of your code: fix it and call again. Check the result with getFrames.", + setEffect: + "Move, resize or re-tune an effect region by id (virtual seconds): its span, its parameter `values`, or its `target`. Only the fields you pass change; a value equal to the default is dropped. Refused if the new span runs into another effect: one runs at a time.", + setEffectDefinition: + "Change an effect itself by effectId — its name, description, WGSL `code` or `params` — and with it every region that uses it. New code is compiled first and refused with the compiler's error if it does not compile. A removed parameter's values drop; the bounds re-clamp the others. Read the current code with getEffect before rewriting it.", + getEffect: + "Read one effect of the project in full: its WGSL code, parameters, description, cost and the regions that use it. getCurrentDocument lists effects without their code.", setCameraFullscreen: "Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.", addAudio: `Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them. If there is none, a file has to be imported first (importMedia, where it is offered): never guess an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it), fadeInMs / fadeOutMs its fades (0 to ${AUDIO_FADE_MAX_MS} ms, default none; a track too short for both shares its span between them). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.`, @@ -263,7 +279,7 @@ export const TOOL_DESCRIPTIONS: Record = { removeTrim: "Delete a trim range by id — the cut is undone and that span plays/exports again. This is how you 'remove a trim'; never re-add a trim to undo one.", removeModifier: - "Delete a modifier (zoom / speed / annotation / camera-fullscreen / audio) by id; the kind is resolved from the id. Touching rows with the same styling render as one pill and are deleted together, so read removedIds in the result for the complete set that disappeared. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.", + "Delete a modifier (zoom / speed / annotation / camera-fullscreen / audio / effect) by id; the kind is resolved from the id. Touching rows with the same styling render as one pill and are deleted together, so read removedIds in the result for the complete set that disappeared. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.", removeClip: "Delete a placed clip by id; remaining clips close the gap and effects anchored to it are dropped. Use only when the user asks to remove a clip — to shorten one, use setClipRange.", getCaptions: @@ -310,6 +326,8 @@ export interface ToolRuntime { availableByAssetId?: Record; /** Asks the compositor whether this machine can cut out the camera background. */ cameraSegmentation?: () => SegmentationSupport; + /** Asks the compositor to check an effect's WGSL. */ + checkShaderEffect?: ShaderEffectChecker; } /** @@ -389,10 +407,22 @@ export async function runDocumentTool( : undefined; const autoZooms = name === "addAutoZooms" && editsAllowed ? await planAutoZooms(document, runtime) : undefined; + // The compositor compiles an effect's code for real, off the main thread: asked here, before + // the synchronous executor reads the answer. + const effectSource = + editsAllowed && runtime.checkShaderEffect ? shaderEffectToCheck(document, name, args) : null; + const checkShaderEffect = + effectSource && runtime.checkShaderEffect + ? shaderEffectCheckLookup( + effectSource, + await runtime.checkShaderEffect(effectSource.code, effectSource.params), + ) + : undefined; return executeAgentTool(document, name, JSON.stringify(args ?? {}), { editsAllowed, cursorTelemetry: { availableByAssetId: runtime.availableByAssetId, load }, cameraSegmentation: runtime.cameraSegmentation, + checkShaderEffect, autoZooms, }); } @@ -504,6 +534,10 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray = [ ["setAnnotation", setAnnotationArgs], ["addCameraFullscreen", addCameraFullscreenArgs], ["setCameraFullscreen", setCameraFullscreenArgs], + ["addEffect", addEffectArgs], + ["setEffect", setEffectArgs], + ["setEffectDefinition", setEffectDefinitionArgs], + ["getEffect", getEffectArgs], ["addAudio", addAudioArgs], ["setAudio", setAudioArgs], ["removeTrim", removeTrimArgs], @@ -556,6 +590,8 @@ export interface InvokeArgs { cursor?: CursorTelemetryReader; /** Injected the same way: whether this machine can cut out the camera background. */ cameraSegmentation?: () => SegmentationSupport; + /** Injected the same way: the compositor's check of an effect's WGSL. */ + checkShaderEffect?: ShaderEffectChecker; } /** One cheap probe per asset, run before the tools are built so the very first @@ -631,6 +667,7 @@ export async function invokeOpenScreenAgent(args: InvokeArgs): Promise { const doc = migrateProjectDataToAxcutDocument({ @@ -40,7 +40,19 @@ async function writeDocument(relative: string, screenVideoPath: string): Promise const speedRegions = [{ id: "s1", startMs: 0, endMs: 500, speed: 2 }]; return make( relative, - JSON.stringify({ ...doc, legacyEditor: { ...doc.legacyEditor, speedRegions } }), + JSON.stringify({ + ...doc, + legacyEditor: { ...doc.legacyEditor, speedRegions }, + shaderEffects: [ + { + id: "fx1", + name: "Pass-through", + contract: 1, + code: "fn effect(uv: vec2) -> vec4 { return frame(uv); }", + }, + ], + shaderEffectRanges: [{ id: "fxr1", startMs: 0, endMs: 500, effectId: "fx1" }], + }), ); } @@ -158,6 +170,7 @@ describe("runInfoCommand", () => { trimRegions: 1, speedRegions: 0, annotationRegions: 0, + shaderEffectRegions: 0, exportFormat: "mp4", screenVideoExists: true, }); @@ -179,6 +192,7 @@ describe("a project the editor saved", () => { zoomRegions: 1, trimRegions: 0, speedRegions: 1, + shaderEffectRegions: 1, }); }); diff --git a/electron/cli/projectCommands.ts b/electron/cli/projectCommands.ts index 6866eb92a6..394c33f532 100644 --- a/electron/cli/projectCommands.ts +++ b/electron/cli/projectCommands.ts @@ -164,6 +164,7 @@ export async function runInfoCommand( trimRegions: fields.trimRegions, speedRegions: fields.speedRegions, annotationRegions: fields.annotationRegions, + shaderEffectRegions: fields.shaderEffectRegions, }; if (json) { @@ -176,7 +177,7 @@ export async function runInfoCommand( `Webcam: ${summary.webcamVideoPath ?? "(none)"}`, `Cursor: ${summary.cursorCaptureMode ?? "(unknown)"}`, `Export: ${summary.exportFormat ?? "?"} / ${summary.exportQuality ?? "?"} / ${summary.aspectRatio ?? "?"}`, - `Timeline: ${summary.zoomRegions} zooms, ${summary.trimRegions} trims, ${summary.speedRegions} speed regions, ${summary.annotationRegions} annotations`, + `Timeline: ${summary.zoomRegions} zooms, ${summary.trimRegions} trims, ${summary.speedRegions} speed regions, ${summary.annotationRegions} annotations, ${summary.shaderEffectRegions} effects`, ].join("\n") + "\n", ); } @@ -197,6 +198,8 @@ interface ProjectInfoFields { trimRegions: number; speedRegions: number; annotationRegions: number; + /** Shader effect regions (stored fragments, like the other counts). A v2 project has none. */ + shaderEffectRegions: number; } const stringOrNull = (value: unknown): string | null => (typeof value === "string" ? value : null); @@ -217,6 +220,7 @@ function legacyInfoFields(data: PackedProjectData): ProjectInfoFields { trimRegions: count("trimRegions"), speedRegions: count("speedRegions"), annotationRegions: count("annotationRegions"), + shaderEffectRegions: 0, }; } @@ -242,5 +246,6 @@ function documentInfoFields(doc: ReturnType): ProjectI trimRegions: doc.timeline.trimRanges.length, speedRegions: Array.isArray(editor.speedRegions) ? editor.speedRegions.length : 0, annotationRegions: doc.annotations.length, + shaderEffectRegions: doc.shaderEffectRanges?.length ?? 0, }; } diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 15f24992d7..ebee7416f0 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -5062,6 +5062,7 @@ export function registerIpcHandlers( editsAllowed: () => mcpSettings.getSettings().allowEdits, cursor: agentCursorTelemetryReader, cameraSegmentation: () => loudnessService.probeSegmentation(), + checkShaderEffect: (code, params) => loudnessService.checkShaderEffect(code, params), // The editor's own engine, in this process: `openscreen mcp` transcribes without the app. transcribe: (request) => getSttManager().transcribe(request), version: app.getVersion(), @@ -5171,6 +5172,7 @@ export function registerIpcHandlers( runners: mcpRunners, documents: aiEditionDocuments, openVideo: openVideoInHiddenWindow, + renderFrames: (...args) => loudnessService.renderFrames(...args), readCursor: async (videoPath) => (await readCursorTelemetryFile(videoPath)).samples, // Never overriding the switch: setPointerTrack is refused while MCP edits is off. runTool: mcpRunTool, @@ -5237,6 +5239,7 @@ export function registerIpcHandlers( runChat(projectId, sessionId, message, getAiEditionLlmConfig(), document, sink, { cursor: agentCursorTelemetryReader, cameraSegmentation: () => loudnessService.probeSegmentation(), + checkShaderEffect: (code, params) => loudnessService.checkShaderEffect(code, params), }), undoAiEditionToolBatch: (_projectId, _sessionId) => ({ success: false, @@ -5263,6 +5266,8 @@ export function registerIpcHandlers( agentServices: { mcpToolDeps, documents: aiEditionDocuments, + // getFrames without an export: this process's compositor renders them. + renderFrames: loudnessService.renderFrames.bind(loudnessService), // The reader the export runner's auto zoom reads through. readCursorTelemetry: async (videoPath: string) => (await readCursorTelemetryFile(videoPath)).samples, diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index 065aaa4b56..46e75ae58a 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -381,6 +381,14 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { return createSuccessResponse(requestId, { mask: await compositorViewService.segmentFrame(request.payload.rgba), }); + case "checkShaderEffect": + // The inspector shows why an effect does not draw on this machine. + return createSuccessResponse(requestId, { + check: await compositorViewService.checkShaderEffect( + request.payload.code, + request.payload.params, + ), + }); case "setRect": compositorViewService.setRect(request.payload.id, request.payload.rect); return createSuccessResponse(requestId, { ok: true }); diff --git a/electron/mcp/effects-contract.test.ts b/electron/mcp/effects-contract.test.ts new file mode 100644 index 0000000000..b8d54cbc87 --- /dev/null +++ b/electron/mcp/effects-contract.test.ts @@ -0,0 +1,45 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { EFFECTS_CONTRACT } from "./effects-contract"; + +// The compositor's prelude is the authority on what an effect can read; the resource agents read +// must name every field of it, or an agent writes against a contract that is not the real one. +describe("the effect contract resource", () => { + const shaderFx = readFileSync( + path.join(__dirname, "../../crates/compositor/src/shader_fx.rs"), + "utf8", + ); + const prelude = shaderFx.slice( + shaderFx.indexOf("pub const PRELUDE_V1"), + shaderFx.indexOf("@group(0) @binding(0)"), + ); + const fields = [...prelude.matchAll(/^\s+([a-z_]+): /gm)].map((m) => m[1]); + + it("names every field of the compositor's FxContext", () => { + expect(fields).toEqual( + expect.arrayContaining([ + "resolution", + "time", + "region", + "cursor", + "clicks", + "footage", + "camera", + ]), + ); + for (const field of fields) expect(EFFECTS_CONTRACT).toContain(`ctx.${field}`); + }); + + it("names every helper the prelude gives", () => { + for (const helper of [ + "frame(uv)", + "fx_px(n)", + "fx_hash(p)", + "fx_to_footage(uv)", + "fx_from_footage(p)", + ]) { + expect(EFFECTS_CONTRACT).toContain(helper); + } + }); +}); diff --git a/electron/mcp/effects-contract.ts b/electron/mcp/effects-contract.ts new file mode 100644 index 0000000000..832e3d16ed --- /dev/null +++ b/electron/mcp/effects-contract.ts @@ -0,0 +1,148 @@ +/** + * The WGSL contract of agent-written shader effects, served as an MCP resource + * (`openscreen://effects/contract`): too long for a tool description, and versioned with the + * contract. The compositor's prelude (`crates/compositor/src/shader_fx.rs`) is the authority; + * `effects-contract.test.ts` checks that every `ctx` field it declares is described here. + * The design is in docs/shader-effects.md. + */ + +export const EFFECTS_CONTRACT_URI = "openscreen://effects/contract"; + +export const EFFECTS_NOTE = `Before writing a shader effect with addEffect, read the resource ${EFFECTS_CONTRACT_URI}: the full WGSL contract and two examples.`; + +export const EFFECTS_CONTRACT = `# OpenScreen shader effects — WGSL contract 1 + +An effect is WGSL that defines one function, called for every pixel of the output: + + fn effect(uv: vec2) -> vec4 + +\`uv\` runs from (0, 0) top-left to (1, 1) bottom-right of the output. Return the pixel's color. +Declare nothing else the compositor provides: bindings, entry points and the inputs below are +added after your code, so compiler errors point at your lines. + +## Inputs + +| Name | Type | Meaning | +|---|---|---| +| frame(uv) | vec4 | The composed picture at uv (bilinear, edges clamped). | +| ctx.resolution | vec2 | Output size in pixels. | +| ctx.aspect | f32 | Width over height, for round distances in uv. | +| ctx.time | f32 | Programme seconds, continuous through cuts, clips and speed changes. | +| ctx.region | vec4 | x: progress 0 → 1 across the whole region; y: its length in s; z: seconds since it began. | +| ctx.cursor | vec4 | xy: the pointer tip in uv; z: 1 when it is on screen. | +| ctx.clicks[i] | vec4 | The last 4 clicks of the last 2 s, newest first. xy: where it happened, in uv; z: seconds since; w: 1 when the slot holds one. | +| ctx.footage | vec4 | The recording's rect in uv (x, y, w, h), after zoom and layout; under a 3D tilt, its bounding box. | +| ctx.camera | vec4 | The webcam's rect in uv; zero-sized without a camera. | +| ctx.footage_aspect | f32 | The recording's width over height on its own surface, for round distances on it. | +| ctx.footage_corners, ctx.footage_warp | | What the two functions below read: the recording's corners and its warp. Use the functions. | +| fx_to_footage(uv) | vec2 | The point of the recording's surface under uv: (0, 0) its top-left corner, (1, 1) its bottom-right, extended past its edges. Far outside 0..1 where no point of the surface lands. | +| fx_from_footage(p) | vec2 | Back: where point p of the recording's surface lands, in uv. | +| params. | per type | Your declared parameters. | +| fx_px(n) | f32 | n pixels of a 1080p output, as a fraction of the height. | +| fx_hash(p) | f32 | A random number in [0, 1) for a vec2, identical on every GPU. | + +Everything in uv is already placed through zoom, layout, frame and tilt: never recompute where +the screen, the camera or the pointer sit. + +There are two spaces to work in, and the effect picks the one that fits it. Output uv is the +picture as it is framed. The recording's surface is the recording itself, wherever the zoom, the +layout and the 3D camera put it, through the same warp the compositor draws it with: something +computed there moves, scales and tilts with the recording. Map into it with fx_to_footage, compute, +and map back with fx_from_footage before calling frame(). + +## Parameters + +Declare each one as \`{ type, name, label, default }\`; a number also takes \`min\` and \`max\`. + +| type | WGSL | value in the document | +|---|---|---| +| number | f32 | a number between min and max | +| toggle | f32 (0 or 1) | true or false | +| color | vec4 (RGBA, 0..1) | "#rrggbb" or "#rrggbbaa" | +| point | vec2 (uv) | { "x": 0.5, "y": 0.5 } | + +Names are snake_case. Up to 8 is recommended. Declare a parameter only when moving it visibly +changes the effect over its whole range: each one becomes a control in the editor. + +## Rules + +- **Pure.** The result depends only on uv, frame(), ctx and params. No state between frames, no + clock, no frame counter: the preview jumps between frames when seeking, the export renders them + in order, and both must show the same picture. +- **Resolution-free.** The preview renders smaller than the export: measure in uv or fx_px. +- **Random through fx_hash.** \`fract(sin(x) * 43758.5453)\` differs from one GPU to another. +- **Loops are \`for\` with literal bounds**: \`for (var i = 0; i < 16; i++)\`. Any other loop is + refused, and so is more than 4096 iterations per pixel: a loop that never ends stalls the GPU. + This is the one hard rule. +- **Alpha.** For the targets image and both, the returned color replaces the pixel; return + \`frame(uv).a\` as alpha. For the target text, frame() is the text layer in premultiplied alpha, + transparent outside the text, and the effect must return premultiplied alpha. + +## Targets + +- **image** (default): the picture under the annotations and captions; the text stays readable. +- **text**: the annotations and captions alone, drawn over the untouched picture afterwards. +- **both**: everything, captions included. + +Use text or both only when the request asks for them. + +**One effect at a time.** Two effects cannot overlap in time. To combine two looks at the same +moment, write them in one shader: an effect is written for what the user asked, not stacked from +parts. + +## Cost + +The result of addEffect gives the cost in frame() reads per pixel, counted from the code (loops +multiplied by their bound), and its tier. Per 1080p frame, measured on a fast discrete GPU (RTX +4070 Ti) and on an integrated one (Apple M1): + +| Tier | Reads | Per frame, RTX 4070 Ti | Per frame, M1 | Recommended for | +|---|---|---|---|---| +| light | ≤ 8 | ≤ 0.06 ms | ≤ 1.3 ms | anywhere, the whole video included | +| medium | ≤ 32 | ≤ 0.14 ms | ≤ 4.3 ms | up to a minute | +| heavy | ≤ 128 | ≤ 0.54 ms | ≤ 16 ms | up to 10 s: a transition, a key moment | +| very heavy | more | 2 ms at 512 | 49 ms at 512 | a few seconds | + +A 4K frame costs about four times as much. The tiers are recommendations, never refusals. + +## Example: a ripple on the recording, on every click + +Parameters: strength (number, 0–2, default 1), radius (number, 0.05–0.5, default 0.2), +duration (number, 0.3–2, default 0.9). The wave is computed on the recording's surface, so it +follows the zoom and lies flat on the recording under the 3D camera. + + fn ripple(p: vec2, click: vec4) -> vec2 { + if (click.w < 0.5 || click.z > params.duration) { return vec2(0.0); } + let t = click.z / params.duration; + let metric = vec2(ctx.footage_aspect, 1.0); + let d = (p - fx_to_footage(click.xy)) * metric; + let r = length(d); + let x = params.radius * (1.0 - pow(1.0 - t, 3.0)) - r; + let env = smoothstep(-0.008, 0.0, x) * exp(-max(x, 0.0) / 0.07); + let wave = sin(x / 0.03 * 6.2831853) * env * (1.0 - t) * (1.0 - t) * params.strength; + let dir = select(vec2(0.0), d / r, r > 1e-5); + return dir * wave * 0.01 / metric; + } + + fn effect(uv: vec2) -> vec4 { + let p = fx_to_footage(uv); + var offset = vec2(0.0); + for (var i = 0; i < 4; i++) { offset += ripple(p, ctx.clicks[i]); } + if (dot(offset, offset) == 0.0) { return frame(uv); } + return frame(fx_from_footage(p - offset)); + } + +## Example: a vignette that breathes across the region + +Parameters: depth (number, 0–1, default 0.6), tint (color, default "#000000"). + + fn effect(uv: vec2) -> vec4 { + let c = frame(uv); + let v = (uv - 0.5) * vec2(ctx.aspect, 1.0); + let pulse = sin(ctx.region.x * 3.14159265); + let k = smoothstep(0.35, 0.9, length(v)) * params.depth * pulse; + return vec4(mix(c.rgb, params.tint.rgb, k), c.a); + } + +Check the result with getFrames at the moments that matter. +`; diff --git a/electron/mcp/media-tools.test.ts b/electron/mcp/media-tools.test.ts index b86ee8a90b..aaaf3cb3ac 100644 --- a/electron/mcp/media-tools.test.ts +++ b/electron/mcp/media-tools.test.ts @@ -120,6 +120,8 @@ let openVideoFails: boolean; let client: Client; /** The stdio server's media deps: the in-app tests reuse its stand-ins. */ let stdioDeps: MediaToolDeps; +/** The addon's frame renderer, when a test gives it one: none by default, as an older addon. */ +let renderFrames: MediaToolDeps["renderFrames"]; /** The pointer recorded with the take, as the sidecar beside the video. */ async function readSidecar(video: string): Promise { @@ -140,6 +142,7 @@ beforeEach(async () => { runners = new FakeRunners(videoPath); closed = 0; openVideoFails = false; + renderFrames = undefined; const toolDeps: McpToolDeps = { transport: "stdio", @@ -170,6 +173,7 @@ beforeEach(async () => { }; }, readCursor: readSidecar, + renderFrames: async (...args) => (renderFrames ? renderFrames(...args) : null), runTool, now: () => 1_000, }; @@ -774,6 +778,89 @@ describe("getFrames on a project", () => { }); }); +describe("getFrames without an export that shows the project", () => { + let asked: Array<{ clips: number; fps: number; frames: number[]; maxWidth: number }>; + beforeEach(() => { + asked = []; + renderFrames = async (clips, _scene, fps, frames, maxWidth) => { + asked.push({ clips: clips.length, fps, frames, maxWidth }); + return frames.map((index) => ({ + index, + width: 1280, + height: 720, + jpeg: Buffer.from(`render@${index}`), + })); + }; + }); + const images = (result: CallToolResult) => + result.content.flatMap((part) => + part.type === "image" ? [Buffer.from(part.data, "base64").toString()] : [], + ); + + it("renders them from the project, on the export's frame grid, when it has no export", async () => { + const { projectId } = await recordTake(); + // A cut of 2 s early in the take: ruler 6 s is 4 s into what the export would play. + await call("addTrim", { projectId, startSec: 2, endSec: 4 }); + const frames = await call("getFrames", { projectId, atSec: [1, 6] }); + expect(frames.isError).toBeFalsy(); + expect(textOf(frames)).toContain("as its export would show them: it has no export yet"); + expect(asked).toEqual([{ clips: 2, fps: 60, frames: [60, 240], maxWidth: 1280 }]); + expect(images(frames)).toEqual(["render@60", "render@240"]); + }); + + it("reads a current export, and renders again once the project changed after it", async () => { + const { projectId } = await recordTake(); + runners.results.export = { success: true, outputPath: path.join(dir, "demo.mp4") }; + await call("exportVideo", { projectId, path: path.join(dir, "demo.mp4") }); + const fresh = await call("getFrames", { projectId, atSec: [1] }); + expect(fresh.content).toContainEqual({ + type: "text", + text: "1.00s on the ruler, 1.00s into the video", + }); + expect(asked).toEqual([]); + + await call("addTrim", { projectId, startSec: 2, endSec: 4 }); + const stale = await call("getFrames", { projectId, atSec: [6] }); + expect(textOf(stale)).toContain("it changed after its last export"); + expect(images(stale)).toEqual(["render@240"]); + }); + + it("says why it could not render, and asks for an export when the addon cannot", async () => { + const { projectId } = await recordTake(); + renderFrames = async () => { + throw new Error("no GPU"); + }; + const failed = await call("getFrames", { projectId, atSec: [1] }); + expect(failed.isError).toBe(true); + expect(textOf(failed)).toContain("could not be rendered: no GPU"); + renderFrames = async () => null; + expect(textOf(await call("getFrames", { projectId, atSec: [1] }))).toContain("no export yet"); + }); + + it("names a frame the render did not return, and refuses a clip without media", async () => { + const { projectId } = await recordTake(); + renderFrames = async () => []; + const short = await call("getFrames", { projectId, atSec: [1] }); + expect(short.content).toContainEqual({ + type: "text", + text: "No frame here: the rendered video ends before it.", + }); + + const document = await documents.getProject(projectId); + // The schema requires a path on every asset: a clip has no media when its asset is gone. + await documents.saveProject({ + ...document, + timeline: { + ...document.timeline, + clips: document.timeline.clips.map((clip) => ({ ...clip, assetId: "gone" })), + }, + }); + const missing = await call("getFrames", { projectId, atSec: [1] }); + expect(missing.isError).toBe(true); + expect(textOf(missing)).toContain("no media file"); + }); +}); + describe("getFrames", () => { it("returns each frame as an image, labelled with its time", async () => { const result = await call("getFrames", { path: "take.mp4", atSec: [1, 5] }); diff --git a/electron/mcp/media-tools.ts b/electron/mcp/media-tools.ts index eb45333aaf..c112d2bda2 100644 --- a/electron/mcp/media-tools.ts +++ b/electron/mcp/media-tools.ts @@ -34,7 +34,12 @@ import { MP4_FRAME_RATES, VALID_GIF_FRAME_RATES, } from "../../src/lib/exporter/types"; -import type { CursorRecordingData, CursorRecordingSample } from "../../src/native/contracts"; +import type { + CompositorClipInput, + CursorRecordingData, + CursorRecordingSample, +} from "../../src/native/contracts"; +import { buildNativeClipList, buildSceneDescription } from "../../src/native/sceneDescription"; import { runDocumentTool } from "../ai-edition/deep-agent/service"; import type { DocumentService } from "../ai-edition/document-service"; import type { RunnerHandlers } from "../cli/runnerHost"; @@ -105,6 +110,16 @@ export interface MediaToolDeps { "createProject" | "addAsset" | "saveProject" | "filePathFor" | "getProject" >; openVideo(videoPath: string): Promise; + /** Frames of a timeline composed as its export would compose them, without exporting: the + * addon's `renderFrames` (`frames` are indices on the `fps` grid, JPEG at most `maxWidth` + * wide). `null`, or absent, when the addon cannot: getFrames then needs an export. */ + renderFrames?( + clips: CompositorClipInput[], + sceneJson: string, + fps: number, + frames: number[], + maxWidth: number, + ): Promise | null>; /** The pointer samples recorded with a video; empty when there are none. */ readCursor(videoPath: string): Promise; /** The document tools' runner: a fresh project's checkpoint and every edit go through it. */ @@ -115,6 +130,8 @@ export interface MediaToolDeps { export const MAX_FRAMES = 12; // Wide enough to read UI text, small enough that a dozen frames stay cheap. const FRAME_WIDTH = 1280; +/** exportVideo's frame rate when none is asked: the grid getFrames renders on without an export. */ +const EXPORT_FPS = 60; // A seek to the exact end shows no frame in Chromium; stop one frame short of it. const END_MARGIN_SEC = 1 / 60; @@ -496,6 +513,58 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { /** Each project's last export of this session, and when it was made: what getFrames reads. */ const lastExports = new Map(); + /** getFrames when no export shows the project as it is (`docs/shader-effects.md`, E.4): its + * frames composed as its export would compose them, on the export's default frame grid, + * without encoding a video. `null` when the addon cannot render them: the export decides. */ + async function renderProjectFrames( + document: AxcutDocument, + atSec: number[] | null, + count: number, + why: string, + ): Promise { + if (!deps.renderFrames) return null; + // The frame grid below counts every clip, the render only those with media: past a clip + // without one, the two would disagree on which frame a moment is. + const paths = new Map(document.assets.map((asset) => [asset.id, asset.originalPath])); + if (document.timeline.clips.some((clip) => !paths.get(clip.assetId))) { + return text("A clip on this timeline has no media file: its frames cannot be placed.", true); + } + const clips = buildNativeClipList(document); + if (clips.length === 0) return text("This project's timeline has no clip to show.", true); + const rulerEndSec = Math.max(0, ...document.timeline.clips.map((clip) => clip.timelineEndSec)); + const times = frameTimes(rulerEndSec, atSec, count); + // The output frame showing at that moment: the last one that starts at or before it. + const indices = times.map((sec) => + Math.floor(programmeSecOf(document, sec) * EXPORT_FPS + 1e-6), + ); + let frames: Awaited>>; + try { + const scene = JSON.stringify(buildSceneDescription(document)); + frames = await deps.renderFrames(clips, scene, EXPORT_FPS, indices, FRAME_WIDTH); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return text(`The frames could not be rendered: ${message}`, true); + } + if (!frames) return null; + const byIndex = new Map(frames.map((frame) => [frame.index, frame])); + const content: CallToolResult["content"] = [ + { + type: "text", + text: `${times.length} frame(s) rendered from the project as its export would show them: ${why}.`, + }, + ]; + times.forEach((sec, i) => { + content.push({ type: "text", text: `${sec.toFixed(2)}s on the ruler` }); + const frame = byIndex.get(indices[i]); + content.push( + frame + ? { type: "image", data: frame.jpeg.toString("base64"), mimeType: "image/jpeg" } + : { type: "text", text: "No frame here: the rendered video ends before it." }, + ); + }); + return { content }; + } + /** A fresh project's checkpoint, so a client can edit it at once and take it back in one step. */ async function checkpointOf(projectId: string): Promise { // Never fails the call: the project exists either way, and its id is the answer. @@ -767,7 +836,7 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { // The Export dialog's defaults, not whatever an older editor stored. // A GIF carries no quality: the export runner refuses one, and gifSize sizes it. quality: gif ? null : (args.quality ?? "good"), - mp4FrameRate: args.fps ?? 60, + mp4FrameRate: args.fps ?? EXPORT_FPS, gifFrameRate: args.gifFrameRate ?? 15, gifSizePreset: args.gifSize ?? "medium", gifLoop: args.gifLoop ?? true, @@ -828,10 +897,13 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { { description: [ `Look at a video as still images, at most ${MAX_FRAMES} per call. Ask for the moments you need to check, such as each zoom and each callout.`, - "To check an edit, pass projectId: you get the project's last export, and atSec are the ruler seconds the document tools use, mapped through the trims and speed regions to the moment the export plays them. With path instead, you get any video file (a take's videoPath, say) and atSec are that file's own seconds.", + "To check an edit, pass projectId: atSec are the ruler seconds the document tools use, mapped through the trims and speed regions to the moment the export plays them. You get the project's last export, or, when it has none or has changed since, the frames its export would show, rendered from the project without exporting it. With path instead, you get any video file (a take's videoPath, say) and atSec are that file's own seconds.", ].join("\n"), inputSchema: z.object({ - projectId: z.string().optional().describe("Look at this project's last export."), + projectId: z + .string() + .optional() + .describe("Look at this project: its last export, or the frames an export would show."), path: z .string() .optional() @@ -866,6 +938,23 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { let note = ""; if (args.projectId !== undefined) { const exported = lastExports.get(args.projectId); + // A project deleted since still left its export on disk: show it, unflagged. + const current = await deps.documents.getProject(args.projectId).catch(() => null); + const stale = + exported !== undefined && + current !== null && + Date.parse(current.project.updatedAt) > exported.at.getTime(); + // No export shows the project as it is: render its frames as an export would. + if (current && (!exported || stale)) { + const why = exported ? "it changed after its last export" : "it has no export yet"; + const rendered = await renderProjectFrames( + current, + args.atSec ?? null, + args.count ?? 6, + why, + ); + if (rendered) return rendered; + } if (!exported) { return text("This project has no export yet: call exportVideo first.", true); } @@ -875,9 +964,7 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { videoPath = exported.path; toVideoSec = (sec) => programmeSecOf(rendered, sec); rulerEndSec = Math.max(0, ...rendered.timeline.clips.map((clip) => clip.timelineEndSec)); - // A project deleted since still left its export on disk: show it, unflagged. - const current = await deps.documents.getProject(args.projectId).catch(() => null); - if (current && Date.parse(current.project.updatedAt) > exported.at.getTime()) { + if (stale) { note = " The project changed after this export: export it again to see the changes."; } } else { diff --git a/electron/mcp/openscreen-mcp-server.ts b/electron/mcp/openscreen-mcp-server.ts index 7b0f3ec38f..257bd70897 100644 --- a/electron/mcp/openscreen-mcp-server.ts +++ b/electron/mcp/openscreen-mcp-server.ts @@ -54,6 +54,7 @@ import { withTranscript, } from "../../src/lib/ai-edition/document/transcript"; import { type AxcutDocument, documentSchema } from "../../src/lib/ai-edition/schema"; +import type { ShaderEffectChecker } from "../../src/lib/ai-edition/shaderEffects"; import { segmentsFromStt, transcriptFromSegments, @@ -83,6 +84,7 @@ import { type ProjectVersion, } from "../ai-edition/document-service"; import type { SttTranscribeRequest, SttTranscribeResponse } from "../stt/transcriptionContract"; +import { EFFECTS_CONTRACT, EFFECTS_CONTRACT_URI, EFFECTS_NOTE } from "./effects-contract"; import type { MediaTools } from "./media-tools"; import { IMPORT_MEDIA_TOOL, @@ -170,6 +172,8 @@ export interface McpToolDeps extends ImportMediaDeps { cursor?: CursorTelemetryReader; /** Whether this machine can cut out the camera background; absent where nothing can ask. */ cameraSegmentation?: () => SegmentationSupport; + /** The compositor's check of an agent-written effect's WGSL; absent where nothing can ask. */ + checkShaderEffect?: ShaderEffectChecker; /** The main process's on-device Whisper (`SttManager.transcribe`). Absent: no transcribe tool. */ transcribe?(request: SttTranscribeRequest): Promise; version: string; @@ -295,6 +299,7 @@ export function mcpInstructions(options: { ] : []), ...(options.transcribe ? [TRANSCRIBE_NOTE] : []), + EFFECTS_NOTE, "", ].join("\n"); return ( @@ -574,6 +579,7 @@ export function createToolRunner(deps: McpToolDeps): McpToolRunner { cursor: deps.cursor, availableByAssetId, cameraSegmentation: deps.cameraSegmentation, + checkShaderEffect: deps.checkShaderEffect, }); } @@ -925,6 +931,20 @@ export function createOpenScreenMcpServer( ); } media?.register(server); + // The effect contract: too long for addEffect's description, and versioned with the contract. + server.registerResource( + "effects-contract", + EFFECTS_CONTRACT_URI, + { + title: "Shader effect contract", + description: + "The WGSL contract of addEffect: the inputs an effect reads, its parameters, the rules, the cost tiers and two examples.", + mimeType: "text/markdown", + }, + async (uri) => ({ + contents: [{ uri: uri.href, mimeType: "text/markdown", text: EFFECTS_CONTRACT }], + }), + ); return server; } diff --git a/electron/mcp/stdio-server.ts b/electron/mcp/stdio-server.ts index 54dbfdc634..e38e66ece4 100644 --- a/electron/mcp/stdio-server.ts +++ b/electron/mcp/stdio-server.ts @@ -31,6 +31,7 @@ export async function serveMcpOverStdio(options: { readCursorTelemetry: MediaToolDeps["readCursor"]; runners: MediaToolDeps["runners"]; openVideo: MediaToolDeps["openVideo"]; + renderFrames?: MediaToolDeps["renderFrames"]; /** The client went away: nothing is left to serve. */ onClose(): void; }): Promise { @@ -52,6 +53,7 @@ export async function serveMcpOverStdio(options: { runners: options.runners, documents: options.documents, openVideo: options.openVideo, + renderFrames: options.renderFrames, readCursor: options.readCursorTelemetry, runTool, }); diff --git a/electron/native-bridge/services/compositorViewService.test.ts b/electron/native-bridge/services/compositorViewService.test.ts index 01817e826c..40987fe62b 100644 --- a/electron/native-bridge/services/compositorViewService.test.ts +++ b/electron/native-bridge/services/compositorViewService.test.ts @@ -271,6 +271,34 @@ function writeCargoConfig(root: string, body: string): void { ); } +describe("CompositorViewService checks an effect's code off the main thread", () => { + const service = (checkShaderEffect: CompositorViewAddon["checkShaderEffect"]) => + new CompositorViewService({ addon: { checkShaderEffect } as unknown as CompositorViewAddon }); + + it("passes the addon's answer through", async () => { + const check = { tier: "light" as const, frameReads: 1, iterations: 0 }; + const checker = vi.fn(async () => check); + expect( + await service(checker).checkShaderEffect("code", [{ name: "level", type: "number" }]), + ).toEqual(check); + expect(checker).toHaveBeenCalledWith( + "code", + JSON.stringify([{ name: "level", type: "number" }]), + ); + }); + + it("reads an addon that fails, whether it throws or rejects, as not checked", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const rejects = vi.fn(() => Promise.reject(new Error("native"))); + const throws = vi.fn(() => { + throw new Error("native"); + }); + expect(await service(rejects).checkShaderEffect("code", [])).toBeNull(); + expect(await service(throws).checkShaderEffect("code", [])).toBeNull(); + warn.mockRestore(); + }); +}); + describe("ffmpegSharedBinCandidates", () => { let tmpRoot: string; diff --git a/electron/native-bridge/services/compositorViewService.ts b/electron/native-bridge/services/compositorViewService.ts index ebab7e0baf..34f4004d8e 100644 --- a/electron/native-bridge/services/compositorViewService.ts +++ b/electron/native-bridge/services/compositorViewService.ts @@ -3,6 +3,7 @@ import { createRequire } from "node:module"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { app, sharedTexture, type WebFrameMain } from "electron"; +import type { ShaderEffectCheck } from "../../../src/lib/ai-edition/shaderEffects"; import { type CursorKind, readCursorAsArrow, @@ -588,6 +589,29 @@ export class CompositorViewService { return resolveSceneAssetPath(SEGMENTATION_MODEL_ASSET) ? "ready" : "no-model"; } + /** Checks an agent-written effect's WGSL in the compositor (`docs/shader-effects.md`, E.3): the + * compile error, or the cost tier. `null` when this addon cannot answer, which the agent tools + * report as "not checked" rather than as a compile failure. */ + async checkShaderEffect( + code: string, + params: ReadonlyArray<{ name: string; type: string }>, + ): Promise { + const addon = this.ensureAddon(); + if (!addon || typeof addon.checkShaderEffect !== "function") return null; + try { + // `await`, not a bare return: the check is a promise now, and a rejection must land in + // this catch rather than escape it. + return await addon.checkShaderEffect( + code, + JSON.stringify(params.map(({ name, type }) => ({ name, type }))), + ); + } catch (err) { + // A native throw is the addon failing, not the effect: "not checked", as documented. + console.warn("[compositor-view] checkShaderEffect failed:", err); + return null; + } + } + /** Subject mask for one camera frame, no view needed — see `segmentFrame` in the addon. * `null` when the addon (or this version of it) or the model is missing. */ async segmentFrame(rgba: Uint8Array): Promise { @@ -826,6 +850,20 @@ export class CompositorViewService { ); } + /** Frames of a timeline as an export composes them, without the export: see `renderFrames` + * in the addon. `null` when this addon cannot render them. */ + async renderFrames( + clips: ClipInput[], + sceneJson: string, + fps: number, + frames: number[], + maxWidth: number, + ): Promise | null> { + const addon = this.ensureAddon(); + if (!addon || typeof addon.renderFrames !== "function") return null; + return addon.renderFrames(clips, resolveSceneAssetPaths(sceneJson), fps, frames, maxWidth); + } + /** `startGifExport` for MP4, except that an addon built before MP4 cancellation still * exports: the main export must not fail over a missing Cancel. That job runs to the end * and refuses to cancel. */ diff --git a/electron/native/compositor-view/addon.d.ts b/electron/native/compositor-view/addon.d.ts index 5a285bb4eb..1b1ad5e14c 100644 --- a/electron/native/compositor-view/addon.d.ts +++ b/electron/native/compositor-view/addon.d.ts @@ -63,6 +63,18 @@ export interface NativeSharedFramePacket { sourceTimeSec?: number; } +/** What `checkShaderEffect` learns of an agent-written effect (`docs/shader-effects.md`, E.3). */ +export interface ShaderEffectCheck { + /** The error as the agent must read it (line, column, excerpt); absent when accepted. */ + error?: string | null; + /** The indicative cost tier: never a refusal. */ + tier?: "light" | "medium" | "heavy" | "very-heavy" | null; + /** `frame()` reads per pixel, loops multiplied by their bound. */ + frameReads?: number | null; + /** Loop iterations per pixel. */ + iterations?: number | null; +} + export interface ExportStats { frames: number; wallS: number; @@ -143,6 +155,11 @@ export interface CompositorViewAddon { /** What this machine offers, asked without allocating a view — the export dialog * needs the answer before any preview exists. Cached native-side. */ probeBackend(): CompositorBackend; + /** Checks an effect's WGSL: validation, literal loop bounds, the real compile for this machine's + * backend (FXC on Windows, Metal on macOS, SPIR-V on Linux), and the cost tier. `paramsJson` + * is the declared parameters, `[{ "name": "strength", "type": "number" }, …]`. Runs on the + * libuv pool: a Metal compile takes 120 ms and more. Absent from older builds. */ + checkShaderEffect?(code: string, paramsJson: string): Promise; /** Whether the ONNX Runtime library is where the app staged it, and therefore whether a * segmentation mask can be produced at all. Asked of the machine rather than guessed from @@ -231,6 +248,17 @@ export interface CompositorViewAddon { * a control: that addon's MP4 export cannot be cancelled. */ createMp4ExportControl?(): object; cancelMp4Export?(control: object): boolean; + /** Frames of the timeline composed as `exportMulti` composes them, without encoding a video: + * getFrames when the export is missing or stale. Same clips and scene as `exportMulti`, sized + * by the scene's output. `frames` are indices on the `fps` grid, and one past the timeline's + * end renders nothing. JPEG, at most `maxWidth` wide. Absent from older builds. */ + renderFrames?( + clips: ClipInput[], + sceneJson: string | undefined, + fps: number, + frames: number[], + maxWidth: number, + ): Promise>; /** Native GIF export. Identical inputs to `exportMulti` — same clips, same * scene — because it is the same render: both drive `walk_composited_timeline` * in the compositor crate and differ only in the encoder. Cursor, background, diff --git a/skills/openscreen-demo/SKILL.md b/skills/openscreen-demo/SKILL.md index 8fff292bfe..2bfdfb2420 100644 --- a/skills/openscreen-demo/SKILL.md +++ b/skills/openscreen-demo/SKILL.md @@ -86,6 +86,7 @@ Use the document tools, always with the `projectId`. Read the project first with - **Frame the app:** when the take shows more than the app, `setClipRange` with a `cropRegion` crops the clip to it. - **Callouts:** one `addAnnotation` per beat, short, placed clear of the action. Text sits on the output frame and stays put when the picture zooms. Use its look for hierarchy: a large title with `animation: "typewriter"`, step callouts with `animation: "pop"`. An arrow, `type: "figure"`, points at a control the text names. `setAnnotation` moves or restyles one. - **Hide what is private:** an email address, a token or a name that shows on screen gets `addAnnotation` with `type: "blur"` over the span it is visible. Its x, y, width and height are percent of the recording's picture, not of the frame, so it stays on what it hides when a zoom moves the picture. Check it with `getFrames`. +- **An effect no setting offers** (a ripple on each click, a glitch on a cut, a color grade): read the resource `openscreen://effects/contract`, then write it as WGSL with `addEffect`. It reads the clicks and the pointer already placed on the frame. Code that does not compile comes back with the compiler's error at your line: fix it and call again. The result gives a cost tier; keep heavy effects to short spans. Check the effect with `getFrames` at the moments it acts. - **Music or a voiceover:** `importMedia` with the file's absolute path adds it to the project's media, then `addAudio` plays it over a span. `removeMedia` takes back a file the timeline no longer uses. - **Name:** `setProjectInfo` renames the project, if the `title` given to `stopRecording` no longer fits. - **Captions, for a narrated take** (`startRecording` with `microphone: true`): @@ -102,7 +103,7 @@ Keep the rest as it is: the background, padding, shadow and cursor style are cho ## 7. Export and review 1. `exportVideo` with the `projectId` and an absolute path: resolve, say, `demo/demo.mp4` against your working directory first, since the server in the app refuses a relative path. For a pull request video, pass `quality: "medium"` (720p) to keep the file small. For a GIF (a README, say), end the path in `.gif` and pass `gifSize: "small"` (480p). -2. `getFrames` with the `projectId`, at the ruler seconds of your beats. +2. `getFrames` with the `projectId`, at the ruler seconds of your beats. While you edit, you can call it before exporting: without a current export it renders the frames the export would show. 3. Check every frame: - Is each callout readable and clear of the action? - Is each zoom on the thing that changes? diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx index 685f80f14c..97d3e6efae 100644 --- a/src/cli/CliExportRunner.tsx +++ b/src/cli/CliExportRunner.tsx @@ -13,22 +13,18 @@ import { pickExtremeDims, resolveAspectRatioValue, } from "@/lib/ai-edition/document/outputFormat"; -import type { AxcutDocument } from "@/lib/ai-edition/schema"; import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings"; import { appendAutoZoomSuggestions, collectAutoZoomSuggestionsForDocument, } from "@/lib/ai-edition/timeline/apply-auto-zooms"; -import { assetCameraSource } from "@/lib/ai-edition/timeline/camera"; -import { resolveClipSourceEndSec } from "@/lib/ai-edition/timeline/clipDuration"; import type { CliDoneResult, CliExportRequest } from "@/lib/cliContracts"; import { GIF_SIZE_PRESETS, type GifSizePreset } from "@/lib/exporter"; import { calculateMp4ExportSettings } from "@/lib/exporter/mp4ExportSettings"; import { outputFrameCount } from "@/lib/exporter/outputFrameCount"; import { mixVoiceoverIntoVideo } from "@/lib/exporter/voiceoverMix"; import { exportGifNative, exportMultiNative, nativeBridgeClient } from "@/native"; -import type { CompositorClipInput } from "@/native/contracts"; -import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription"; +import { buildNativeClipList, buildSceneDescription } from "@/native/sceneDescription"; import { type ExportProject, loadDocumentProject } from "./exportProject"; import { loadCliProject, probeVideoDimensions } from "./loadCliProject"; @@ -36,31 +32,6 @@ function replaceExtension(filePath: string, newExtension: string): string { return filePath.replace(/\.(openscreen|json)$/i, "") + newExtension; } -/** Mirrors ExportDialog.buildNativeClipList: trim-narrowed visible clips mapped - * onto the native multiclip contract. Kept in lock-step with - * buildSceneDescription so export and scene agree on the clip stream. */ -function buildNativeClipList(axcutDocument: AxcutDocument): CompositorClipInput[] { - const assetById = new Map(axcutDocument.assets.map((asset) => [asset.id, asset])); - return resolveVisibleClips(axcutDocument).flatMap((clip) => { - const asset = assetById.get(clip.assetId); - if (!asset?.originalPath) { - return []; - } - const camera = assetCameraSource(asset); - const sourceEndSec = resolveClipSourceEndSec(clip, asset); - return [ - { - screenPath: asset.originalPath, - webcamPath: camera.path, - sourceStartSec: clip.sourceStartSec, - sourceEndSec, - webcamOffsetSec: camera.offsetSec, - hasAudio: true, - }, - ]; - }); -} - /** Mirrors ExportDialog.gifOutputDims: from the "Source" size, cap height at the preset, keep even. */ function gifOutputDims( preset: GifSizePreset, diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index 08b52f21e5..60a209b6ff 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -20,7 +20,13 @@ import { setDocumentWordText, } from "@/lib/ai-edition/document/transcript"; import { isModalOpen } from "@/lib/ai-edition/modalGuard"; -import type { AxcutAudioTrack, AxcutClip, AxcutDocument } from "@/lib/ai-edition/schema"; +import type { + AxcutAudioTrack, + AxcutClip, + AxcutDocument, + AxcutShaderEffectRange, +} from "@/lib/ai-edition/schema"; +import { withPastedShaderEffect } from "@/lib/ai-edition/shaderEffects"; import { useMcpDocumentHost } from "@/lib/ai-edition/store/mcpDocumentHost"; import { saveWithDeadline, useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { @@ -114,6 +120,9 @@ function NativePlaybackSync({ export const DEFAULT_TIMELINE_HEIGHT_PX = 392; export const MIN_TIMELINE_HEIGHT_PX = 160; export const MAX_TIMELINE_HEIGHT_PX = 560; +/** The Effects lane's height (`.tlLane`: 24px and a 5px margin), added while the project has an + * effect: the lane only shows then (`V4Timeline`). */ +export const EFFECT_LANE_EXPANSION_PX = 29; /** * What one `loadedmetadata` event does, from the front of the write queue. @@ -243,10 +252,13 @@ export function NewEditorShell() { () => computeAudioRowCount(document?.audioTracks ?? []), [document?.audioTracks], ); - const extraAudioHeightPx = Math.max(0, audioRowCount - 1) * AUDIO_ROW_EXPANSION_PX; + const hasEffects = (document?.shaderEffectRanges?.length ?? 0) > 0; + const extraLaneHeightPx = + Math.max(0, audioRowCount - 1) * AUDIO_ROW_EXPANSION_PX + + (hasEffects ? EFFECT_LANE_EXPANSION_PX : 0); const timelineHeightPx = Math.min( MAX_TIMELINE_HEIGHT_PX, - Math.max(MIN_TIMELINE_HEIGHT_PX, timelineBaseHeightPx + extraAudioHeightPx), + Math.max(MIN_TIMELINE_HEIGHT_PX, timelineBaseHeightPx + extraLaneHeightPx), ); const [inspectorOpen, setInspectorOpen] = useState(true); const [facet, setFacet] = useState("effects"); @@ -1058,7 +1070,12 @@ export function NewEditorShell() { // Land it at the playhead, keeping the copied length. const timeMs = Math.round(useProjectStore.getState().currentTimeSec * 1000); const src = snapshot.region as { startMs: number; endMs: number }; - const prefix = snapshot.kind === "annotation" ? "ann" : snapshot.kind; + const prefix = + snapshot.kind === "annotation" + ? "ann" + : snapshot.kind === "shaderEffect" + ? "fxr" + : snapshot.kind; // Audio re-ventilates through its own anchorer, which advances each // fragment's source offset — the generic one would copy the offset into @@ -1112,6 +1129,19 @@ export function NewEditorShell() { }, { history: true }, ); + } else if (snapshot.kind === "shaderEffect") { + // With its definition; refused onto another effect, as the agent's would be. + const next = withPastedShaderEffect( + doc, + snapshot.definition, + anchored as AxcutShaderEffectRange[], + () => createId("fx"), + ); + if (!next) { + toast.error(te("regionClipboard.pasteEffectOverlap")); + return; + } + await saveDocument(next, { history: true }); } else { // speed and cameraFullscreen are both plain spans on legacyEditor. const key = snapshot.kind === "speed" ? "speedRegions" : "cameraFullscreenRegions"; @@ -1173,6 +1203,28 @@ export function NewEditorShell() { return; } + // An effect region goes with its definition (`RegionSnapshot`), and with the span of the + // whole pill: one cut by a clip boundary is several fragments. + if (sel.kind === "shaderEffect") { + const { coalesceRegionsForRuler } = await import("@/lib/ai-edition/timeline/timelineMap"); + const pill = coalesceRegionsForRuler(tl.shaderEffectRanges).find((p) => + p.ids.includes(sel.id), + ); + const definition = pill && tl.shaderEffects.find((e) => e.id === pill.member.effectId); + if (!pill || !definition) return; + copyRegion({ + kind: "shaderEffect", + region: { + ...pill.member, + startMs: Math.round(pill.start * 1000), + endMs: Math.round(pill.end * 1000), + } as unknown as Record, + definition, + }); + setCopiedClipId(null); + toast.success("Region copied"); + return; + } const source = sel.kind === "zoom" ? tl.zoomRegions @@ -1453,9 +1505,9 @@ export function NewEditorShell() { // timeline, since it sits below the handle. const renderedTarget = Math.min( MAX_TIMELINE_HEIGHT_PX, - Math.max(MIN_TIMELINE_HEIGHT_PX, startBase + extraAudioHeightPx - (ev.clientY - startY)), + Math.max(MIN_TIMELINE_HEIGHT_PX, startBase + extraLaneHeightPx - (ev.clientY - startY)), ); - latestBase = Math.max(MIN_TIMELINE_HEIGHT_PX, renderedTarget - extraAudioHeightPx); + latestBase = Math.max(MIN_TIMELINE_HEIGHT_PX, renderedTarget - extraLaneHeightPx); setTimelineBaseHeightPx(latestBase); }; const up = () => { @@ -1466,7 +1518,7 @@ export function NewEditorShell() { window.addEventListener("pointermove", move); window.addEventListener("pointerup", up); }, - [timelineBaseHeightPx, extraAudioHeightPx], + [timelineBaseHeightPx, extraLaneHeightPx], ); const transcriptProps = { diff --git a/src/components/ai-edition/v4/EditorShellV4.module.css b/src/components/ai-edition/v4/EditorShellV4.module.css index 862ac1c68f..77bc77fd4e 100644 --- a/src/components/ai-edition/v4/EditorShellV4.module.css +++ b/src/components/ai-edition/v4/EditorShellV4.module.css @@ -1665,6 +1665,9 @@ .laneCameraFullscreen { --_c: #a855f7; } +.laneShaderEffect { + --_c: #14b8a6; +} /* A zoom's camera moves (#1028): a trail on each side of the pill, fading away from it, BELOW it (a chained neighbour's pill stays on top). Thinner than the pill so it reads as part of it, not as another region. The part a trim cuts is hatched in the trim lane's red. */ diff --git a/src/components/ai-edition/v4/FloatingInspector.test.tsx b/src/components/ai-edition/v4/FloatingInspector.test.tsx index 2bbe078bb1..090f87fbc7 100644 --- a/src/components/ai-edition/v4/FloatingInspector.test.tsx +++ b/src/components/ai-edition/v4/FloatingInspector.test.tsx @@ -37,6 +37,7 @@ vi.mock("../RightPanes", async (importOriginal) => ({ CursorPane: () =>
CursorPane
, LayoutPane: () =>
LayoutPane
, SliderCell: () =>
SliderCell
, + Toggle: () =>
Toggle
, TranscriptPane: () =>
TranscriptPane
, VideoEffectsPane: () =>
VideoEffectsPane
, })); @@ -55,6 +56,13 @@ vi.mock("@/lib/ai-edition/store/useEditorSettings", async (importOriginal) => { }; }); +// What this machine's compositor says of an effect's code, as the inspector asks it. +const checkShaderEffectCode = vi.hoisted(() => vi.fn()); +vi.mock("@/native/compositorViewClient", async (importOriginal) => ({ + ...(await importOriginal()), + checkShaderEffectCode, +})); + vi.mock("../CaptionsPane", () => ({ CaptionsPane: () =>
CaptionsPane
, })); @@ -615,6 +623,79 @@ describe("FloatingInspector", () => { }); }); +describe("FloatingInspector on an agent-written effect", () => { + const effect = { + id: "fx_1", + name: "Dim", + description: "", + contract: 1, + code: "fn effect(uv: vec2) -> vec4 { return frame(uv) * params.level; }", + params: [{ type: "number", name: "level", label: "Level", default: 1, min: 0, max: 2 }], + origin: "agent", + cost: { tier: "light", frameReads: 1 }, + }; + const region = { + id: "fxr_1", + startMs: 0, + endMs: 2000, + effectId: "fx_1", + values: {}, + target: "image", + }; + const renderEffect = () => { + const updateShaderEffect = vi.fn(); + const tl = { + selection: { kind: "shaderEffect", id: "fxr_1" }, + clearSelection: vi.fn(), + selectedAudioTrackId: null, + shaderEffectRanges: [region], + shaderEffects: [effect], + updateShaderEffect, + updateShaderEffectLive: vi.fn(), + commitShaderEffectChange: vi.fn(), + removeRegion: vi.fn(), + } as unknown as React.ComponentProps["tl"]; + render( + ["transcriptProps"] + } + tl={tl} + />, + ); + return { updateShaderEffect }; + }; + + beforeEach(() => { + checkShaderEffectCode.mockReset(); + }); + + it("shows its name, its cost and its code, and why it does not draw on this machine", async () => { + checkShaderEffectCode.mockResolvedValue({ error: "1:12 no definition in scope for `nope`" }); + renderEffect(); + expect(screen.getByRole("heading", { name: "Dim" })).toBeInTheDocument(); + expect(await screen.findByRole("alert")).toHaveTextContent("no definition in scope for `nope`"); + expect(screen.getByText("editor.inspector.effectTierLight")).toBeInTheDocument(); + expect(screen.getByText(effect.code)).toBeInTheDocument(); + expect(checkShaderEffectCode).toHaveBeenCalledWith(effect.code, effect.params); + }); + + it("retargets the effect in one write, and says nothing of code that compiles", async () => { + checkShaderEffectCode.mockResolvedValue({ tier: "light", frameReads: 1, iterations: 0 }); + const { updateShaderEffect } = renderEffect(); + fireEvent.click(screen.getByRole("button", { name: "editor.inspector.effectTargetText" })); + expect(updateShaderEffect).toHaveBeenCalledWith("fxr_1", { target: "text" }); + await waitFor(() => expect(checkShaderEffectCode).toHaveBeenCalled()); + expect(screen.queryByRole("alert")).toBeNull(); + }); +}); + describe("AnnotationSizeField", () => { const commitTyped = (typed: string) => { const onCommit = vi.fn(); diff --git a/src/components/ai-edition/v4/FloatingInspector.tsx b/src/components/ai-edition/v4/FloatingInspector.tsx index 96070498be..b82a0e9cd8 100644 --- a/src/components/ai-edition/v4/FloatingInspector.tsx +++ b/src/components/ai-edition/v4/FloatingInspector.tsx @@ -10,6 +10,7 @@ import { AudioLines, Camera, ChevronRight, + Copy, EyeOff, FileText, ImageIcon, @@ -19,6 +20,7 @@ import { Pencil, Scissors, SlidersHorizontal, + Sparkles, Trash2, Type, Undo2, @@ -54,15 +56,23 @@ import { TEXT_ANIMATION_VALUES, } from "@/lib/ai-edition/annotations/textAnimation"; import { resolveAspectRatioValue } from "@/lib/ai-edition/document/outputFormat"; -import type { AxcutAnnotationRegion, AxcutClip } from "@/lib/ai-edition/schema"; +import type { + AxcutAnnotationRegion, + AxcutClip, + AxcutShaderEffect, + AxcutShaderEffectParam, + AxcutShaderEffectRange, +} from "@/lib/ai-edition/schema"; +import { resolveShaderEffectValue } from "@/lib/ai-edition/shaderEffects"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { rafCoalesce } from "@/lib/ai-edition/store/rafCoalesce"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; -import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; +import type { ShaderEffectPatch, useTimeline } from "@/lib/ai-edition/store/useTimeline"; import { formatSeconds } from "@/lib/ai-edition/timeline/format"; import { coalescedTrimGroups } from "@/lib/ai-edition/timeline/trim-mapping"; import { zoomLevelFor } from "@/lib/ai-edition/timeline/zoom-scale"; import { clampToBound } from "@/lib/projectDefaults"; +import { checkShaderEffectCode } from "@/native/compositorViewClient"; import { annotationFootageRect, zoomScaleLimit } from "@/native/sceneDescription"; import { ColorField } from "../ColorField"; import shell from "../NewEditorShell.module.css"; @@ -73,6 +83,7 @@ import { CursorPane, LayoutPane, SliderCell, + Toggle, TranscriptPane, VideoEffectsPane, } from "../RightPanes"; @@ -1264,6 +1275,23 @@ function SelectionPane({ tl, onClose }: { tl: TimelineApi; onClose: () => void } ); } + if (selection.kind === "shaderEffect") { + const region = tl.shaderEffectRanges.find((r) => r.id === selection.id); + const effect = region ? tl.shaderEffects.find((e) => e.id === region.effectId) : undefined; + if (!region || !effect) return null; + return ( + + ); + } + if (selection.kind === "cameraFullscreen") { const region = tl.cameraFullscreenRegions.find((c) => c.id === selection.id); if (!region) return null; @@ -1320,6 +1348,212 @@ function SelectionPane({ tl, onClose }: { tl: TimelineApi; onClose: () => void } * slab outshouted every setting above it. */ const PANE_BUTTON = `${shell.btn} ${shell.btnSecondary}`; +/** The word an effect's cost tier shows as (docs/shader-effects.md, C.7). */ +const EFFECT_TIER_KEY = { + light: "inspector.effectTierLight", + medium: "inspector.effectTierMedium", + heavy: "inspector.effectTierHeavy", + "very-heavy": "inspector.effectTierVeryHeavy", +} as const; + +const EFFECT_CODE_STYLE: React.CSSProperties = { + margin: 0, + padding: 10, + borderRadius: 8, + background: "var(--surface-1)", + color: "var(--fg-2)", + fontFamily: "ui-monospace, SFMono-Regular, Menlo, Consolas, monospace", + fontSize: 11, + lineHeight: 1.5, + whiteSpace: "pre-wrap", + wordBreak: "break-word", + maxHeight: 240, + overflow: "auto", +}; + +/** + * An agent-written effect (docs/shader-effects.md, F.2): one control per declared parameter, its + * target, its cost tier, its code (read only: an agent edits it), and the compiler's error when + * it does not draw on this machine. A component of its own for the check's state. + */ +function ShaderEffectPane({ + tl, + region, + effect, + onClose, + onDelete, + bodyStyle, +}: { + tl: TimelineApi; + region: AxcutShaderEffectRange; + effect: AxcutShaderEffect; + onClose: () => void; + onDelete: () => void; + bodyStyle: React.CSSProperties; +}) { + const te = useScopedT("editor"); + const tc = useScopedT("common"); + // Asked of this machine's compositor: a region whose code does not compile here draws + // nothing, and only the compiler's error says why. + const [error, setError] = useState(null); + useEffect(() => { + let current = true; + setError(null); + void checkShaderEffectCode(effect.code, effect.params).then((check) => { + if (current) setError(check?.error ?? null); + }); + return () => { + current = false; + }; + }, [effect.code, effect.params]); + // One live write per frame while a slider or a colour moves, one save on release. + const liveRef = useRef(tl.updateShaderEffectLive); + liveRef.current = tl.updateShaderEffectLive; + const live = useMemo( + () => rafCoalesce((id: string, patch: ShaderEffectPatch) => liveRef.current(id, patch)), + [], + ); + const commit = () => { + live.flush(); + void tl.commitShaderEffectChange(); + }; + const setLive = (name: string, value: unknown) => live(region.id, { values: { [name]: value } }); + const paramValue = (param: AxcutShaderEffectParam) => + resolveShaderEffectValue(param, region.values[param.name]); + + return ( +
+ {paneHeader(, effect.name, onClose, tc("actions.close"))} +
+ {error ? ( +
+ + {te("inspector.effectError")} + + 
{error}
+
+ ) : null} + {effect.params.map((param) => { + if (param.type === "number") { + return ( + setLive(param.name, next)} + onCommit={commit} + /> + ); + } + if (param.type === "toggle") { + return ( +
+ {paneRow( + param.label, + + void tl.updateShaderEffect(region.id, { values: { [param.name]: next } }) + } + />, + )} +
+ ); + } + if (param.type === "color") { + return ( +
+ {paneRow( + param.label, + setLive(param.name, next)} + onCommit={commit} + />, + )} +
+ ); + } + const point = paramValue(param) as { x: number; y: number }; + return ( +
+ setLive(param.name, { ...point, x })} + onCommit={commit} + /> + setLive(param.name, { ...point, y })} + onCommit={commit} + /> +
+ ); + })} + void tl.updateShaderEffect(region.id, { target })} + /> + {effect.cost + ? paneRow( + te("inspector.effectCost"), + + {te(EFFECT_TIER_KEY[effect.cost.tier])} + , + ) + : null} +
+ + {te("inspector.effectCode")} + +
+ 
{effect.code}
+ +
+
+ +
+
+ ); +} + function FacetBody({ facet, onCollapse, diff --git a/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx b/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx index 3eba711971..bebe463b1d 100644 --- a/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx +++ b/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx @@ -120,6 +120,8 @@ function renderTimeline( annotationRegions: [annotation], speedRegions: [], cameraFullscreenRegions: [], + shaderEffectRanges: [], + shaderEffects: [], zoomRegions: [], trimRanges: [], hasEditRegions: true, @@ -360,6 +362,27 @@ describe("V4Timeline lane pills", () => { }); }); +describe("V4Timeline effects lane", () => { + // CSS modules hash the class to `_tlLane_` under Vitest. + const lanes = () => + Array.from(document.querySelectorAll("div")).filter((el) => + Array.from(el.classList).some((c) => /^_tlLane_/.test(c)), + ).length; + + it("shows only while the project has an effect, and names the pill after it", () => { + renderTimeline(); + const without = lanes(); + cleanup(); + + renderTimeline(undefined, undefined, undefined, undefined, { + shaderEffects: [{ id: "fx_1", name: "Dim" }], + shaderEffectRanges: [{ id: "fxr_1", startMs: 20_000, endMs: 21_000, effectId: "fx_1" }], + }); + expect(lanes()).toBe(without + 1); + expect(screen.getByTitle("Dim").className).toContain("laneShaderEffect"); + }); +}); + describe("V4Timeline lane pill keyboard", () => { // A pill carries `role="button"` and `tabIndex={0}`, so it is reachable by Tab and // announced as activatable. Selection was pointer-only, which meant a keyboard user @@ -700,6 +723,8 @@ describe("V4Timeline audio lane drag", () => { annotationRegions: [], speedRegions: [], cameraFullscreenRegions: [], + shaderEffectRanges: [], + shaderEffects: [], zoomRegions: [], trimRanges: [], selection: null, diff --git a/src/components/ai-edition/v4/V4Timeline.tsx b/src/components/ai-edition/v4/V4Timeline.tsx index 8ca8158835..bb026fcd8d 100644 --- a/src/components/ai-edition/v4/V4Timeline.tsx +++ b/src/components/ai-edition/v4/V4Timeline.tsx @@ -596,7 +596,7 @@ const AudioLanePill = memo(function AudioLanePill({ interface LanePill { id: string; - kind: "annotation" | "speed" | "trim" | "zoom" | "cameraFullscreen"; + kind: "annotation" | "speed" | "trim" | "zoom" | "cameraFullscreen" | "shaderEffect"; start: number; end: number; label: string; @@ -767,6 +767,15 @@ export function V4Timeline({ sourceIds: p.ids, }), ); + const effectNames = new Map(tl.shaderEffects.map((effect) => [effect.id, effect.name])); + const shaderEffectPills: LanePill[] = coalesceRegionsForRuler(tl.shaderEffectRanges).map((p) => ({ + id: p.ids[0], + kind: "shaderEffect", + start: p.start, + end: p.end, + label: effectNames.get(p.member.effectId) ?? "", + sourceIds: p.ids, + })); const zoomRuns = coalesceRegionsForRuler(tl.zoomRegions); const zoomPills: LanePill[] = zoomRuns.map((p) => ({ id: p.ids[0], @@ -940,6 +949,7 @@ export function V4Timeline({ speed: tl.speedRegions, zoom: tl.zoomRegions, cameraFullscreen: tl.cameraFullscreenRegions, + shaderEffect: tl.shaderEffectRanges, // Nothing to edit on a trim but its span, and a span edit moves the whole pill. trim: [], }; @@ -1015,6 +1025,8 @@ export function V4Timeline({ await tl.updateAnnotationSpan(grabbed, s * 1000, en * 1000); else if (pill.kind === "cameraFullscreen") await tl.updateCameraFullscreenSpan(grabbed, s * 1000, en * 1000); + else if (pill.kind === "shaderEffect") + await tl.updateShaderEffectSpan(grabbed, s * 1000, en * 1000); else { // Trims are stored in source-time per asset but manipulated on the // timeline like every other pill. Ventilate the new span across the @@ -1444,7 +1456,9 @@ export function V4Timeline({ ? styles.laneTrim : kind === "cameraFullscreen" ? styles.laneCameraFullscreen - : styles.laneZoom; + : kind === "shaderEffect" + ? styles.laneShaderEffect + : styles.laneZoom; const pillIcon = (kind: LanePill["kind"]) => kind === "annotation" ? ( @@ -1454,6 +1468,8 @@ export function V4Timeline({ ) : kind === "cameraFullscreen" ? ( + ) : kind === "shaderEffect" ? ( + ) : ( ); @@ -2307,6 +2323,13 @@ export function V4Timeline({ : ts("layout.noWebcam"), )} + {/* Agent-written shader effects: one at a time, so one row. Only when the project + has one: an agent writes them, nothing in the editor adds one, so an empty + row would offer nothing. The shell grows by its height + (`EFFECT_LANE_EXPANSION_PX`). */} + {tl.shaderEffectRanges.length > 0 ? ( +
{renderPills(shaderEffectPills, "")}
+ ) : null} {/* Imported audio tracks (issue #350). Always shown, like every other lane — "Add audio" is a toolbar peer of the region tools now (and has a keyboard shortcut), so an empty lane advertises the shortcut diff --git a/src/components/ai-edition/v4/V4Timeline.waveform.test.tsx b/src/components/ai-edition/v4/V4Timeline.waveform.test.tsx index b84915af2d..37e9c32c62 100644 --- a/src/components/ai-edition/v4/V4Timeline.waveform.test.tsx +++ b/src/components/ai-edition/v4/V4Timeline.waveform.test.tsx @@ -120,6 +120,8 @@ function renderBars(atGainDb: number): string[] { annotationRegions: [], speedRegions: [], cameraFullscreenRegions: [], + shaderEffectRanges: [], + shaderEffects: [], zoomRegions: [], trimRanges: [], selection: null, diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index 766dccb9e3..143f19e57a 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "حدد منطقة لنسخ سماتها", "nothingToPaste": "لم يتم نسخ أي سمات بعد", "pasteAssetMissing": "ملف هذا المسار الصوتي غير موجود في هذا المشروع", + "pasteEffectOverlap": "يوجد تأثير آخر هناك بالفعل: تأثير واحد في كل مرة", "kinds": { "zoom": "تكبير", "speed": "سرعة", @@ -228,7 +229,19 @@ "cropDescription": "أعد تأطير التسجيل — اختر نسبة العرض إلى الارتفاع وقم بالتكبير على المنطقة التي تريد الاحتفاظ بها.", "openCrop": "فتح القص…", "deleteRegion": "حذف المنطقة", - "freehandRendersAsBox": "الشكل الحر يُغطى بمستطيله المحيط في التصدير — تغطية زائدة، وليست ناقصة." + "freehandRendersAsBox": "الشكل الحر يُغطى بمستطيله المحيط في التصدير — تغطية زائدة، وليست ناقصة.", + "effectError": "لا يُترجَم برمجيًا على هذا الحاسوب", + "effectTarget": "يُطبَّق على", + "effectTargetImage": "الصورة", + "effectTargetText": "النصوص", + "effectTargetBoth": "كلاهما", + "effectCost": "التكلفة", + "effectTierLight": "خفيف", + "effectTierMedium": "متوسط", + "effectTierHeavy": "ثقيل", + "effectTierVeryHeavy": "ثقيل جدًا", + "effectCode": "الشيفرة", + "copyEffectCode": "نسخ الشيفرة" }, "chat": { "changeProvider": "تغيير المزوّد", diff --git a/src/i18n/locales/cs/editor.json b/src/i18n/locales/cs/editor.json index d1166a803b..53ba126672 100644 --- a/src/i18n/locales/cs/editor.json +++ b/src/i18n/locales/cs/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "Vyberte oblast pro zkopírování jejích atributů", "nothingToPaste": "Zatím nebyly zkopírovány žádné atributy", "pasteAssetMissing": "Soubor této zvukové stopy není v tomto projektu", + "pasteEffectOverlap": "Tam už běží jiný efekt: vždy jen jeden efekt", "kinds": { "zoom": "Přiblížení", "speed": "Rychlost", @@ -228,7 +229,19 @@ "cropDescription": "Přeformátujte nahrávku: vyberte poměr stran a přibližte oblast, kterou chcete zachovat.", "openCrop": "Otevřít oříznutí…", "deleteRegion": "Smazat oblast", - "freehandRendersAsBox": "Při exportu je volně kreslený tvar pokryt svým ohraničujícím rámečkem: vždy překrytý, nikdy nepokrytý." + "freehandRendersAsBox": "Při exportu je volně kreslený tvar pokryt svým ohraničujícím rámečkem: vždy překrytý, nikdy nepokrytý.", + "effectError": "Na tomto počítači se nezkompiluje", + "effectTarget": "Platí pro", + "effectTargetImage": "Obraz", + "effectTargetText": "Texty", + "effectTargetBoth": "Obojí", + "effectCost": "Náročnost", + "effectTierLight": "Lehký", + "effectTierMedium": "Střední", + "effectTierHeavy": "Těžký", + "effectTierVeryHeavy": "Velmi těžký", + "effectCode": "Kód", + "copyEffectCode": "Kopírovat kód" }, "chat": { "changeProvider": "Změnit poskytovatele", diff --git a/src/i18n/locales/de/editor.json b/src/i18n/locales/de/editor.json index 79619528fc..196e06f513 100644 --- a/src/i18n/locales/de/editor.json +++ b/src/i18n/locales/de/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "Wähle einen Bereich, um seine Attribute zu kopieren", "nothingToPaste": "Noch keine Attribute kopiert", "pasteAssetMissing": "Die Datei dieser Audiospur liegt nicht in diesem Projekt", + "pasteEffectOverlap": "Dort läuft bereits ein anderer Effekt: immer nur ein Effekt", "kinds": { "zoom": "Zoom", "speed": "Tempo", @@ -228,7 +229,19 @@ "cropDescription": "Rahme die Aufnahme neu — wähle ein Seitenverhältnis und zoome auf den Bereich, den du behalten willst.", "openCrop": "Zuschneiden öffnen…", "deleteRegion": "Bereich löschen", - "freehandRendersAsBox": "Im Export wird eine Freihandform von ihrem umschließenden Rechteck abgedeckt — eher zu viel als zu wenig." + "freehandRendersAsBox": "Im Export wird eine Freihandform von ihrem umschließenden Rechteck abgedeckt — eher zu viel als zu wenig.", + "effectError": "Lässt sich auf diesem Computer nicht kompilieren", + "effectTarget": "Gilt für", + "effectTargetImage": "Bild", + "effectTargetText": "Texte", + "effectTargetBoth": "Beides", + "effectCost": "Kosten", + "effectTierLight": "Leicht", + "effectTierMedium": "Mittel", + "effectTierHeavy": "Schwer", + "effectTierVeryHeavy": "Sehr schwer", + "effectCode": "Code", + "copyEffectCode": "Code kopieren" }, "chat": { "changeProvider": "Anbieter wechseln", diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index 45d50e5c00..fb0cb89170 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "Select a region to copy its attributes", "nothingToPaste": "No attributes copied yet", "pasteAssetMissing": "That audio track's file isn't in this project", + "pasteEffectOverlap": "Another effect already plays there: one effect at a time", "kinds": { "zoom": "Zoom", "speed": "Speed", @@ -228,7 +229,19 @@ "cropDescription": "Reframe the recording — pick an aspect ratio and zoom into the region you want to keep.", "openCrop": "Open crop…", "deleteRegion": "Delete region", - "freehandRendersAsBox": "In the export, the blur fills the rectangle around your freehand shape, so it may hide a little more than you drew." + "freehandRendersAsBox": "In the export, the blur fills the rectangle around your freehand shape, so it may hide a little more than you drew.", + "effectError": "Doesn't compile on this computer", + "effectTarget": "Applies to", + "effectTargetImage": "Image", + "effectTargetText": "Text", + "effectTargetBoth": "Both", + "effectCost": "Cost", + "effectTierLight": "Light", + "effectTierMedium": "Medium", + "effectTierHeavy": "Heavy", + "effectTierVeryHeavy": "Very heavy", + "effectCode": "Code", + "copyEffectCode": "Copy code" }, "chat": { "changeProvider": "Change provider", diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index 1ed25722c4..89bf7d390c 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "Selecciona una región para copiar sus atributos", "nothingToPaste": "Aún no se han copiado atributos", "pasteAssetMissing": "El archivo de esa pista de audio no está en este proyecto", + "pasteEffectOverlap": "Ya hay otro efecto ahí: un solo efecto a la vez", "kinds": { "zoom": "Zoom", "speed": "Velocidad", @@ -228,7 +229,19 @@ "cropDescription": "Reencuadra la grabación: elige una relación de aspecto y haz zoom en la región que deseas conservar.", "openCrop": "Abrir recorte…", "deleteRegion": "Eliminar región", - "freehandRendersAsBox": "En la exportación, una forma libre se cubre con su caja contenedora: cubre de más, nunca de menos." + "freehandRendersAsBox": "En la exportación, una forma libre se cubre con su caja contenedora: cubre de más, nunca de menos.", + "effectError": "No compila en este ordenador", + "effectTarget": "Se aplica a", + "effectTargetImage": "Imagen", + "effectTargetText": "Textos", + "effectTargetBoth": "Ambos", + "effectCost": "Coste", + "effectTierLight": "Ligero", + "effectTierMedium": "Medio", + "effectTierHeavy": "Pesado", + "effectTierVeryHeavy": "Muy pesado", + "effectCode": "Código", + "copyEffectCode": "Copiar código" }, "chat": { "changeProvider": "Cambiar proveedor", diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index c65efa693c..92ec22bcb3 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "Sélectionnez une région pour copier ses attributs", "nothingToPaste": "Aucun attribut copié pour l'instant", "pasteAssetMissing": "Le fichier de cette piste audio n'est pas dans ce projet", + "pasteEffectOverlap": "Un autre effet joue déjà à cet endroit : un seul effet à la fois", "kinds": { "zoom": "Zoom", "speed": "Vitesse", @@ -228,7 +229,19 @@ "cropDescription": "Recadrer l'enregistrement — choisissez un format et zoomez sur la région à conserver.", "openCrop": "Ouvrir le recadrage…", "deleteRegion": "Supprimer la zone", - "freehandRendersAsBox": "À l'export, un tracé libre est couvert par sa boîte englobante — on couvre trop, jamais trop peu." + "freehandRendersAsBox": "À l'export, un tracé libre est couvert par sa boîte englobante — on couvre trop, jamais trop peu.", + "effectError": "Ne compile pas sur cet ordinateur", + "effectTarget": "S'applique à", + "effectTargetImage": "Image", + "effectTargetText": "Textes", + "effectTargetBoth": "Les deux", + "effectCost": "Coût", + "effectTierLight": "Léger", + "effectTierMedium": "Moyen", + "effectTierHeavy": "Lourd", + "effectTierVeryHeavy": "Très lourd", + "effectCode": "Code", + "copyEffectCode": "Copier le code" }, "chat": { "changeProvider": "Changer de fournisseur", diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index 547217a876..e9a61562ee 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "Seleziona una regione per copiarne gli attributi", "nothingToPaste": "Nessun attributo copiato", "pasteAssetMissing": "Il file di questa traccia audio non è in questo progetto", + "pasteEffectOverlap": "Lì c'è già un altro effetto: un effetto alla volta", "kinds": { "zoom": "Zoom", "speed": "Velocità", @@ -228,7 +229,19 @@ "cropDescription": "Reinquadra la registrazione — scegli un rapporto d'aspetto e ingrandisci l'area da mantenere.", "openCrop": "Apri ritaglio…", "deleteRegion": "Elimina regione", - "freehandRendersAsBox": "Nell'esportazione, una forma libera è coperta dal suo rettangolo di delimitazione: copre di più, mai di meno." + "freehandRendersAsBox": "Nell'esportazione, una forma libera è coperta dal suo rettangolo di delimitazione: copre di più, mai di meno.", + "effectError": "Non compila su questo computer", + "effectTarget": "Si applica a", + "effectTargetImage": "Immagine", + "effectTargetText": "Testi", + "effectTargetBoth": "Entrambi", + "effectCost": "Costo", + "effectTierLight": "Leggero", + "effectTierMedium": "Medio", + "effectTierHeavy": "Pesante", + "effectTierVeryHeavy": "Molto pesante", + "effectCode": "Codice", + "copyEffectCode": "Copia codice" }, "chat": { "changeProvider": "Cambia provider", diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index c2fd846ffe..10d8f7aab5 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "属性をコピーする領域を選択してください", "nothingToPaste": "コピーされた属性がありません", "pasteAssetMissing": "このオーディオトラックのファイルはこのプロジェクトにありません", + "pasteEffectOverlap": "そこには別のエフェクトがあります。エフェクトは一度に1つだけです", "kinds": { "zoom": "ズーム", "speed": "速度", @@ -228,7 +229,19 @@ "cropDescription": "録画をリフレーム — アスペクト比を選び、保持したい範囲にズームします。", "openCrop": "切り抜きを開く…", "deleteRegion": "区間を削除", - "freehandRendersAsBox": "書き出しでは、フリーハンドの形は外接する矩形で覆われます。覆いすぎることはあっても、足りないことはありません。" + "freehandRendersAsBox": "書き出しでは、フリーハンドの形は外接する矩形で覆われます。覆いすぎることはあっても、足りないことはありません。", + "effectError": "このコンピューターではコンパイルできません", + "effectTarget": "適用先", + "effectTargetImage": "画像", + "effectTargetText": "テキスト", + "effectTargetBoth": "両方", + "effectCost": "負荷", + "effectTierLight": "軽い", + "effectTierMedium": "中", + "effectTierHeavy": "重い", + "effectTierVeryHeavy": "非常に重い", + "effectCode": "コード", + "copyEffectCode": "コードをコピー" }, "chat": { "changeProvider": "プロバイダーを変更", diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index efe680db06..fa1dbc148b 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "속성을 복사할 영역을 선택하세요", "nothingToPaste": "복사된 속성이 없습니다", "pasteAssetMissing": "이 오디오 트랙의 파일이 이 프로젝트에 없습니다", + "pasteEffectOverlap": "그 위치에 이미 다른 효과가 있습니다. 효과는 한 번에 하나만 사용할 수 있습니다", "kinds": { "zoom": "줌", "speed": "속도", @@ -228,7 +229,19 @@ "cropDescription": "녹화 화면을 다시 프레이밍 — 화면 비율을 선택하고 유지할 영역으로 확대합니다.", "openCrop": "자르기 열기…", "deleteRegion": "구간 삭제", - "freehandRendersAsBox": "내보내기에서 자유형 도형은 외곽 사각형으로 덮입니다. 더 많이 덮되, 부족하게 덮지는 않습니다." + "freehandRendersAsBox": "내보내기에서 자유형 도형은 외곽 사각형으로 덮입니다. 더 많이 덮되, 부족하게 덮지는 않습니다.", + "effectError": "이 컴퓨터에서 컴파일되지 않습니다", + "effectTarget": "적용 대상", + "effectTargetImage": "이미지", + "effectTargetText": "텍스트", + "effectTargetBoth": "둘 다", + "effectCost": "부하", + "effectTierLight": "가벼움", + "effectTierMedium": "보통", + "effectTierHeavy": "무거움", + "effectTierVeryHeavy": "매우 무거움", + "effectCode": "코드", + "copyEffectCode": "코드 복사" }, "chat": { "changeProvider": "제공업체 변경", diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index c0e7102932..8dcaff24a8 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "Selecione uma região para copiar seus atributos", "nothingToPaste": "Nenhum atributo copiado ainda", "pasteAssetMissing": "O arquivo dessa faixa de áudio não está neste projeto", + "pasteEffectOverlap": "Já há outro efeito ali: um efeito por vez", "kinds": { "zoom": "Zoom", "speed": "Velocidade", @@ -228,7 +229,19 @@ "cropDescription": "Reenquadre a gravação — escolha uma proporção e dê zoom na região que deseja manter.", "openCrop": "Abrir corte…", "deleteRegion": "Excluir região", - "freehandRendersAsBox": "Na exportação, uma forma livre é coberta pela sua caixa delimitadora — cobre demais, nunca de menos." + "freehandRendersAsBox": "Na exportação, uma forma livre é coberta pela sua caixa delimitadora — cobre demais, nunca de menos.", + "effectError": "Não compila neste computador", + "effectTarget": "Aplica-se a", + "effectTargetImage": "Imagem", + "effectTargetText": "Textos", + "effectTargetBoth": "Ambos", + "effectCost": "Custo", + "effectTierLight": "Leve", + "effectTierMedium": "Médio", + "effectTierHeavy": "Pesado", + "effectTierVeryHeavy": "Muito pesado", + "effectCode": "Código", + "copyEffectCode": "Copiar código" }, "chat": { "changeProvider": "Trocar provedor", diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index 16fa07d248..827488c319 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "Выберите регион, чтобы скопировать его атрибуты", "nothingToPaste": "Атрибуты ещё не скопированы", "pasteAssetMissing": "Файл этой аудиодорожки отсутствует в проекте", + "pasteEffectOverlap": "Там уже есть другой эффект: только один эффект одновременно", "kinds": { "zoom": "Масштаб", "speed": "Скорость", @@ -228,7 +229,19 @@ "cropDescription": "Перекадрируйте запись — выберите соотношение сторон и увеличьте область, которую нужно сохранить.", "openCrop": "Открыть обрезку…", "deleteRegion": "Удалить участок", - "freehandRendersAsBox": "При экспорте произвольная форма закрывается своим ограничивающим прямоугольником — с запасом, а не с недостатком." + "freehandRendersAsBox": "При экспорте произвольная форма закрывается своим ограничивающим прямоугольником — с запасом, а не с недостатком.", + "effectError": "Не компилируется на этом компьютере", + "effectTarget": "Применяется к", + "effectTargetImage": "Изображение", + "effectTargetText": "Тексты", + "effectTargetBoth": "Оба", + "effectCost": "Нагрузка", + "effectTierLight": "Лёгкий", + "effectTierMedium": "Средний", + "effectTierHeavy": "Тяжёлый", + "effectTierVeryHeavy": "Очень тяжёлый", + "effectCode": "Код", + "copyEffectCode": "Копировать код" }, "chat": { "changeProvider": "Сменить провайдера", diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index f31f975151..78efe59a15 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "Özniteliklerini kopyalamak için bir bölge seçin", "nothingToPaste": "Henüz öznitelik kopyalanmadı", "pasteAssetMissing": "Bu ses parçasının dosyası bu projede yok", + "pasteEffectOverlap": "Orada zaten başka bir efekt var: aynı anda tek efekt", "kinds": { "zoom": "Yakınlaştırma", "speed": "Hız", @@ -228,7 +229,19 @@ "cropDescription": "Kaydı yeniden çerçeveleyin — bir en boy oranı seçin ve saklamak istediğiniz bölgeye yakınlaşın.", "openCrop": "Kırpmayı aç…", "deleteRegion": "Bölgeyi sil", - "freehandRendersAsBox": "Dışa aktarmada serbest çizim, sınırlayıcı dikdörtgeniyle kapatılır — eksik değil, fazla kapatır." + "freehandRendersAsBox": "Dışa aktarmada serbest çizim, sınırlayıcı dikdörtgeniyle kapatılır — eksik değil, fazla kapatır.", + "effectError": "Bu bilgisayarda derlenmiyor", + "effectTarget": "Uygulandığı yer", + "effectTargetImage": "Görüntü", + "effectTargetText": "Metinler", + "effectTargetBoth": "İkisi de", + "effectCost": "Maliyet", + "effectTierLight": "Hafif", + "effectTierMedium": "Orta", + "effectTierHeavy": "Ağır", + "effectTierVeryHeavy": "Çok ağır", + "effectCode": "Kod", + "copyEffectCode": "Kodu kopyala" }, "chat": { "changeProvider": "Sağlayıcıyı değiştir", diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index 133380596b..5341e70214 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "Chọn một vùng để sao chép thuộc tính của nó", "nothingToPaste": "Chưa sao chép thuộc tính nào", "pasteAssetMissing": "Tệp của bản âm thanh này không có trong dự án", + "pasteEffectOverlap": "Ở đó đã có một hiệu ứng khác: mỗi lúc chỉ một hiệu ứng", "kinds": { "zoom": "Thu phóng", "speed": "Tốc độ", @@ -228,7 +229,19 @@ "cropDescription": "Đóng khung lại bản ghi — chọn tỷ lệ khung hình và phóng to vào vùng bạn muốn giữ lại.", "openCrop": "Mở cắt…", "deleteRegion": "Xóa vùng", - "freehandRendersAsBox": "Khi xuất, hình vẽ tay được che bằng khung bao của nó — che dư, không bao giờ che thiếu." + "freehandRendersAsBox": "Khi xuất, hình vẽ tay được che bằng khung bao của nó — che dư, không bao giờ che thiếu.", + "effectError": "Không biên dịch được trên máy này", + "effectTarget": "Áp dụng cho", + "effectTargetImage": "Hình ảnh", + "effectTargetText": "Văn bản", + "effectTargetBoth": "Cả hai", + "effectCost": "Chi phí", + "effectTierLight": "Nhẹ", + "effectTierMedium": "Trung bình", + "effectTierHeavy": "Nặng", + "effectTierVeryHeavy": "Rất nặng", + "effectCode": "Mã", + "copyEffectCode": "Sao chép mã" }, "chat": { "changeProvider": "Đổi nhà cung cấp", diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index a5799cda7b..a50819a0f4 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "选择一个区域以复制其属性", "nothingToPaste": "尚未复制任何属性", "pasteAssetMissing": "该音频轨道的文件不在此项目中", + "pasteEffectOverlap": "该处已有另一个效果：同一时间只能有一个效果", "kinds": { "zoom": "缩放", "speed": "速度", @@ -228,7 +229,19 @@ "cropDescription": "重新构图录制内容 — 选择宽高比并缩放到要保留的区域。", "openCrop": "打开裁剪…", "deleteRegion": "删除区间", - "freehandRendersAsBox": "导出时，手绘形状会以其外接矩形遮挡——宁可多遮，绝不少遮。" + "freehandRendersAsBox": "导出时，手绘形状会以其外接矩形遮挡——宁可多遮，绝不少遮。", + "effectError": "无法在这台电脑上编译", + "effectTarget": "作用于", + "effectTargetImage": "画面", + "effectTargetText": "文字", + "effectTargetBoth": "两者", + "effectCost": "开销", + "effectTierLight": "轻", + "effectTierMedium": "中", + "effectTierHeavy": "重", + "effectTierVeryHeavy": "很重", + "effectCode": "代码", + "copyEffectCode": "复制代码" }, "chat": { "changeProvider": "更换提供方", diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index 0528b1b319..f0a7c97036 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -80,6 +80,7 @@ "nothingToCopy": "選擇一個區域以複製其屬性", "nothingToPaste": "尚未複製任何屬性", "pasteAssetMissing": "該音訊軌道的檔案不在此專案中", + "pasteEffectOverlap": "該處已有另一個效果：同一時間只能有一個效果", "kinds": { "zoom": "縮放", "speed": "速度", @@ -228,7 +229,19 @@ "cropDescription": "重新構圖錄製內容 — 選擇長寬比並縮放至想保留的區域。", "openCrop": "開啟裁切…", "deleteRegion": "刪除區間", - "freehandRendersAsBox": "匯出時，手繪形狀會以其外接矩形遮蔽——寧可多遮，絕不少遮。" + "freehandRendersAsBox": "匯出時，手繪形狀會以其外接矩形遮蔽——寧可多遮，絕不少遮。", + "effectError": "無法在這台電腦上編譯", + "effectTarget": "作用於", + "effectTargetImage": "畫面", + "effectTargetText": "文字", + "effectTargetBoth": "兩者", + "effectCost": "開銷", + "effectTierLight": "輕", + "effectTierMedium": "中", + "effectTierHeavy": "重", + "effectTierVeryHeavy": "很重", + "effectCode": "程式碼", + "copyEffectCode": "複製程式碼" }, "chat": { "changeProvider": "更換提供者", diff --git a/src/lib/ai-edition/document/timeline.test.ts b/src/lib/ai-edition/document/timeline.test.ts index 7f203f32a0..765e07202c 100644 --- a/src/lib/ai-edition/document/timeline.test.ts +++ b/src/lib/ai-edition/document/timeline.test.ts @@ -1962,6 +1962,50 @@ describe("removeClip — delete a clip, close the gap, drop its pills", () => { expect(next.zoomRanges[0]).toMatchObject({ startMs: 2000, endMs: 4000 }); }); + // A shader effect is a region like a zoom: it moves with its clip and goes with it. Its + // definition goes with its last region, so no dead code stays in the document. + it("moves a shader effect with its clip, and drops it and its definition with the clip", () => { + const effect = { + id: "fx", + name: "Pass-through", + description: "", + contract: 1, + code: "fn effect(uv: vec2) -> vec4 { return frame(uv); }", + params: [], + origin: "agent" as const, + }; + const range = (id: string, clipId: string, startMs: number) => ({ + id, + clipId, + sourceStartSec: 2, + sourceEndSec: 4, + startMs, + endMs: startMs + 2000, + effectId: "fx", + values: {}, + target: "image" as const, + lane: 0, + }); + const before = { ...doc(), shaderEffects: [effect] }; + const onB = removeClip( + { + ...before, + shaderEffectRanges: [range("fx_a", "clip_a", 2000), range("fx_b", "clip_b", 12000)], + }, + "clip_a", + ); + expect(onB.shaderEffectRanges).toEqual([ + expect.objectContaining({ id: "fx_b", startMs: 2000 }), + ]); + expect(onB.shaderEffects).toEqual([effect]); + const gone = removeClip( + { ...before, shaderEffectRanges: [range("fx_a", "clip_a", 2000)] }, + "clip_a", + ); + expect(gone.shaderEffectRanges).toEqual([]); + expect(gone.shaderEffects).toEqual([]); + }); + it("preserves a bare clipId that is not a complete source anchor", () => { const before = doc(); before.zoomRanges.push( diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.ts index 4e1a751419..2b8366ba31 100644 --- a/src/lib/ai-edition/document/timeline.ts +++ b/src/lib/ai-edition/document/timeline.ts @@ -21,6 +21,7 @@ import type { */ export type PlaybackSegment = AxcutClip; +import { pruneUnusedShaderEffects } from "../shaderEffects"; import { type Interval, subtractInterval } from "../timeline/intervals"; import { keptRawSpans } from "../timeline/programme-time"; import { @@ -43,7 +44,14 @@ import { createId } from "./ids"; * exist" has exactly one definition. `trim` is a source-time cut; the rest are pill-merged * effects (zoom / speed / annotation / camera-fullscreen). Clips are removed via * {@link removeClip}, not here — deleting a clip reflows the whole timeline. */ -export type RegionKind = "zoom" | "trim" | "annotation" | "speed" | "cameraFullscreen" | "audio"; +export type RegionKind = + | "zoom" + | "trim" + | "annotation" + | "speed" + | "cameraFullscreen" + | "audio" + | "shaderEffect"; /** Length a clip is given before its media has been probed. Lives here, in the pure * document layer, because that layer decides which clips are still waiting for a real @@ -378,12 +386,17 @@ export function mapAllRegionCollections( ? (legacy?.cameraFullscreenRegions as StoredRegion[]) : undefined; - return { + // A clip edit can take a shader effect's last fragment with it: its definition goes too. + return pruneUnusedShaderEffects({ ...document, zoomRanges: fn( document.zoomRanges as unknown as StoredRegion[], "zoom", ) as unknown as AxcutDocument["zoomRanges"], + shaderEffectRanges: fn( + (document.shaderEffectRanges ?? []) as unknown as StoredRegion[], + "fxr", + ) as unknown as AxcutDocument["shaderEffectRanges"], annotations: fn( document.annotations as unknown as StoredRegion[], "ann", @@ -420,7 +433,7 @@ export function mapAllRegionCollections( : {}), } : document.legacyEditor, - }; + }); } /** A clamped fragment this short has no surviving content — treat it as fully @@ -734,6 +747,7 @@ function anchoredRegionsOf(document: AxcutDocument): Array<{ id: string; clipId: document.annotations as unknown as StoredRegion[], (legacy?.speedRegions as StoredRegion[] | undefined) ?? [], (legacy?.cameraFullscreenRegions as StoredRegion[] | undefined) ?? [], + (document.shaderEffectRanges ?? []) as unknown as StoredRegion[], ]; return collections .flat() @@ -1197,6 +1211,12 @@ export function removeRegion(document: AxcutDocument, kind: RegionKind, id: stri ); return { ...document, legacyEditor: { ...legacy, cameraFullscreenRegions: prev } }; } + case "shaderEffect": + // The definition goes with its last region: no dead code stays in the document. + return pruneUnusedShaderEffects({ + ...document, + shaderEffectRanges: dropPillById(document.shaderEffectRanges ?? [], id), + }); default: { // ponytail: exhaustive — TS errors here if a new RegionKind is added. const exhaustive: never = kind; @@ -1241,6 +1261,10 @@ const EDIT_REGIONS: Record< count: (d) => legacyRegionCount(d, "cameraFullscreenRegions"), clear: (d) => withLegacyRegionsCleared(d, "cameraFullscreenRegions"), }, + shaderEffect: { + count: (d) => (d.shaderEffectRanges ?? []).length, + clear: (d) => ({ ...d, shaderEffectRanges: [], shaderEffects: [] }), + }, }; /** How many edit regions the document holds, stored rows and not pills. Pure. */ diff --git a/src/lib/ai-edition/schema/index.test.ts b/src/lib/ai-edition/schema/index.test.ts index 38aabd8705..5178c5490a 100644 --- a/src/lib/ai-edition/schema/index.test.ts +++ b/src/lib/ai-edition/schema/index.test.ts @@ -13,6 +13,8 @@ import { legacyEditorSchema, parseDocumentFile, rangeSchema, + shaderEffectRangeSchema, + shaderEffectSchema, timelineSchema, trimRangeSchema, upgradeV7DocumentToV8, @@ -1245,3 +1247,44 @@ describe("follow-cursor zooms read as orbits under auto focus", () => { expect(migrateRawDocumentToCurrent(manualOrbit)).toBe(manualOrbit); }); }); + +describe("shader effects (docs/shader-effects.md)", () => { + const effect = { + id: "fx", + name: "Dim", + contract: 1, + code: "fn effect(uv: vec2) -> vec4 { return frame(uv); }", + params: [{ type: "number", name: "level", default: 1, min: 0, max: 2 }], + }; + + it("fills the defaults of a definition and of a region", () => { + expect(shaderEffectSchema.parse(effect)).toMatchObject({ + description: "", + origin: "agent", + params: [{ name: "level", label: "" }], + }); + expect( + shaderEffectRangeSchema.parse({ id: "r", startMs: 0, endMs: 1000, effectId: "fx" }), + ).toMatchObject({ values: {}, target: "image" }); + }); + + it("refuses a parameter declared twice, or a default out of its bounds", () => { + const twice = { ...effect, params: [effect.params[0], effect.params[0]] }; + expect(shaderEffectSchema.safeParse(twice).success).toBe(false); + const outside = { ...effect, params: [{ ...effect.params[0], default: 3 }] }; + expect(shaderEffectSchema.safeParse(outside).success).toBe(false); + }); + + // A newer build may write a contract this one does not know: the document must still open, + // and the compositor leaves those regions undrawn. + it("loads a contract version it does not know", () => { + expect(shaderEffectSchema.safeParse({ ...effect, contract: 2 }).success).toBe(true); + }); + + it("keeps a document without effects free of both keys", () => { + const doc = createEmptyDocument({ title: "t", projectId: "p" }); + const parsed = documentSchema.parse(doc); + expect("shaderEffects" in parsed).toBe(false); + expect("shaderEffectRanges" in parsed).toBe(false); + }); +}); diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index 05edecab07..c3422b5978 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -528,6 +528,111 @@ export const zoomRegionSchema = endGteStart( "startMs", ); +// Agent-written shader effects (docs/shader-effects.md, B). A definition carries the WGSL and its +// declared parameters; a range poses it on the timeline, clip-anchored like a zoom, with its own +// parameter values. Additive: a document written before these existed loads unchanged. +const shaderEffectParamName = z.string().regex(/^[a-z][a-z0-9_]{0,31}$/); +const shaderEffectColor = z.string().regex(/^#(?:[0-9a-fA-F]{6}|[0-9a-fA-F]{8})$/); +const shaderEffectPoint = z.object({ x: z.number().min(0).max(1), y: z.number().min(0).max(1) }); +// Written by the agent, in the user's language, shown as-is: content, not interface copy. +const shaderEffectLabel = z.string().max(40).default(""); + +export const shaderEffectParamSchema = z.discriminatedUnion("type", [ + z.object({ + type: z.literal("number"), + name: shaderEffectParamName, + label: shaderEffectLabel, + default: z.number(), + min: z.number(), + max: z.number(), + }), + z.object({ + type: z.literal("toggle"), + name: shaderEffectParamName, + label: shaderEffectLabel, + default: z.boolean(), + }), + z.object({ + type: z.literal("color"), + name: shaderEffectParamName, + label: shaderEffectLabel, + default: shaderEffectColor, + }), + z.object({ + type: z.literal("point"), + name: shaderEffectParamName, + label: shaderEffectLabel, + default: shaderEffectPoint, + }), +]); + +export const shaderEffectValueSchema = z.union([ + z.number(), + z.boolean(), + shaderEffectColor, + shaderEffectPoint, +]); + +export const shaderEffectSchema = z + .object({ + id: z.string().min(1), + // What the pill shows. + name: z.string().min(1).max(60), + // For the next agent: what the effect does, in one sentence. + description: z.string().max(500).default(""), + // The WGSL contract version. Not a literal: a version this build does not know (written by + // a newer one) must still load; the compositor leaves its regions undrawn. + contract: z.number().int().positive(), + code: z.string().min(1).max(65_536), + // The `max` here is a technical ceiling, not a budget: see the cost tiers. + params: z.array(shaderEffectParamSchema).max(32).default([]), + origin: z.enum(["agent", "user"]).default("agent"), + // The compositor's cost estimate, cached when the code was last checked. Derived, never an + // input: `frameReads` is `frame()` reads per pixel, `tier` its indicative tier. + cost: z + .object({ + tier: z.enum(["light", "medium", "heavy", "very-heavy"]), + frameReads: z.number().nonnegative(), + }) + .optional(), + }) + .superRefine((effect, ctx) => { + const names = new Set(); + for (const [index, param] of effect.params.entries()) { + if (names.has(param.name)) { + ctx.addIssue({ + code: "custom", + message: `parameter "${param.name}" is declared twice`, + path: ["params", index], + }); + } + names.add(param.name); + if (param.type === "number" && !(param.min <= param.default && param.default <= param.max)) { + ctx.addIssue({ + code: "custom", + message: `parameter "${param.name}": min ≤ default ≤ max`, + path: ["params", index], + }); + } + } + }); + +export const shaderEffectRangeSchema = endGteStart( + z.object({ + id: z.string().min(1), + startMs: z.number().nonnegative(), + endMs: z.number().nonnegative(), + ...clipAnchorShape, + effectId: z.string().min(1), + // Only the values that differ from the definition's defaults. + values: z.record(z.string(), shaderEffectValueSchema).default({}), + // What the effect transforms: the picture under the texts, the texts alone, or both. + target: z.enum(["image", "text", "both"]).default("image"), + }), + "endMs", + "startMs", +); + // External audio import (issue #350) — voiceover / BGM / SFX layered over the // programme. Unlike zoom/speed/annotation/trim, an audio track is NOT // clip-anchored: it floats over the whole timeline, addressed in RAW/document @@ -630,6 +735,11 @@ const documentSchemaShape = z.object({ }), annotations: z.array(annotationRegionSchema).default([]), zoomRanges: z.array(zoomRegionSchema).default([]), + // Agent-written shader effects (docs/shader-effects.md). Optional rather than defaulted, like + // `vocabulary`: a document without effects carries no trace of them, and every reader treats + // an absent list as empty. An older build drops both keys when it saves. + shaderEffects: z.array(shaderEffectSchema).optional(), + shaderEffectRanges: z.array(shaderEffectRangeSchema).optional(), // Imported audio tracks (issue #350). Defaulted so every document written // before this loads unchanged; an older build simply strips the key on save. audioTracks: z.array(audioTrackSchema).default([]), @@ -1265,6 +1375,10 @@ export type AxcutAnnotationRegion = z.infer; export type AxcutZoomRegion = z.infer; export type AxcutCameraTrack = z.infer; export type AxcutAudioTrack = z.infer; +export type AxcutShaderEffect = z.infer; +export type AxcutShaderEffectParam = z.infer; +export type AxcutShaderEffectRange = z.infer; +export type AxcutShaderEffectValue = z.infer; export type AxcutLegacyEditor = z.infer; export type AxcutDocument = z.infer; export type AxcutDocumentInput = z.input; diff --git a/src/lib/ai-edition/shaderEffects.test.ts b/src/lib/ai-edition/shaderEffects.test.ts new file mode 100644 index 0000000000..3c61961afe --- /dev/null +++ b/src/lib/ai-edition/shaderEffects.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest"; +import type { AxcutDocument, AxcutShaderEffect, AxcutShaderEffectRange } from "./schema"; +import { createEmptyDocument } from "./schema"; +import { withPastedShaderEffect } from "./shaderEffects"; + +const definition: AxcutShaderEffect = { + id: "fx_1", + name: "Dim", + description: "", + contract: 1, + code: "fn effect(uv: vec2) -> vec4 { return frame(uv); }", + params: [], + origin: "agent", +}; +const fragment = (id: string, startMs: number, endMs: number): AxcutShaderEffectRange => ({ + id, + startMs, + endMs, + effectId: "fx_1", + values: {}, + target: "image", +}); +const newId = () => "fx_new"; + +describe("withPastedShaderEffect", () => { + const empty: AxcutDocument = createEmptyDocument({ projectId: "p", title: "t" }); + + it("recreates the definition in a project that does not have it", () => { + const next = withPastedShaderEffect(empty, definition, [fragment("fxr_1", 0, 1000)], newId); + expect(next?.shaderEffects).toEqual([definition]); + expect(next?.shaderEffectRanges).toHaveLength(1); + }); + + it("reuses the definition the project already has, and refuses a paste onto an effect", () => { + const doc = withPastedShaderEffect( + empty, + definition, + [fragment("fxr_1", 0, 1000)], + newId, + ) as AxcutDocument; + const again = withPastedShaderEffect(doc, definition, [fragment("fxr_2", 2000, 3000)], newId); + expect(again?.shaderEffects).toHaveLength(1); + expect(again?.shaderEffectRanges).toHaveLength(2); + expect( + withPastedShaderEffect(doc, definition, [fragment("fxr_3", 500, 1500)], newId), + ).toBeNull(); + }); + + it("gives the paste its own id when the project holds another effect under that one", () => { + const doc = withPastedShaderEffect( + empty, + definition, + [fragment("fxr_1", 0, 1000)], + newId, + ) as AxcutDocument; + // Copied, then edited on the other side: same id, other code. + const edited = { ...definition, code: definition.code.replace("frame(uv)", "frame(uv) * 0.5") }; + const next = withPastedShaderEffect(doc, edited, [fragment("fxr_2", 2000, 3000)], newId); + expect(next?.shaderEffects?.map((e) => [e.id, e.code])).toEqual([ + ["fx_1", definition.code], + ["fx_new", edited.code], + ]); + expect(next?.shaderEffectRanges?.map((r) => r.effectId)).toEqual(["fx_1", "fx_new"]); + }); +}); diff --git a/src/lib/ai-edition/shaderEffects.ts b/src/lib/ai-edition/shaderEffects.ts new file mode 100644 index 0000000000..a1a3fbef4f --- /dev/null +++ b/src/lib/ai-edition/shaderEffects.ts @@ -0,0 +1,196 @@ +import type { + AxcutDocument, + AxcutShaderEffect, + AxcutShaderEffectParam, + AxcutShaderEffectRange, + AxcutShaderEffectValue, +} from "./schema"; + +/** + * Agent-written shader effects (docs/shader-effects.md): the pure helpers the agent tools, the + * scene projection and the editor share. The WGSL itself is checked and compiled by the + * compositor (`crates/compositor/src/shader_fx.rs`). + */ + +/** The WGSL contract this build writes and draws. */ +export const SHADER_EFFECT_CONTRACT = 1; + +export type ShaderEffectTier = "light" | "medium" | "heavy" | "very-heavy"; + +/** What the compositor says of an effect's code (`checkShaderEffect` in the addon). */ +export interface ShaderEffectCheck { + error?: string | null; + tier?: ShaderEffectTier | null; + frameReads?: number | null; + iterations?: number | null; +} + +/** Asks the compositor about a code; `null` where nothing can answer (no addon, a test). A + * promise: the compositor compiles for real, which takes 120 ms and more on Metal, off the main + * thread. */ +export type ShaderEffectChecker = ( + code: string, + params: ReadonlyArray>, +) => Promise; + +/** The checker's answer as the tool executor reads it, synchronously: `runDocumentTool` asked + * first (`shaderEffectToCheck`). `null` for a code that was not asked about. */ +export type ShaderEffectCheckLookup = ( + code: string, + params: ReadonlyArray>, +) => ShaderEffectCheck | null; + +export function findShaderEffect( + document: AxcutDocument, + effectId: string, +): AxcutShaderEffect | undefined { + return (document.shaderEffects ?? []).find((effect) => effect.id === effectId); +} + +const clamp = (value: number, min: number, max: number) => Math.min(max, Math.max(min, value)); +const isColor = (value: unknown): value is string => + typeof value === "string" && /^#(?:[0-9a-fA-F]{6}|[0-9a-fA-F]{8})$/.test(value); +const isPoint = (value: unknown): value is { x: number; y: number } => + typeof value === "object" && + value !== null && + Number.isFinite((value as { x?: unknown }).x) && + Number.isFinite((value as { y?: unknown }).y); + +/** The value a parameter takes: `value` when it has the right type, clamped to the bounds, the + * default otherwise. */ +export function resolveShaderEffectValue( + param: AxcutShaderEffectParam, + value: unknown, +): AxcutShaderEffectValue { + switch (param.type) { + case "number": + return typeof value === "number" && Number.isFinite(value) + ? clamp(value, param.min, param.max) + : param.default; + case "toggle": + return typeof value === "boolean" ? value : param.default; + case "color": + return isColor(value) ? value : param.default; + case "point": + return isPoint(value) ? { x: clamp(value.x, 0, 1), y: clamp(value.y, 0, 1) } : param.default; + } +} + +const sameValue = (a: AxcutShaderEffectValue, b: AxcutShaderEffectValue) => + JSON.stringify(a) === JSON.stringify(b); + +/** What a region stores: only the values that differ from the defaults, each resolved against + * its parameter. Names the definition does not declare are dropped. */ +export function storedShaderEffectValues( + effect: Pick, + values: Record, +): Record { + const stored: Record = {}; + for (const param of effect.params) { + if (!(param.name in values)) continue; + const value = resolveShaderEffectValue(param, values[param.name]); + if (!sameValue(value, param.default)) stored[param.name] = value; + } + return stored; +} + +function hexToRgba(hex: string): [number, number, number, number] { + const channel = (i: number) => Number.parseInt(hex.slice(1 + i * 2, 3 + i * 2), 16) / 255; + return [channel(0), channel(1), channel(2), hex.length === 9 ? channel(3) : 1]; +} + +/** A parameter's value as the compositor packs it: one float, or 2 (point) or 4 (color). */ +export function nativeShaderEffectValue( + param: AxcutShaderEffectParam, + value: unknown, +): number | number[] { + const resolved = resolveShaderEffectValue(param, value); + switch (param.type) { + case "number": + return resolved as number; + case "toggle": + return resolved ? 1 : 0; + case "color": + return hexToRgba(resolved as string); + case "point": { + const { x, y } = resolved as { x: number; y: number }; + return [x, y]; + } + } +} + +/** The document without the definitions no region uses: no dead code is kept. */ +export function pruneUnusedShaderEffects(document: AxcutDocument): AxcutDocument { + const used = new Set((document.shaderEffectRanges ?? []).map((range) => range.effectId)); + if ((document.shaderEffects ?? []).every((effect) => used.has(effect.id))) return document; + return { + ...document, + shaderEffects: (document.shaderEffects ?? []).filter((effect) => used.has(effect.id)), + }; +} + +/** The tier of a number of `frame()` reads per pixel, as the compositor computes it. */ +export function shaderEffectTierOf(frameReads: number): ShaderEffectTier { + if (frameReads <= 8) return "light"; + if (frameReads <= 32) return "medium"; + if (frameReads <= 128) return "heavy"; + return "very-heavy"; +} + +/** The longest span a tier is recommended for, in seconds (spec C.7). */ +const TIER_SPAN_SEC: Record = { + light: Number.POSITIVE_INFINITY, + medium: 60, + heavy: 10, + "very-heavy": 3, +}; + +/** The recommendation an agent gets when a tier runs longer than it is meant to, else null. */ +export function shaderEffectCostAdvice(tier: ShaderEffectTier, spanSec: number): string | null { + if (spanSec <= TIER_SPAN_SEC[tier]) return null; + const lighter = + tier === "very-heavy" ? "heavy or lighter" : tier === "heavy" ? "medium or light" : "light"; + return `${tier} over ${spanSec.toFixed(1)} s: this tier is meant for at most ${TIER_SPAN_SEC[tier]} s; for a region this long, aim ${lighter}. It is a recommendation, not a limit.`; +} + +/** Two definitions that draw and read the same: everything but the cached cost. */ +function sameShaderEffect(a: AxcutShaderEffect, b: AxcutShaderEffect): boolean { + return ( + a.name === b.name && + a.description === b.description && + a.contract === b.contract && + a.code === b.code && + JSON.stringify(a.params) === JSON.stringify(b.params) + ); +} + +/** + * `document` with a pasted effect region (its fragments, already anchored) and its definition, + * recreated when the project does not have it: a region copied from another project. A project + * holding a different definition under that id (the effect was copied, then edited on one side) + * keeps it, and the paste gets its own (`newId`). `null` when the region would overlap an effect + * already there, since one effect runs at a time. + */ +export function withPastedShaderEffect( + document: AxcutDocument, + definition: AxcutShaderEffect, + fragments: AxcutShaderEffectRange[], + newId: () => string, +): AxcutDocument | null { + const ranges = document.shaderEffectRanges ?? []; + const overlaps = fragments.some((f) => + ranges.some((r) => r.startMs < f.endMs && r.endMs > f.startMs), + ); + if (overlaps) return null; + const effects = document.shaderEffects ?? []; + const held = effects.find((e) => e.id === definition.id); + if (held && sameShaderEffect(held, definition)) { + return { ...document, shaderEffectRanges: [...ranges, ...fragments] }; + } + const pasted = held ? { ...definition, id: newId() } : definition; + return { + ...document, + shaderEffects: [...effects, pasted], + shaderEffectRanges: [...ranges, ...fragments.map((f) => ({ ...f, effectId: pasted.id }))], + }; +} diff --git a/src/lib/ai-edition/store/documentWriteAudit.test.ts b/src/lib/ai-edition/store/documentWriteAudit.test.ts index 3f11af6847..575d96f5b0 100644 --- a/src/lib/ai-edition/store/documentWriteAudit.test.ts +++ b/src/lib/ai-edition/store/documentWriteAudit.test.ts @@ -167,6 +167,7 @@ const DECLARED: WritePath[] = [ w("src/components/ai-edition/NewEditorShell.tsx", "pasteRegion", "save", "gesture"), w("src/components/ai-edition/NewEditorShell.tsx", "pasteRegion", "save", "gesture"), w("src/components/ai-edition/NewEditorShell.tsx", "pasteRegion", "save", "gesture"), + w("src/components/ai-edition/NewEditorShell.tsx", "pasteRegion", "save", "gesture"), // The window is closing and the user answered "save". w("src/components/ai-edition/NewEditorShell.tsx", "unsubSaveBeforeClose", "save", "gesture"), @@ -259,13 +260,16 @@ const DECLARED: WritePath[] = [ w("src/lib/ai-edition/store/useTimeline.ts", "addZoomsBulk", "save", "gesture"), // The Clear timeline button: every edit region in one write, so one undo step. w("src/lib/ai-edition/store/useTimeline.ts", "clearTimeline", "save", "gesture"), - // The two drag commits. One undo step per gesture, recorded on release and only + // The drag commits. One undo step per gesture, recorded on release and only // if the write lands — `historyBase` carries the pre-drag document. w("src/lib/ai-edition/store/useTimeline.ts", "commitAnnotationChange", "save", "gesture"), // And the rollback each of them runs when that write does not land: the pre-drag // document back into the store, around the writers, so it records nothing. There is // nothing to take off either — the commit records only on success. w("src/lib/ai-edition/store/useTimeline.ts", "commitAnnotationChange", "state", "unrecorded"), + // The effect inspector's sliders and colours, committed as the annotations' are. + w("src/lib/ai-edition/store/useTimeline.ts", "commitShaderEffectChange", "save", "gesture"), + w("src/lib/ai-edition/store/useTimeline.ts", "commitShaderEffectChange", "state", "unrecorded"), w("src/lib/ai-edition/store/useTimeline.ts", "commitZoomFocus", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "commitZoomFocus", "state", "unrecorded"), w("src/lib/ai-edition/store/useTimeline.ts", "duplicateClip", "save", "gesture"), @@ -292,10 +296,13 @@ const DECLARED: WritePath[] = [ w("src/lib/ai-edition/store/useTimeline.ts", "removeRegions", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "saveZoomPatch", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "setTrimEntries", "save", "gesture"), - // The live halves of the two drags. + // The live halves of the drags. w("src/lib/ai-edition/store/useTimeline.ts", "updateAnnotationLive", "set", "automatic"), w("src/lib/ai-edition/store/useTimeline.ts", "updateAnnotationSpan", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "updateCameraFullscreenSpan", "save", "gesture"), + w("src/lib/ai-edition/store/useTimeline.ts", "updateShaderEffect", "save", "gesture"), + w("src/lib/ai-edition/store/useTimeline.ts", "updateShaderEffectLive", "set", "automatic"), + w("src/lib/ai-edition/store/useTimeline.ts", "updateShaderEffectSpan", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "updateSpeedSpan", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "updateSpeedValue", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "updateTrim", "save", "gesture"), diff --git a/src/lib/ai-edition/store/regionClipboard.ts b/src/lib/ai-edition/store/regionClipboard.ts index f0d6d7d104..9ae66a8ab0 100644 --- a/src/lib/ai-edition/store/regionClipboard.ts +++ b/src/lib/ai-edition/store/regionClipboard.ts @@ -5,6 +5,7 @@ // this path). import { useCallback, useEffect, useState } from "react"; +import type { AxcutShaderEffect } from "../schema"; export type RegionSnapshot = | { kind: "zoom"; region: Record } @@ -15,6 +16,9 @@ export type RegionSnapshot = // paste is a second placement of the same audio, like every other kind. The // snapshot is the COLLAPSED pill, never a stored fragment. | { kind: "audio"; region: Record } + // An effect region carries its definition: pasted into another project, the definition is + // recreated there, since the region alone would name an effect that project does not have. + | { kind: "shaderEffect"; region: Record; definition: AxcutShaderEffect } // A trim carries no user-visible properties, so all there is to copy is how // LONG it was — `{ durationSec }`. That is not a special case so much as the // general one made obvious: every paste keeps the copied properties and takes diff --git a/src/lib/ai-edition/store/useTimeline.test.ts b/src/lib/ai-edition/store/useTimeline.test.ts index 917188529c..f8e6ba8a39 100644 --- a/src/lib/ai-edition/store/useTimeline.test.ts +++ b/src/lib/ai-edition/store/useTimeline.test.ts @@ -680,6 +680,110 @@ describe("useTimeline.addCameraFullscreen (camera gate)", () => { }); }); +describe("useTimeline shader effects (the Effects lane and its inspector)", () => { + const effect = { + id: "fx_1", + name: "Dim", + description: "", + contract: 1, + code: "fn effect(uv: vec2) -> vec4 { return frame(uv) * params.level; }", + params: [ + { type: "number" as const, name: "level", label: "Level", default: 1, min: 0, max: 2 }, + ], + origin: "agent" as const, + }; + const range = (id: string, startMs: number, endMs: number) => ({ + id, + startMs, + endMs, + clipId: "clip_a", + sourceStartSec: startMs / 1000, + sourceEndSec: endMs / 1000, + effectId: "fx_1", + values: {}, + target: "image" as const, + }); + const effectDoc: AxcutDocument = { + ...sampleDoc, + shaderEffects: [effect], + // Another value on the second, so it stays a pill of its own beside the first. + shaderEffectRanges: [ + range("fxr_1", 1000, 3000), + { ...range("fxr_2", 5000, 7000), values: { level: 0.5 } }, + ], + }; + const ranges = () => useProjectStore.getState().document?.shaderEffectRanges ?? []; + + beforeEach(() => { + useProjectStore.getState().clear(); + clearHistory(); + for (const mock of Object.values(bridgeMocks)) mock.mockReset(); + bridgeMocks.save.mockImplementation(async (doc: typeof sampleDoc) => ({ + success: true, + document: doc, + })); + useProjectStore.setState({ + projectId: "proj_test", + document: effectDoc, + currentTimeSec: 1, + revision: 1, + status: "ready", + error: null, + }); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it("moves an effect up to its neighbour, never over it: one effect at a time", async () => { + const { result } = renderTimeline(); + await act(async () => { + await result.current.updateShaderEffectSpan("fxr_1", 2000, 6000); + }); + const [moved, wall] = coalesceRegionsForRuler(ranges()); + expect(moved.end).toBeLessThanOrEqual(5); + expect([wall.start, wall.end]).toEqual([5, 7]); + }); + + it("re-tunes a value live, then saves it as one undo step; retargets in one write", async () => { + const { result } = renderTimeline(); + const before = past.length; + act(() => { + result.current.updateShaderEffectLive("fxr_1", { values: { level: 1.5 } }); + result.current.updateShaderEffectLive("fxr_1", { values: { level: 0.5 } }); + }); + await act(async () => { + await result.current.commitShaderEffectChange(); + }); + expect(ranges().find((r) => r.id === "fxr_1")?.values).toEqual({ level: 0.5 }); + expect(past.length).toBe(before + 1); + await act(async () => { + await result.current.updateShaderEffect("fxr_1", { target: "text", values: { level: 1 } }); + }); + // Back at its default, the value is not stored. + expect(ranges().find((r) => r.id === "fxr_1")).toMatchObject({ target: "text", values: {} }); + await act(async () => { + await undo(); + await undo(); + }); + expect(ranges().find((r) => r.id === "fxr_1")).toMatchObject({ target: "image", values: {} }); + }); + + it("deletes effects in a batch, and their definition with the last", async () => { + const { result } = renderTimeline(); + await act(async () => { + await result.current.removeRegions([ + { kind: "shaderEffect", id: "fxr_1" }, + { kind: "shaderEffect", id: "fxr_2" }, + ]); + }); + const doc = useProjectStore.getState().document; + expect(doc?.shaderEffectRanges).toEqual([]); + expect(doc?.shaderEffects).toEqual([]); + }); +}); + describe("useTimeline.addAnnotation", () => { beforeEach(() => { useProjectStore.getState().clear(); @@ -750,6 +854,41 @@ describe("useTimeline.clearTimeline", () => { annotation: (tl) => tl.addAnnotation(), speed: (tl) => tl.addSpeed(), cameraFullscreen: (tl) => tl.addCameraFullscreen(), + // The editor has no effect lane: agents add effects, so this one is written as they do. + shaderEffect: async () => { + const doc = useProjectStore.getState().document as AxcutDocument; + const clip = doc.timeline.clips[0]; + await useProjectStore.getState().saveDocument( + { + ...doc, + shaderEffects: [ + { + id: "fx_1", + name: "Pass-through", + description: "", + contract: 1, + code: "fn effect(uv: vec2) -> vec4 { return frame(uv); }", + params: [], + origin: "agent", + }, + ], + shaderEffectRanges: [ + { + id: "fxr_1", + startMs: 0, + endMs: 1000, + clipId: clip.id, + sourceStartSec: clip.sourceStartSec, + sourceEndSec: clip.sourceStartSec + 1, + effectId: "fx_1", + values: {}, + target: "image", + }, + ], + }, + { history: true }, + ); + }, }; const editKinds = Object.keys(addOf) as EditKind[]; @@ -761,6 +900,7 @@ describe("useTimeline.clearTimeline", () => { annotation: doc?.annotations.length ?? 0, speed: doc ? readSpeedRegions(doc).length : 0, cameraFullscreen: legacy.cameraFullscreenRegions?.length ?? 0, + shaderEffect: doc?.shaderEffectRanges?.length ?? 0, }; }; const noRegions: Record = { @@ -769,6 +909,7 @@ describe("useTimeline.clearTimeline", () => { annotation: 0, speed: 0, cameraFullscreen: 0, + shaderEffect: 0, }; // A project holding everything that is NOT an edit: two clips, a webcam, an imported @@ -928,6 +1069,7 @@ describe("useTimeline.clearTimeline", () => { annotation: 1, speed: 1, cameraFullscreen: 1, + shaderEffect: 1, }); await act(async () => { @@ -967,6 +1109,7 @@ describe("useTimeline.clearTimeline", () => { annotation: 1, speed: 1, cameraFullscreen: 1, + shaderEffect: 1, }); act(() => { diff --git a/src/lib/ai-edition/store/useTimeline.ts b/src/lib/ai-edition/store/useTimeline.ts index 7c173f870f..7d650ed85b 100644 --- a/src/lib/ai-edition/store/useTimeline.ts +++ b/src/lib/ai-edition/store/useTimeline.ts @@ -37,7 +37,17 @@ import { resequenceClips, setClipSourceRange, } from "../document/timeline"; -import type { AxcutAudioTrack, AxcutClipCropRegion, AxcutDocument } from "../schema"; +import type { + AxcutAudioTrack, + AxcutClipCropRegion, + AxcutDocument, + AxcutShaderEffectRange, +} from "../schema"; +import { + findShaderEffect, + pruneUnusedShaderEffects, + storedShaderEffectValues, +} from "../shaderEffects"; import { appendAutoZoomSuggestions } from "../timeline/apply-auto-zooms"; import { hasAnyClipWithCamera } from "../timeline/camera"; import { probeAudioDuration, probeVideoDimensions, probeVideoDuration } from "../timeline/duration"; @@ -45,6 +55,7 @@ import { anchorRegionsWithDerivedMs, dropPillsByIds, replacePillSpan, + resolvePillIds, resolveRegionIds, } from "../timeline/timelineMap"; import { dropTrimPillsByIds, resolveTimelineSpanToTrim } from "../timeline/trim-mapping"; @@ -116,6 +127,41 @@ function playheadSec(): number { return useProjectStore.getState().currentTimeSec; } +/** What the inspector changes on an effect region: its parameter values, or its target. */ +export interface ShaderEffectPatch { + values?: Record; + target?: AxcutShaderEffectRange["target"]; +} + +/** Every fragment of the effect pill `id` (one region, cut at clip boundaries), patched alike so + * it stays one pill. Values keep only what differs from the definition's defaults. `null` when + * the region or its definition is gone. */ +function patchShaderEffectPill( + doc: AxcutDocument, + id: string, + patch: ShaderEffectPatch, +): AxcutDocument | null { + const ranges = doc.shaderEffectRanges ?? []; + const region = ranges.find((r) => r.id === id); + const definition = region ? findShaderEffect(doc, region.effectId) : null; + if (!region || !definition) return null; + const ids = new Set(resolvePillIds(ranges, id)); + return { + ...doc, + shaderEffectRanges: ranges.map((r) => + ids.has(r.id) + ? { + ...r, + ...(patch.values + ? { values: storedShaderEffectValues(definition, { ...r.values, ...patch.values }) } + : {}), + ...(patch.target ? { target: patch.target } : {}), + } + : r, + ), + }; +} + export function useTimeline() { const ts = useScopedT("settings"); const document = useProjectStore((s) => s.document); @@ -151,6 +197,8 @@ export function useTimeline() { } | null>(null); const annotationRollbackRef = useRef(null); const annotationLiveRef = useRef(null); + const shaderEffectRollbackRef = useRef(null); + const shaderEffectLiveRef = useRef(null); // A drag does not always end in a commit: `ZoomFocusOverlay` unmounts the moment // `focusMode` flips to "auto", so `endDrag` never runs and the snapshot outlives the @@ -165,6 +213,8 @@ export function useTimeline() { zoomFocusEditRef.current = null; annotationRollbackRef.current = null; annotationLiveRef.current = null; + shaderEffectRollbackRef.current = null; + shaderEffectLiveRef.current = null; }, [projectId]); const hasDoc = document !== null && projectId !== null; @@ -1052,6 +1102,68 @@ export function useTimeline() { [document, saveDocument], ); + const updateShaderEffectSpan = useCallback( + async (id: string, startMs: number, endMs: number) => { + if (!document) return; + const s = finiteMs(startMs); + const e = finiteMs(endMs); + // Clamped against the neighbouring effects: one effect runs at a time. + const next: AxcutDocument = { + ...document, + shaderEffectRanges: replacePillSpan( + document.shaderEffectRanges ?? [], + id, + Math.min(s, e), + Math.max(s, e), + document.timeline.clips, + () => createId("fxr"), + ), + }; + await saveDocument(next, { history: true }); + }, + [document, saveDocument], + ); + + // A discrete inspector edit (the target, a toggle): one save, one undo step. + const updateShaderEffect = useCallback( + async (id: string, patch: ShaderEffectPatch) => { + if (!document) return; + const next = patchShaderEffectPill(document, id, patch); + if (next) await saveDocument(next, { history: true }); + }, + [document, saveDocument], + ); + + // A continuous one (a slider, a colour): local writes while it lasts, then one save and one + // undo step from `commitShaderEffectChange`. The annotations' live path, for effects. + const updateShaderEffectLive = useCallback( + (id: string, patch: ShaderEffectPatch) => { + const doc = useProjectStore.getState().document; + if (!doc) return; + const next = patchShaderEffectPill(doc, id, patch); + if (!next) return; + if (shaderEffectLiveRef.current !== doc) shaderEffectRollbackRef.current = doc; + setDocument(next, { history: false }); + shaderEffectLiveRef.current = next; + }, + [setDocument], + ); + + // `commitAnnotationChange`, for the effect refs: the pre-gesture document is the undo + // target, recorded only if the write lands, and put back if it does not. + const commitShaderEffectChange = useCallback(async () => { + const doc = useProjectStore.getState().document; + if (!doc) return; + const rollback = shaderEffectLiveRef.current === doc ? shaderEffectRollbackRef.current : null; + shaderEffectRollbackRef.current = null; + shaderEffectLiveRef.current = null; + if (!(await saveDocument(doc, { history: true, historyBase: rollback })) && rollback) { + useProjectStore.setState((state) => + state.document === doc ? { document: rollback, revision: state.revision + 1 } : {}, + ); + } + }, [saveDocument]); + const updateSpeedValue = useCallback( async (id: string, speed: number) => { if (!document) return; @@ -1100,6 +1212,9 @@ export function useTimeline() { const cameraFullscreenIds = new Set( handles.filter((h) => h.kind === "cameraFullscreen").map((h) => h.id), ); + const shaderEffectIds = new Set( + handles.filter((h) => h.kind === "shaderEffect").map((h) => h.id), + ); const legacy = (document.legacyEditor as Record) ?? {}; const prevSpeed = dropPillsByIds( (legacy.speedRegions as Array<{ id: string; startMs: number; endMs: number }>) ?? [], @@ -1110,10 +1225,18 @@ export function useTimeline() { [], cameraFullscreenIds, ); - const next: AxcutDocument = { + const next: AxcutDocument = pruneUnusedShaderEffects({ ...document, zoomRanges: dropPillsByIds(document.zoomRanges, zoomIds) as AxcutDocument["zoomRanges"], annotations: dropPillsByIds(document.annotations, annotationIds), + ...(shaderEffectIds.size > 0 + ? { + shaderEffectRanges: dropPillsByIds( + document.shaderEffectRanges ?? [], + shaderEffectIds, + ), + } + : {}), timeline: { ...document.timeline, // Whole-pill delete, same as the zoom/annotation lines above — a trim grown @@ -1129,7 +1252,7 @@ export function useTimeline() { speedIds.size > 0 || cameraFullscreenIds.size > 0 ? { ...legacy, speedRegions: prevSpeed, cameraFullscreenRegions: prevCameraFullscreen } : document.legacyEditor, - }; + }); if (!(await saveDocument(next, { history: true }))) return; setSelection(null); setMultiSelection([]); @@ -1430,6 +1553,10 @@ export function useTimeline() { }>) ?? []) : []; + // One region per effect, never two at once; each names its definition. + const shaderEffectRanges = hasDoc ? (document.shaderEffectRanges ?? []) : []; + const shaderEffects = hasDoc ? (document.shaderEffects ?? []) : []; + // --- Timeline audio tracks (issue #350) ------------------------------------- // CLIP-ANCHORED like every region above: one user-visible track is one pill // over one-or-more stored fragments, so these ops go through the shared pill @@ -1615,6 +1742,8 @@ export function useTimeline() { annotationRegions: (document?.annotations ?? []) as unknown as AnnotationRegion[], speedRegions, cameraFullscreenRegions, + shaderEffectRanges, + shaderEffects, clips: document?.timeline.clips ?? [], assets: document?.assets ?? [], // The timeline marks where the user has ADDED words — text with no audio behind it. @@ -1668,6 +1797,10 @@ export function useTimeline() { updateSpeedSpan, updateSpeedValue, updateCameraFullscreenSpan, + updateShaderEffectSpan, + updateShaderEffect, + updateShaderEffectLive, + commitShaderEffectChange, // T19 — drives the preview video during trim-edge resize. setCurrentTime: useProjectStore((s) => s.setCurrentTime), }; diff --git a/src/native/compositorViewClient.ts b/src/native/compositorViewClient.ts index 131d8909cb..990a08674c 100644 --- a/src/native/compositorViewClient.ts +++ b/src/native/compositorViewClient.ts @@ -7,6 +7,7 @@ * `readCompositorFrame` and paints them into a ``. */ +import type { ShaderEffectCheck } from "@/lib/ai-edition/shaderEffects"; import { requireNativeBridgeData } from "./client"; import type { CompositorBackend, @@ -60,6 +61,24 @@ export async function probeSegmentationSupport(): Promise { } } +/** An effect's code checked by this machine's compositor, as the agent tools check it: the + * compiler's error, or the cost tier. `null` when nothing could check it (no bridge, no addon). */ +export async function checkShaderEffectCode( + code: string, + params: ReadonlyArray<{ name: string; type: string }>, +): Promise { + try { + const { check } = await requireNativeBridgeData<{ check: ShaderEffectCheck | null }>({ + domain: "compositor", + action: "checkShaderEffect", + payload: { code, params: params.map(({ name, type }) => ({ name, type })) }, + }); + return check; + } catch { + return null; + } +} + /** The segmentation model's input size (`segmentation::MODEL_WIDTH` x `MODEL_HEIGHT`). */ export const SEGMENTATION_WIDTH = 256; export const SEGMENTATION_HEIGHT = 144; diff --git a/src/native/contracts.ts b/src/native/contracts.ts index 38bfaf1dfb..c837b216ea 100644 --- a/src/native/contracts.ts +++ b/src/native/contracts.ts @@ -841,6 +841,13 @@ export type NativeBridgeRequest = action: "probeSegmentation"; requestId?: string; } + | { + domain: "compositor"; + action: "checkShaderEffect"; + /** An effect's code and declared parameters, as the agent tools check them. */ + payload: { code: string; params: Array<{ name: string; type: string }> }; + requestId?: string; + } | { domain: "compositor"; action: "segmentFrame"; diff --git a/src/native/sceneDescription.test.ts b/src/native/sceneDescription.test.ts index 0d84461b54..567fe925eb 100644 --- a/src/native/sceneDescription.test.ts +++ b/src/native/sceneDescription.test.ts @@ -102,6 +102,8 @@ function makeDoc( annotations: overrides.annotations ?? [], zoomRanges: overrides.zoomRanges ?? [], audioTracks: overrides.audioTracks ?? [], + ...(overrides.shaderEffects ? { shaderEffects: overrides.shaderEffects } : {}), + ...(overrides.shaderEffectRanges ? { shaderEffectRanges: overrides.shaderEffectRanges } : {}), legacyEditor: overrides.legacyEditor ?? null, }; } @@ -2949,3 +2951,96 @@ describe("buildSceneDescription.audioTracks", () => { expect(buildSceneDescription(doc).audioTracks).toEqual([]); }); }); + +// --- shaderEffects ------------------------------------------------------------- + +describe("buildSceneDescription.shaderEffects", () => { + const code = "fn effect(uv: vec2) -> vec4 { return frame(uv); }"; + const effect = { + id: "fx1", + name: "Tint", + description: "", + contract: 1, + code, + origin: "agent" as const, + params: [ + { type: "number" as const, name: "level", label: "", default: 1, min: 0, max: 2 }, + { type: "color" as const, name: "tint", label: "", default: "#000000" }, + { type: "toggle" as const, name: "on", label: "", default: false }, + ], + }; + + it("is omitted when there are none", () => { + expect(buildSceneDescription(makeDoc()).shaderEffects).toBeUndefined(); + }); + + // A trim inside the region cuts it in two fragments on the source clock, but it stays one + // pill: both fragments carry the pill's span on the programme clock, which the trim shortened. + it("gives every fragment the whole pill on the programme clock, and every value", () => { + const doc = makeDoc({ + assets: [makeAsset({ id: "a", originalPath: "/a.mp4" })], + clips: [ + makeClip({ + id: "c1", + assetId: "a", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + }), + ], + timeline: { + trimRanges: [ + { id: "t1", assetId: "a", startSec: 2, endSec: 4, reason: "", origin: "user" }, + ], + }, + shaderEffects: [effect], + shaderEffectRanges: [ + { + id: "fxr1", + startMs: 1000, + endMs: 6000, + effectId: "fx1", + values: { tint: "#ff0000", on: true }, + target: "text", + }, + ], + }); + const { shaderEffects } = buildSceneDescription(doc); + expect(shaderEffects?.map((e) => [e.startSec, e.endSec, e.clipIndex])).toEqual([ + [1, 2, 0], + [4, 6, 1], + ]); + for (const fragment of shaderEffects ?? []) { + expect(fragment).toMatchObject({ + programmeStartSec: 1, + programmeEndSec: 4, + target: "text", + contract: 1, + code, + params: [ + { name: "level", type: "number" }, + { name: "tint", type: "color" }, + { name: "on", type: "toggle" }, + ], + values: { level: 1, tint: [1, 0, 0, 1], on: 1 }, + }); + } + }); + + it("drops a region whose definition is gone", () => { + const doc = makeDoc({ + shaderEffectRanges: [ + { + id: "fxr1", + startMs: 0, + endMs: 1000, + effectId: "fx_gone", + values: {}, + target: "image", + }, + ], + }); + expect(buildSceneDescription(doc).shaderEffects).toBeUndefined(); + }); +}); diff --git a/src/native/sceneDescription.ts b/src/native/sceneDescription.ts index 5cf2a8e5d5..71e920b38c 100644 --- a/src/native/sceneDescription.ts +++ b/src/native/sceneDescription.ts @@ -41,12 +41,16 @@ import { resolvePlaybackSegments, } from "@/lib/ai-edition/document/timeline"; import type { AxcutAudioTrack, AxcutClip, AxcutDocument } from "@/lib/ai-edition/schema"; +import { nativeShaderEffectValue } from "@/lib/ai-edition/shaderEffects"; import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings"; import { assetCameraSource } from "@/lib/ai-edition/timeline/camera"; import { resolveClipSourceEndSec } from "@/lib/ai-edition/timeline/clipDuration"; import { removedRawSpans } from "@/lib/ai-edition/timeline/programme-time"; import { takeProgramme } from "@/lib/ai-edition/timeline/take-programme"; -import { projectRegionsToSource } from "@/lib/ai-edition/timeline/timelineMap"; +import { + coalesceRegionsForRuler, + projectRegionsToSource, +} from "@/lib/ai-edition/timeline/timelineMap"; import { MAGNIFICATION_REFERENCE_PX, MAX_ZOOM_SCALE, @@ -120,6 +124,25 @@ export interface SceneZoomRegion { hideCursor?: boolean; } +/** One fragment of a shader effect region (`(document.shaderEffectRanges ?? [])`), projected onto its + * clip's source time like the other regions, with its definition. `scene.rs` + * `SceneShaderEffect` reads it; docs/shader-effects.md, B.5. */ +export interface SceneShaderEffect { + startSec: number; + endSec: number; + clipIndex?: number; + /** The WHOLE pill on the programme clock (trims skipped, speed applied): the compositor's + * `ctx.region` runs across it instead of restarting at each fragment. */ + programmeStartSec: number; + programmeEndSec: number; + target: "image" | "text" | "both"; + contract: number; + code: string; + params: Array<{ name: string; type: "number" | "toggle" | "color" | "point" }>; + /** Every declared parameter, defaults filled in: one float, or 2 (point) or 4 (color). */ + values: Record; +} + /** A "Full Camera" timeline region (from `legacyEditor.cameraFullscreenRegions`). Times in seconds. */ export interface SceneCameraFullscreenRegion { startSec: number; @@ -479,6 +502,8 @@ export interface SceneDescription { * source-time span after `projectRegionsToSourceTime`). Empty when none set. */ cameraFullscreenRegions: SceneCameraFullscreenRegion[]; + /** Agent-written shader effects, omitted when there are none. */ + shaderEffects?: SceneShaderEffect[]; /** * Speed regions projected onto each clip's source time (one entry per * source-time span after `projectRegionsToSourceTime`). Empty when none set. @@ -661,6 +686,40 @@ export function resolveVisibleClips(document: AxcutDocument): PlaybackSegment[] .filter((clip) => clipAssetIsResolvable(clip, assetById)); } +/** + * The clip list a native export renders: the trim-narrowed visible clips, mapped onto the native + * multiclip contract. `openscreen export` and getFrames share it, beside `buildSceneDescription` + * so that the clip stream and the scene agree. + * ponytail: the Export dialog keeps an identical copy, which its tests' module mocks rely on. + */ +export function buildNativeClipList(document: AxcutDocument): CompositorClipInput[] { + const assetById = new Map(document.assets.map((a) => [a.id, a])); + return resolveVisibleClips(document).flatMap((clip) => { + const asset = assetById.get(clip.assetId); + if (!asset?.originalPath) { + return []; + } + const camera = assetCameraSource(asset); + // sourceEndSec is optional in the schema (unknown until probed) — fall back through + // the single canonical precedence used by every consumer (clip.probe → asset.duration + // → timeline-length guess). See `resolveClipSourceEndSec` for the full order. + const sourceEndSec = resolveClipSourceEndSec(clip, asset); + // ponytail: `hasAudio` stays optimistic for the same reason as in + // `buildSceneDescription` — nothing populates `asset.audio` yet, and the + // native side degrades cleanly on a stream-less file. + return [ + { + screenPath: asset.originalPath, + webcamPath: camera.path, + sourceStartSec: clip.sourceStartSec, + sourceEndSec, + webcamOffsetSec: camera.offsetSec, + hasAudio: true, + }, + ]; + }); +} + /** A clip's screen source size in pixels: its recording × its crop (see `screenSourceSizeOf`). */ function screenSourceSize( video: { width?: number; height?: number } | null | undefined, @@ -1085,6 +1144,58 @@ export function buildSceneDescription( () => createId("speed"), ); + // Shader effects: projected like the zooms, and each fragment told where its WHOLE pill sits + // on the programme clock, through the same projection the exporter plays the timeline with + // (trims skipped, speed applied). A pill cut by a trim or a clip boundary keeps one progress. + const effectsById = new Map((document.shaderEffects ?? []).map((effect) => [effect.id, effect])); + const programmeSec = (rawSec: number) => + projectRawTimelineSecToPlayback( + projectedClips, + document.timeline.trimRanges, + rawSec, + rawSpeedRegions, + ); + const pillSpanById = new Map(); + for (const pill of coalesceRegionsForRuler(document.shaderEffectRanges ?? [])) { + const span = { + programmeStartSec: programmeSec(pill.start), + programmeEndSec: programmeSec(pill.end), + }; + for (const id of pill.ids) pillSpanById.set(id, span); + } + const projectedShaderEffects = projectRegionsToSource( + (document.shaderEffectRanges ?? []).map((range) => ({ + ...range, + ...pillSpanById.get(range.id), + })), + visibleClips, + document.timeline.clips, + () => createId("fxr"), + ); + const sceneShaderEffects: SceneShaderEffect[] = projectedShaderEffects.flatMap((region) => { + const effect = effectsById.get(region.effectId); + if (!effect) return []; + return [ + { + startSec: region.startMs / 1000, + endSec: region.endMs / 1000, + clipIndex: region.clipIndex, + programmeStartSec: region.programmeStartSec ?? region.startMs / 1000, + programmeEndSec: region.programmeEndSec ?? region.endMs / 1000, + target: region.target, + contract: effect.contract, + code: effect.code, + params: effect.params.map(({ name, type }) => ({ name, type })), + values: Object.fromEntries( + effect.params.map((param) => [ + param.name, + nativeShaderEffectValue(param, region.values[param.name]), + ]), + ), + }, + ]; + }); + // Webcam rect, single source of truth between preview & native : // on résout le rect AVEC LA MÊME maths que `PreviewCanvas.computeCompositeLayout` et on // l'envoie au natif dans `layout.webcamRect` (fractions du cadre de sortie). Le natif le @@ -1470,6 +1581,7 @@ export function buildSceneDescription( clipIndex: region.clipIndex, })), cropByClip, + ...(sceneShaderEffects.length > 0 ? { shaderEffects: sceneShaderEffects } : {}), output: { ...pickOutputDims(document, settings.aspectRatio), fps: null }, // Omitted rather than sent as `{mode:"none"}`: the Rust side defaults the field, and // every project without a webcam effect would otherwise carry it for nothing. diff --git a/technical-documentation/architecture/ai-agent.md b/technical-documentation/architecture/ai-agent.md index d36f20614e..b09493e133 100644 --- a/technical-documentation/architecture/ai-agent.md +++ b/technical-documentation/architecture/ai-agent.md @@ -41,7 +41,7 @@ The agent is built with LangChain's `createAgent`, not `deepagents`' `createDeep ## Tool schema -The model never free-writes the project document. It can only call the fixed set of 26 tools built by `deep-agent/service.ts#buildTools`, described by `TOOL_DESCRIPTIONS` in the same file and validated against the Zod argument schemas in `agent-tools.ts`; the executor parses JSON, validates arguments, and returns either a new schema-valid snapshot or an error. Those three surfaces (descriptions, built tools, executor cases) are pinned to one another by `deep-agent/service.test.ts` — an earlier fourth surface, `AGENT_TOOL_SPECS`, described the tools in JSON Schema for a provider it had stopped reaching, and drifted. The tools operate on the same [document model](document-model.md) as manual editing. +The model never free-writes the project document. It can only call the fixed set of tools built by `deep-agent/service.ts#buildTools` (`OPENSCREEN_TOOL_NAMES`), described by `TOOL_DESCRIPTIONS` in the same file and validated against the Zod argument schemas in `agent-tools.ts`; the executor parses JSON, validates arguments, and returns either a new schema-valid snapshot or an error. Those three surfaces (descriptions, built tools, executor cases) are pinned to one another by `deep-agent/service.test.ts` — an earlier fourth surface, `AGENT_TOOL_SPECS`, described the tools in JSON Schema for a provider it had stopped reaching, and drifted. The tools operate on the same [document model](document-model.md) as manual editing. | Tool | What it does | What it mutates | |---|---|---| @@ -64,11 +64,15 @@ The model never free-writes the project document. It can only call the fixed set | `setAnnotation` | Moves, resizes, retypes or restyles an annotation. A type change goes through `convertAnnotationKind` with the footage under each fragment (`annotationFootageRect`), as the inspector's type switch does; `figureData` / `blurData` merge only the keys sent. Arguments of another kind (width on a text, blurData on an arrow) are refused. | The corresponding clip-anchored `annotations` fragments. | | `addCameraFullscreen` | Adds a camera-fullscreen region over virtual timeline time; refused when no clip under the span comes from an asset with a linked `cameraTrack`, since such a region can only render nothing. | `legacyEditor.cameraFullscreenRegions`. | | `setCameraFullscreen` | Moves or resizes a camera-fullscreen pill, under the same camera requirement as `addCameraFullscreen`. | The corresponding `legacyEditor.cameraFullscreenRegions` fragments. | +| `addEffect` | Writes a shader effect (WGSL, [docs/shader-effects.md](../../docs/shader-effects.md)) and poses it over virtual timeline time, or poses one of the project's effects again by `effectId`. The compositor checks the code first (`checkShaderEffect`); code that does not compile is refused with the error at the agent's line. The result gives the cost tier and, when the tier runs long, a recommendation, never a refusal. One effect runs at a time: two cannot overlap. | `shaderEffects` (a new definition) and `shaderEffectRanges`. | +| `setEffect` | Moves, re-tunes (`values`) or retargets an effect pill. | The pill's `shaderEffectRanges` fragments. | +| `setEffectDefinition` | Changes an effect's name, description, code or parameters, and so every region using it; new code is checked first. A removed parameter's values drop. | The `shaderEffects` entry and its regions' `values`. | +| `getEffect` | Reads one effect in full, code included; `getCurrentDocument` lists effects without their code. | Read only. | | `removeTrim` | Deletes a trim so its source span plays and exports again. | `timeline.trimRanges`. | -| `removeModifier` | Resolves and deletes a zoom, speed, annotation, or camera-fullscreen pill by ID; `removedIds` reports every touching storage row deleted with that pill. | The matching modifier collection. | +| `removeModifier` | Resolves and deletes a zoom, speed, annotation, camera-fullscreen, audio, or effect pill by ID; `removedIds` reports every touching storage row deleted with that pill. | The matching modifier collection. | | `removeClip` | Deletes a placed clip, closes the gap, and drops effects anchored only to it. The result names the modifiers and trims it took with it. | Timeline clips and affected anchored modifiers. | -Clips and trims use source time. Zoom, speed, annotation, and camera-fullscreen tools use virtual edited-timeline time; the executor converts these spans to the clip-anchored millisecond representation used by the document. +Clips and trims use source time. Zoom, speed, annotation, camera-fullscreen, and effect tools use virtual edited-timeline time; the executor converts these spans to the clip-anchored millisecond representation used by the document. ## Checkpoints and undo diff --git a/technical-documentation/architecture/decisions.md b/technical-documentation/architecture/decisions.md index ac249fe6c9..f0f16385da 100644 --- a/technical-documentation/architecture/decisions.md +++ b/technical-documentation/architecture/decisions.md @@ -26,6 +26,7 @@ A decision leaves this list only when the code stops honouring it. | **Migrations are forward-only.** A document is migrated up to the current `schemaVersion` on open and never written back down. | Round-tripping through an older schema loses fields silently. | | **Captions are derived from the transcript, not injected as annotations.** | The earlier design generated annotation objects from captions, which then drifted from the transcript the moment either was edited. The transcript is the SSOT for spoken words. | | **One package, one repository.** No sidecar process, no local HTTP server, no monorepo. | The editing engine was adopted from a project that had a Python worker and a Fastify server; both were replaced by in-process TypeScript and Electron IPC. Adding a second runtime back is a large, permanent cost. | +| **Agent-written shader effects are WGSL, translated by naga 24 and compiled off the render thread; an effect is a pure function of the frame, time, pointer and its parameters.** One source for the three backends; the document keeps only that source, never a binary. Cost is an indicative tier, never a refusal; the one hard rule is literal loop bounds. | The Windows spike measured +0.04 ms per 1080p frame for a heavy effect and an unchanged export time, and preview and export agreeing within codec noise. naga 25+ emits D3D12 sampler heaps that FXC `ps_5_0` rejects. A budget measured on the authoring machine was rejected: it would refuse high-end effects on a modest machine although the export renders them fine. See [docs/shader-effects.md](../../docs/shader-effects.md). | ## Rejected, with the reason diff --git a/technical-documentation/architecture/document-model.md b/technical-documentation/architecture/document-model.md index 5b9940e234..6cdec83d3f 100644 --- a/technical-documentation/architecture/document-model.md +++ b/technical-documentation/architecture/document-model.md @@ -27,6 +27,8 @@ and anything unknown is rejected by the `z.literal(axcutSchemaVersion)` check in | `timeline` | `{ clips[], gaps[], trimRanges[], muteRanges[], speedRanges[], captionRanges[] }` | Clips carry their own in/out (`sourceStartSec`/`sourceEndSec`); trims are anchored to a clip (`clipId?`) since v7. See [timeline-model.md](timeline-model.md). | | `annotations[]` | `AxcutAnnotationRegion[]` | Text/image/figure/blur overlays, anchored to a clip (`clipId?`). | | `zoomRanges[]` | `AxcutZoomRegion[]` | Zoom-in effects, depth 1–6, anchored to a clip (`clipId?`). | +| `shaderEffects[]?` | `AxcutShaderEffect[]` | Agent-written shader effects: WGSL `code`, `name`, `description`, the contract version, declared `params`, and the compositor's cached `cost`. Optional: absent on a document without effects. A definition no region uses is dropped by the edit that removed its last region. See [docs/shader-effects.md](../../docs/shader-effects.md). | +| `shaderEffectRanges[]?` | `AxcutShaderEffectRange[]` | Where each effect applies: clip-anchored regions like the zooms, with the parameter `values` that differ from the defaults, a `target` (`image`, `text`, `both`). One effect at a time: regions do not overlap. Optional, like `shaderEffects`. | | `audioTracks[]` | `AxcutAudioTrack[]` | Imported audio (voiceover / BGM / SFX, issue #350) mixed over the programme. NOT clip-anchored — addressed in RAW/document timeline seconds (`timelineStartSec`), with `trimStartSec`/`trimEndSec` windowing the source and `gainDb` its level. Added from the timeline toolbar; the referenced asset has `kind: "audio"`. | | `legacyEditor` | OpenScreen v2 `ProjectEditorState` passthrough | Appearance/cursor settings not yet first-class in the AI-edition schema. | | `agent` | `{ baseIntent?, pendingQuestions[], suggestions[], lastAppliedOperations[], lastReasoningSummary? }` | LLM agent state. | diff --git a/technical-documentation/architecture/editor-shell.md b/technical-documentation/architecture/editor-shell.md index 53f7fa18fb..05d8cb62f2 100644 --- a/technical-documentation/architecture/editor-shell.md +++ b/technical-documentation/architecture/editor-shell.md @@ -104,20 +104,24 @@ selection pane. ## Adding a region kind An ordered checklist of every file that must move when a new modifier -(`"blur"`, `"captionHighlight"`, …) joins the existing five. The kind is added +(`"blur"`, `"captionHighlight"`, …) joins the existing ones. The kind is added to `RegionKind` in `src/lib/ai-edition/document/timeline.ts` (`"zoom" | -"trim" | "annotation" | "speed" | "cameraFullscreen"`, `:18`); every site that -switches on it is checked below. Each path has been verified on this branch. +"trim" | "annotation" | "speed" | "cameraFullscreen" | "audio" | "shaderEffect"`); +every site that switches on it is checked below. `"shaderEffect"` is the latest +kind through every step. Its lane shows only while the project has an effect, since +an agent writes them, and the shell grows by its height (`EFFECT_LANE_EXPANSION_PX`) +([docs/shader-effects.md](../../docs/shader-effects.md), F). Line numbers refreshed +on 14/10/2026. 1. **Schema** — `src/lib/ai-edition/schema/index.ts`. Add a `xxxRegionSchema` Zod object that extends the modifier base (`{id, startMs, endMs, clipId?, sourceStartSec?, sourceEndSec?, …payload}`), and add the array to - `documentSchemaShape` next to `zoomRanges` and `annotations` (`:496-497`). - The v4→v5 preprocess at `:555-595` re-anchors every region kind — add the + `documentSchemaShape` next to `zoomRanges` and `annotations` (`:736-737`). + The v4→v5 preprocess at `:806-834` re-anchors every region kind — add the new array to the `anchor(...)` block so freshly-loaded v4 documents convert on parse. 2. **Identity** — `timelineMap.regionIdentityKey` already does the right thing: - `NON_IDENTITY_FIELDS` (`:122`) lists only position + provenance, so every + `NON_IDENTITY_FIELDS` (`:141`) lists only position + provenance, so every property of the new kind participates in identity automatically. Confirm with a test in `src/lib/ai-edition/timeline/timelineMap.test.ts`. 3. **Store action in `useTimeline`** — `src/lib/ai-edition/store/useTimeline.ts`. @@ -125,13 +129,13 @@ switches on it is checked below. Each path has been verified on this branch. `anchorRegionsWithDerivedMs` (`:134, :163, :278, :305, :329`), `replacePillSpan`, and `dropPillsByIds` (imports at `:23-28`); expose the array as `tl.xxxRegions` on the returned object. Add the new kind to - `RegionKind` (`document/timeline.ts:18`); add a `removeRegion` case - (`document/timeline.ts:591`) for batch / single deletes. + `RegionKind` (`document/timeline.ts:47`); add a `removeRegion` case + (`document/timeline.ts:1173`) for batch / single deletes. 4. **Lane in `V4Timeline`** — `src/components/ai-edition/v4/V4Timeline.tsx`. Compute the pills at the same call site as the four existing lanes - (`coalesceRegionsForRuler(tl.xxxRegions).map(...)` near `:463-511`), render + (`coalesceRegionsForRuler(tl.xxxRegions).map(...)` near `:743-779`), render them through `renderPills` inside a `
` - block (`:1504-1512`), and extend the `kind` union at `:334` so drag, + block (`:2285-2332`), and extend the `kind` union at `:599` so drag, resize, and delete handler switches route correctly. **Coordinates on the timeline canvas obey one rule**: position and size are @@ -147,13 +151,13 @@ switches on it is checked below. Each path has been verified on this branch. `PILL_SNAP_PX` and `CLIP_GUTTER_PX`, and the invariants in `V4Timeline.geometry.test.tsx`. 5. **Inspector selection pane** — `src/components/ai-edition/v4/FloatingInspector.tsx`. - The `SelectionPane` (`:444`) is the kind-switch site, not the facets; add - an `if (selection.kind === "xxx") { ... }` branch alongside `:513, :612, - :629, :956` that reads `tl.xxxRegions`, calls the store's `updateXxx*` + The `SelectionPane` (`:746`) is the kind-switch site, not the facets; add + an `if (selection.kind === "xxx") { ... }` branch alongside `:834, :955, + :972, :1278, :1295` that reads `tl.xxxRegions`, calls the store's `updateXxx*` methods, and renders a delete button matching the existing style. 6. **Preview path** — `src/components/ai-edition/Preview.tsx` mounts only zoom, annotation, speed, camera-fullscreen, and trim regions (see - `NewEditorShell.tsx` `:1185-1189`). Add the new array to `Preview`'s + `NewEditorShell.tsx` `:1617-1639`). Add the new array to `Preview`'s props, pass it through, and have the renderer consume it from `tl.xxxRegions`. 7. **Native preview scene** — diff --git a/technical-documentation/architecture/export-pipeline.md b/technical-documentation/architecture/export-pipeline.md index e43778772b..8d773d99f5 100644 --- a/technical-documentation/architecture/export-pipeline.md +++ b/technical-documentation/architecture/export-pipeline.md @@ -29,6 +29,8 @@ flowchart LR ## Segment loop +The walk itself is `walk_composited_timeline` ([`timeline_walk.rs`](../../crates/compositor/src/timeline_walk.rs)), shared by the MP4 and GIF exports. A third caller renders chosen frames without an export: `render_frames`, behind the addon's `renderFrames`, which MCP `getFrames` uses when a project has no current export. It walks the same timeline and composes only the frames asked for. The decoders skip the rest and catch up with a keyframe seek, which never changes the source frame shown (see [docs/shader-effects.md](../../docs/shader-effects.md), E.4). + The renderer hands the addon `exportMulti(clips, outPath, sceneJson, params)` through `compositorViewService.exportMulti` ([`compositorViewService.ts:425`](../../electron/native-bridge/services/compositorViewService.ts)). diff --git a/technical-documentation/architecture/mcp-server.md b/technical-documentation/architecture/mcp-server.md index 1fd28b93b5..e9ac925643 100644 --- a/technical-documentation/architecture/mcp-server.md +++ b/technical-documentation/architecture/mcp-server.md @@ -46,6 +46,10 @@ So a tool added to the agent appears over MCP with no further work, and the MCP - **The project tools**, see [The project and its media](#the-project-and-its-media). - MCP metadata: `readOnlyHint` for the reads (`!isMutatingTool`), `listProjects` and `createCheckpoint`, and `destructiveHint` for `replaceTimeline`, the three `remove*` agent tools, `removeMedia` and `restoreCheckpoint`. +### The effect contract + +The server's one resource, `openscreen://effects/contract` ([`effects-contract.ts`](../../electron/mcp/effects-contract.ts)), holds the full WGSL contract of `addEffect`: the inputs an effect reads, its parameters, the rules, the cost tiers and two examples. It is too long for a tool description and versioned with the contract; the instructions point to it. `effects-contract.test.ts` checks that it names every field of the compositor's prelude, and `shader_fx::tests::the_examples_of_the_mcp_contract_compile` compiles its examples. The code an agent writes is checked by `McpToolDeps.checkShaderEffect`, the compositor's `checkShaderEffect`, wired in `registerIpcHandlers` like `cameraSegmentation`; the in-app agent gets the same check through `ChatRunEnv`. + ### The project's look `setEditorSettings` ([`editor-settings-tool.ts`](../../electron/ai-edition/editor-settings-tool.ts)) writes every setting of the editor's look panes: background, frame, format, camera layout, cursor, `autoFocusAll` and `audioGainDb`. It takes a partial `EditorSettingsPatch` and applies it with `applySettingsPatch`, the editor's own write, plus what the panes do around a pick (a first fixed format fills the frame, an untouched camera roundness follows its shape, the camera framing is stored as size and pan). `setAspectRatio` is the same write for the format alone. @@ -171,6 +175,7 @@ Then both servers carry on alike: - `exportVideo` renders in a hidden `cli-export` runner, the same in both processes: in the app, `listSources` and `exportVideo` run the CLI's runners (`createRunnerHost` with `createRunnerWindow`), whose IPC the app registers on first use. The runner reads the project by id (`CliExportRequest.projectId`, the editor's own `aiEdition.get`), not by path: opening a file by path moves the main process's current project and recording-session slot, which in the app is the hand-off from the recorder to the editor. - `exportVideo` takes the Export dialog's options and choices: `quality` and `fps` (24, 30, 60) for an MP4, `gifFrameRate`, `gifSize` and `gifLoop` for a GIF, the dialog's defaults when left out. An option of the other format is refused rather than ignored. Hiding the desktop icons is not a `startRecording` option: the HUD has no such control, so it follows the user's Record mode setting on both servers. Nor are the CLI's `--auto-zoom` and `--audio`: zooms and a voiceover are edits of the project, and an export that added them would not be the project `getFrames` maps through. - `exportVideo` and `getFrames` read projects; they write none. Over HTTP their paths must be absolute. +- `getFrames` with a `projectId` reads the project's last export while it is current. With no export, or one the project changed after, it renders the frames from the project instead (`renderFrames` in the addon, through the export's own walk), so an agent checks an edit without exporting. See [docs/shader-effects.md](../../docs/shader-effects.md), E.4. - `createProject` makes a project with `DocumentService.createProject`, so it starts in the look new projects take (the preset marked for them, else the last edited project's), as the editor's New does. With a `videoPath` it lays the video out with `createProjectFromVideo`, the path a stdio take goes through: one clip, measured, and the zooms on its clicks when a pointer track sits beside it. Either way it returns a checkpoint, as `stopRecording` does, so the client can edit the closed project at once. It runs where the media tools run (in the CLI process for `openscreen mcp`): a new project is one no editor holds. **MCP edits** gates it. ## `openscreen mcp` and the running app diff --git a/technical-documentation/architecture/native-compositor.md b/technical-documentation/architecture/native-compositor.md index 6ce4e305ce..ebc6e21765 100644 --- a/technical-documentation/architecture/native-compositor.md +++ b/technical-documentation/architecture/native-compositor.md @@ -29,6 +29,7 @@ and without that flag the runtime corruption is silent. | [`crates/compositor/src/ffi.rs`](../../crates/compositor/src/ffi.rs) | bindgen-generated `libav*` bindings (ffmpeg 8.x headers) | | [`crates/compositor/src/compositor.rs`](../../crates/compositor/src/compositor_windows.rs) | HLSL compositor — every per-frame pass (`compose_frame`, background, screen, webcam, cursor, annotations, shadows, blur) | | [`crates/compositor/src/scene.rs`](../../crates/compositor/src/scene.rs) | the `Scene` struct parsed from the app's `SceneDescription` JSON | +| [`crates/compositor/src/shader_fx.rs`](../../crates/compositor/src/shader_fx.rs) | agent-written shader effects: the WGSL contract, validation and loop rule, cost tiers, parameter packing, background compilation (FXC on Windows, MSL and the Metal compiler on macOS; on Linux, `compositor_linux.rs` builds wgpu pipelines) | | [`crates/compositor/src/regions.rs`](../../crates/compositor/src/regions.rs) | zoom / speed / Full Camera regions — envelope shapes and per-frame state sampling | | [`crates/compositor/src/pipeline.rs`](../../crates/compositor/src/pipeline_windows.rs) | demux + `D3D11VA` decode + composite + AMF encode + mux (`run_c0` for decode/encode only, `run_composited` for the full path) | | [`crates/compositor/src/audio.rs`](../../crates/compositor/src/audio.rs) | per-clip audio decode, swresample → f32 planar 48 kHz stereo, libavfilter `atempo` speed stretch (WSOLA fallback), multi-track mix, AAC encoder | @@ -173,6 +174,60 @@ flowchart TB BG --> SS --> SV --> CU --> WS --> WV --> ANN ``` +### Shader effects + +An agent-written effect is WGSL that defines `fn effect(uv: vec2) -> vec4`, carried by the +scene (`Scene::shader_effects`) and applied by `compose_frame` as a full-frame pass. The design and +the contract are in [docs/shader-effects.md](../../docs/shader-effects.md); +[`shader_fx.rs`](../../crates/compositor/src/shader_fx.rs) holds everything that is not +backend-specific. + +- **The module.** The agent's code comes first, then the generated `FxParams` struct (its declared + parameters, bound at `b3`), then the contract-v1 prelude: `frame()`, `ctx` (`b2`), `fx_px`, + `fx_hash` and the `fx_vs`/`fx_fs` entry points. Errors therefore point at the agent's lines. +- **Validation** (`analyze`) is naga's, plus the loop rule: every loop is a `for` with literal + bounds, at most 4 096 iterations per pixel. An effect declares no binding beyond the prelude's + four. It also estimates the cost, in `frame()` reads per pixel, as an indicative tier (light, + medium, heavy, very heavy). Nothing is refused on cost. +- **Compilation never happens on the render thread.** `set_scene` starts one background compile per + new source. The cache is keyed by contract, code and parameter layout, and keeps the 64 most + recently used. + - **Windows:** naga to HLSL SM 5.0, then FXC; the device creates the shaders when the bytecode + lands. + - **macOS:** naga to MSL, then `new_library_with_source` and the pipeline state (RGBA8, no + blending), all on the background thread under its own autorelease pool. The bindings are + mapped explicitly (`shader_fx::to_msl`): texture 0, the sampler inlined as a `constexpr` + (linear, clamp to edge, like `sampNV`), `ctx` and `params` at buffers 2 and 3, uploaded with + `set_fragment_bytes`. naga's `force_loop_bounding` is off: `analyze` has already proved every + loop bounded, and the guard cost 7 to 10 % of the pass on an M1. + - **Linux:** the WGSL goes to wgpu as it is. The thread builds the shader module and the + pipeline, against an explicit layout of the prelude's four bindings. wgpu panics on a + validation error, since no `on_uncaptured_error` is set: `analyze` runs first so that none + can happen, and a panic would only kill that thread, leaving the effect undrawn. +- **Blocking.** The export compositor waits for a pending compile and never draws a frame without + its effect. The live view (`set_shader_fx_blocking(false)`) skips the effect until it lands, then + recomposes the paused frame (`shader_fx_landed`). +- **Targets.** `image` effects run before the annotations and captions. `text` effects run on a + transparent text layer, which is then drawn over the image. `both` effects run last. The document holds + one effect at a time; a hand-written scene that overlaps several applies them in its order. Each pass copies its destination into + `ann_copy` and repaints it with blending off. On Metal every pass is encoded in the frame's + command buffer: a blit into `ann_copy`, then a render pass that loads its target (an effect may + `discard`). `draw_annotations` takes the target it opens its pass on, which is how the text layer + receives the annotations; the layer is cleared in a pass of its own, since `draw_annotations` + opens none when nothing is visible. +- **`ctx` comes from the frame geometry** and is in output uv: the pointer and the last four clicks + (`effect_pointer`), the footage rect (`FrameGeometry::footage_quad`, bounding box under a tilt), + the camera rect, the region's progress (over the whole pill in programme time when the app sends + `programmeStartSec`/`programmeEndSec`), and programme time. +- **A hidden cursor still feeds effects.** `effect_pointer` places the pointer and the clicks as + `plan_cursor` places the cursor, but whether or not the cursor is drawn: with *Show cursor* off, + an effect still follows the mouse and still sees each click. Only the recording hiding the + pointer, or a zoom leaving it out of frame, takes the pointer away. + +naga stays at 24. From 25 on, its HLSL output declares samplers in a D3D12 heap +(`register(s0, space0)`), which FXC rejects for `ps_5_0` (X3721). +`shader_fx::tests::a_valid_effect_becomes_hlsl_that_d3d11_accepts` guards that boundary. + ## Scene contract The app hands the native compositor a flat JSON description once per document diff --git a/technical-documentation/architecture/timeline-model.md b/technical-documentation/architecture/timeline-model.md index f8f293eb0a..1bcd379809 100644 --- a/technical-documentation/architecture/timeline-model.md +++ b/technical-documentation/architecture/timeline-model.md @@ -33,7 +33,7 @@ existing. ## Clip-anchored modifiers -Every region (zoom / speed / annotation / cameraFullscreen) is stored as one or more +Every region (zoom / speed / annotation / cameraFullscreen / shader effect) is stored as one or more **clip-anchored fragments** — `{clipId, sourceStartSec, sourceEndSec, …payload}` — keyed to the clip it lives on, in that clip's source media time. A region the user drew across a clip boundary is stored as one fragment per covered clip; the fragments carry diff --git a/technical-documentation/engineering/rendering-performance.md b/technical-documentation/engineering/rendering-performance.md index 93c2387c4d..78c68bf18e 100644 --- a/technical-documentation/engineering/rendering-performance.md +++ b/technical-documentation/engineering/rendering-performance.md @@ -434,6 +434,50 @@ The editor preview was driven over CDP on the same two builds (not computer-use: **The output did not change.** Every row above exports a byte-identical GIF, checked by hash. That needed one fix first: the median cut sorted `HashMap` entries with a stable sort, so ties kept the map's random per-process order and no two exports of the same project were byte-identical. +## Shader effects — 2026-10-14 + +An agent-written effect (`crates/compositor/src/shader_fx.rs`, [docs/shader-effects.md](../../docs/shader-effects.md)) adds one full-frame pass per active region: a copy of the render target into `ann_copy`, then the effect. Measured with `compositor_windows::tests::shader_fx_tier_calibration` (`OPENSCREEN_FX_BENCH=1`, release) on the RTX 4070 Ti: the extra time per composed frame of an effect that reads `frame()` N times, best of five runs of 200 frames. + +| `frame()` reads | Tier | 1080p | 4K | +|---:|---|---:|---:| +| 1 | light | +0.030 ms | +0.138 ms | +| 8 | light | +0.061 ms | +0.211 ms | +| 32 | medium | +0.137 ms | +0.606 ms | +| 128 | heavy | +0.539 ms | +2.365 ms | +| 512 | very heavy | +2.133 ms | +8.805 ms | + +- **The cost is linear in the reads,** about 4 µs per read at 1080p and 17 µs at 4K, over a fixed 0.03 ms (1080p) for the copy and the pass. It is the texture bandwidth that pays, which is why the tiers count reads. +- **The export does not notice a light effect:** 180 frames at 1752×1080 took 520 ms without, 523 ms with the spike's effect, and 521 ms with a heavier one. The preview held the source's 30 fps either way. +- **On lavapipe, the Linux CPU fallback,** the same bench (`compositor_linux::tests::shader_fx_tier_calibration`, release, best of three runs of 5 frames, WSL on 16 threads) gives about 1.4 ms per read at 1080p, 350 times the RTX: + +| `frame()` reads | Tier | 1080p | 4K | +|---:|---|---:|---:| +| none (the frame alone) | | 7.5 ms | 29.5 ms | +| 1 | light | +3.2 ms | +13.4 ms | +| 8 | light | +11.2 ms | +44.4 ms | +| 32 | medium | +42.1 ms | +159 ms | +| 128 | heavy | +234 ms | +958 ms | +| 512 | very heavy | +1 035 ms | +5 068 ms | + +- **The tiers keep their order on lavapipe,** but without a GPU a light effect already doubles the frame. The pipeline is ready 18 ms after `set_scene` there, module and driver compile included. + +**Apple M1 (integrated GPU, Metal), the same bench** (`compositor_macos::tests::shader_fx_tier_calibration`, macOS 26.5, on mains power). Composing a frame without an effect takes 2.42 ms at 1080p and 8.98 ms at 4K. + +| `frame()` reads | Tier | 1080p | 4K | +|---:|---|---:|---:| +| 1 | light | +0.413 ms | +1.225 ms | +| 8 | light | +1.260 ms | +4.575 ms | +| 32 | medium | +4.256 ms | +16.310 ms | +| 128 | heavy | +16.240 ms | +62.903 ms | +| 512 | very heavy | +49.143 ms | +201.583 ms | + +- **About 0.12 ms per read at 1080p and 0.5 ms at 4K,** over a fixed 0.3 ms: thirty times the RTX. The fixed part is the copy into `ann_copy` and the full-frame pass, which are bandwidth on a 68 GB/s unified memory. +- **The reads dominate here too.** The bench effect computes a `sin` and a `cos` per read; with integer offsets instead, 128 reads cost 10.5 ms rather than 16.2. +- **naga's `force_loop_bounding` is off** (`shader_fx::to_msl`): its guard, a `volatile` read per iteration, cost 7 to 10 % of every row above. The validation has already proved each loop bounded. +- **Compiling is ten times slower than FXC.** naga, `new_library_with_source` and the pipeline state take 120 ms in the median for a source Metal has not seen (the C.6 ripple), and 160 to 375 ms for the first one in a process. A source Metal has seen comes back in 0.4 ms in the same process, 3 ms in another. All of it runs on the background thread; the preview shows the effect once it lands. An agent's `addEffect` took 152 ms through the `mcp` CLI, the compile included; `checkShaderEffect` runs on the libuv pool, off the main thread. +- **The export does not notice an effect:** 180 frames at 1752×1080 (`exportMulti`, three alternated runs after a warm-up) took 2469 ms without, 2428 ms with the ripple over the whole video, and 2375 ms with the spike's heavier effect. The M1 export is encoder-bound. The live view held 29.7 fps against a 30 fps source with and without the effect. +- **Not measured yet:** the Radeon 610M. + ## How we got here — the WebCodecs trail > **This section is history.** It records the measurements that killed the browser-based export pipeline and motivated the native one. The code it describes is **gone**: `src/lib/exporter/videoExporter.ts`, `src/bench/runBench.ts` and the `npm run bench:export` script were deleted with the web MP4 pipeline. It is kept because it is the evidence for [why the compositor, not the encoder, was the wall](#the-wall-is-the-compositor) — which is the entire reason `crates/compositor/` exists — and because the [measurement hazards](#measurement-hazards) it uncovered still apply to any new benchmark here. diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 551fc5bfd3..b6a81f3412 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -968,3 +968,4 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-14 | `main` @ `2d98601c4` (includes c541fffaa, the recording-state fix), CI artifact `openscreen-mac-arm64` from build run 38598557320, DMG signed and notarized (`spctl`: accepted, Notarized Developer ID, M4LK7C6S84), installed over the build of the row above; About `2.0.0 (dmg)` | macOS 26.5 (25F71), Mac mini M1 (arm64), one 3840×2160 display at 2×, no camera (MDM) | **Pass, scoped to defect 1 of the row above** | Real OS input through computer-use, launched with `open -n --env OPENSCREEN_MAC_SOURCE_PICKER=legacy` (`-n` because another session's `Openscreen mcp` was running, #1214); MCP calls from an SDK StreamableHTTP client to the in-app server. **In-app `startRecording` (c541fffaa):** on a Finder window, with the microphone and the editable cursor requested while the bar had both off, the call returns in 4.9 s (was a 120 s timeout and a lost take); the editor closes, the bar records with the requested settings and one capture helper runs. `stopRecording` returns in 2.7 s with the `projectId` of the project the editor opens on, in Edit, one clip, no zoom (Record mode's Auto-zoom after recording is Off), take 1840×872, 26.9 s. The bar's microphone, cursor mode and last source are back to their values from before the take. `addZoom` on it shows live and Cmd+Z / Cmd+Shift+Z take it back and forth; `exportVideo` at 30 fps gives 806 frames for 26.87 s as predicted, and `getFrames` shows the zoom at 8 s. **Skipped:** the refusal while the editor has unsaved changes (the drag meant to hold the editor dirty never moved the pill, so the accepted `startRecording` that followed proves nothing either way); zooms after an in-app take with Auto-zoom On; the defects of the row above other than defect 1 (#1214, #1215, #1216 still open). | | 2026-10-14 | `main` @ `61fa43812` (includes #1227, #1228, #1229, the fixes for #1215, #1214, #1216), CI artifact `openscreen-mac-arm64` from build run 38615516865, DMG signed and notarized (`spctl`: accepted, Notarized Developer ID, M4LK7C6S84); `app.asar` and the capture helper carry the new code; About `2.0.0 (dmg)` | macOS 26.5 (25F71), Mac mini M1 (arm64), one 3840×2160 display at 2× | **Pass on the three fixes, 1 new defect, 1 leftover** | Real OS input through computer-use for the Finder launch; MCP calls from SDK clients (stdio to `Openscreen mcp`, StreamableHTTP to the in-app server); numbers are ffprobe/ffmpeg on the files. Every OpenScreen process was quit first, another session's `Openscreen mcp` of an older build included, so no old binary could take the launch. **#1214:** with an `Openscreen mcp` client session open and no GUI, `open /Applications/Openscreen.app` brings the recording bar up in about 1 s (was: exit 0 and nothing); two quick `open`s with the GUI running leave one GUI; after a quit, a double-click on OpenScreen in Finder's Applications opens the bar. **#1215:** a GIF export (`gifSize: "small"`, 778×480) read by `getFrames` through the stdio server by `projectId` (1, 6, 11 s on the ruler) and by `path`, and through the in-app server by `projectId`, gives frames of distinct moments (was "media error 4"). **#1216:** a window take of a Finder window with `systemAudio: true` while `afplay` plays speech reads −10.0 dB mean through the stdio server and −10.0 dB through the in-app server under `legacy` (was −91 dB), video 1840×872, the window only. **New defect (P1 for the in-app server on macOS 15.2+):** on a default launch (no `OPENSCREEN_MAC_SOURCE_PICKER`), in-app `startRecording` with `window` ignores the window and opens Apple's picker ("Share a Window or Entire Screen"), which waits for a click, so the call fails after 120 s with "capture had not started"; the picker then stays on screen after the timeout, and even after the app quits, until someone presses Cancel. The stdio server and a `legacy` launch are not affected (4.3 s to start). **Leftover from #1214:** the `Openscreen mcp` process turns from `UIElement` to `Foreground` within 0.3 s of the launch LaunchServices hands it, and keeps a second OpenScreen icon in the Dock; a click on it is an `activate`, so it opens the GUI rather than nothing. A bare hidden Electron 41 with the same handler, with or without a hidden window, stays `UIElement`, so the cause is in the app's CLI process and not found yet. **Not covered:** the two failure paths of #1229's audio stream (ScreenCaptureKit was not made to refuse it). | | 2026-10-14 | `main` @ `4f4bb997c` (includes #1237 and #1239, the fixes for #1232 and #1233), CI artifact `openscreen-mac-arm64` from build run 38627769728, DMG signed and notarized (`spctl`: accepted, Notarized Developer ID, M4LK7C6S84); About `2.0.0 (dmg)` | macOS 26.5 (25F71), Mac mini M1 (arm64), one 3840×2160 display at 2× | **Pass, scoped to #1232 and #1233** | Every OpenScreen process quit before the install, other sessions' `Openscreen mcp` included; MCP calls from SDK clients; numbers are ffprobe/ffmpeg on the files. **#1233:** with an `Openscreen mcp` client session open and no GUI, `open /Applications/Openscreen.app` starts the GUI and the `mcp` process stays `UIElement` at 0.5, 2.5 and 8.5 s (was `Foreground` within 0.3 s, with its own Dock icon). The two OpenScreen icons left in the Dock carry no running dot once the GUI quits: they are the Dock's recent-apps entries, not processes. **#1232:** on a default launch (no `OPENSCREEN_MAC_SOURCE_PICKER`, so Apple's picker owns the bar's sources), in-app `startRecording` with `window` and `systemAudio: true` returns in 5.0 s with no picker window on screen (was a 120 s timeout and a picker left behind); the take is the window only, 1840×872, system audio at −10.3 dB, and `stopRecording` returns in 2.7 s. **Not covered:** the refusal message when Screen Recording is denied (the grant was not revoked on this Mac). | +| 2026-10-14 | dev, local branch `test/shader-fx-macos-e2e`: `feat/shader-effects-macos` (the Metal pass) with #1202 merged in locally, never pushed, plus a local-only line that writes each export's scene to a file. Compositor addon rebuilt from that tree (string probe `no Metal device` HIT, control `wallpaper image` HIT), `npm run build-vite` | macOS 26.5 (25F71), Mac mini M1, on mains power | Partial: agent-written shader effects only, pass | Headless, so no computer-use: `electron . --user-data-dir= mcp` over stdio, and the addon loaded in Node for the live view (`createView`, `setScene`, `setParam` as `pushAllNativeParams` does, `presentTime`, `readFrame`). A 6 s 1920×1080 test pattern with a hand-written `.cursor.json` of three clicks. **Agent loop:** `addEffect` with a typo refused in 6 ms at the agent's line 10; the C.6 ripple accepted in 152 ms, real Metal compile included, `validated: true`, tier light; `exportVideo` then `getFrames` show the ripple after each click and nothing at 0.5 s. **Preview against export** (BT.709): 3.1/255 over the whole frame with the effect, 3.0 without; on the pixels the effect changes, 13 to 16/255 against the export with the effect and 14 to 16 against the one without on the same pixels with no effect, which is the codec on sharp edges (two exports of the same project decode to identical pixels). The live view held 29.7 fps against the 30 fps source with and without. **Export** of 180 frames at 1752×1080: 2469 ms without, 2428 with the ripple, 2375 with a heavier effect. **Driver note:** without the `setParam` calls the editor makes, the headless view draws the cursor at its default size, which reads as a preview/export mismatch. Not run: real capture, the editor UI (effects have no track until the UI PR), the in-app agent, a `text`-target effect over a real caption (`an_annotation_stays_out_of_image_effects_and_inside_text_effects` covers the routing of a figure annotation on the Metal device, not a caption). | diff --git a/website/docs/ai-editing.md b/website/docs/ai-editing.md index e5188ac3d1..587191f21c 100644 --- a/website/docs/ai-editing.md +++ b/website/docs/ai-editing.md @@ -41,7 +41,7 @@ The ChatGPT and GitHub Copilot sign-in options were **removed in 1.8.0**. They w ## Using the agent -Describe the edit in plain language — "cut the dead air in the intro", "zoom in when I open the terminal". The agent works through real, undoable timeline operations, not a re-render: it can add and adjust trims, zooms, speed regions, annotations (text, arrows, and blurs that hide private details on screen), imported audio tracks with their fades, and Full Camera segments, add automatic zooms on your clicks, edit a clip's in/out points and crop, place another source's footage, duplicate, reorder or remove clips, and read the transcript to find what you're referring to. It also sets the look of the video: background, padding, rounded corners, shadow, device frame, format, the camera layout, the cursor and the master volume. It can also turn the [captions](./captions.md) on, style and place them, and translate them into another language itself. +Describe the edit in plain language — "cut the dead air in the intro", "zoom in when I open the terminal". The agent works through real, undoable timeline operations, not a re-render: it can add and adjust trims, zooms, speed regions, annotations (text, arrows, and blurs that hide private details on screen), imported audio tracks with their fades, and Full Camera segments, add automatic zooms on your clicks, edit a clip's in/out points and crop, place another source's footage, duplicate, reorder or remove clips, and read the transcript to find what you're referring to. It also sets the look of the video: background, padding, rounded corners, shadow, device frame, format, the camera layout, the cursor and the master volume. It can also turn the [captions](./captions.md) on, style and place them, and translate them into another language itself. And it can write a visual effect the settings don't offer: see [Custom effects](./effects.md). The panel around it: diff --git a/website/docs/editing-timeline.md b/website/docs/editing-timeline.md index 8bb042ac7f..a1eabdd0ff 100644 --- a/website/docs/editing-timeline.md +++ b/website/docs/editing-timeline.md @@ -78,6 +78,10 @@ A row of preset buttons (0.5×, 1×, 1.5×, 2×, 4×) and a free numeric field f A span where the webcam fills the frame instead of sitting in its layout box — useful for a talking-head intro in the middle of a screen recording. Only meaningful when the recording has a webcam track. +### Effect regions + +A visual effect an AI agent wrote for the project. Its row shows under the Full Camera segments once the project has one: see [Custom effects](./effects.md#in-the-editor). + ### Annotations Four types, picked from the **Type** row in the inspector. Switching type keeps the region's span and its place in the picture. diff --git a/website/docs/effects.md b/website/docs/effects.md new file mode 100644 index 0000000000..206a60f1a3 --- /dev/null +++ b/website/docs/effects.md @@ -0,0 +1,64 @@ +--- +id: effects +title: Custom effects +description: "Ask an AI agent for a visual effect the settings don't offer. It writes a GPU shader that OpenScreen draws in the preview and in every export." +keywords: + - custom video effects + - shader effects + - WGSL + - AI video effects + - GPU effects +--- + +# Custom effects + +When the settings don't have the look you want, ask an agent for it: a ripple from each click, a vignette that breathes, a fade to black over the last seconds. The agent writes the effect as a small GPU shader, and OpenScreen draws it in the preview and in every export, MP4 and GIF, on Windows, macOS and Linux. + +Effects come from an agent: the built-in chat or an MCP client such as Claude Code or Codex (see [AI editing](./ai-editing.md)). Once an effect is in the project, you adjust it yourself in the editor. + +## Asking for an effect + +Describe what you want to see, and when: "a ripple on every click", "fade to black over the last two seconds, captions included". The agent writes the effect, places it on the timeline, and can look at the result with `getFrames` without exporting the video. + +An effect knows where the cursor is and where each click landed, where the recording and the webcam sit in the video after zooms and layout, and the time. So a halo can follow the mouse, and a ripple can start from each click, wherever the zoom has moved it. + +## In the editor + +Effects get their own row on the timeline, under the Full Camera segments, once the project has one. Each block carries the effect's name. Move it, resize it, copy and paste it or delete it like any other region, and `Ctrl/Cmd + Z` undoes the change. One effect plays at a time: a paste onto another effect is refused. + +Click a block to open its inspector: + +- **A control for each setting** the agent gave the effect: a slider, a switch, a color or a point. +- **Applies to**, where the effect is drawn: + - **Image**, the default: the recording, the webcam, the cursor and the background, under your annotations and captions, which stay legible. + - **Text**: the annotations and captions only. + - **Both**: the whole frame, text included, for a fade to black that also covers the captions. +- **Cost**: **Light**, **Medium**, **Heavy** or **Very heavy** (see [below](#cost)). +- **Code**: the shader, read-only, with **Copy code**. To change it, ask the agent. + +A copied effect pasted into another project brings its code along. + +## Cost + +The cost is estimated from the code, by counting how many times the effect reads the picture for each pixel. The same code gets the same tier on every computer. Nothing is refused for its cost: the heavier the effect, the shorter the stretch it suits. + +| Tier | Typical effects | Suits | +|---|---|---| +| **Light** | vignette, color grade, click ripple | the whole video | +| **Medium** | chromatic aberration, small blur, glitch | a few tens of seconds | +| **Heavy** | strong blur, bokeh | a transition, a highlight | +| **Very heavy** | raymarching, simulations | a few seconds: the export stays exact but slower, and the preview can slow down on a small GPU | + +## If an effect doesn't compile + +An agent's code is compiled before the effect is accepted. Code that fails is refused with the compiler's error, and the project doesn't change. + +A project opened on another computer can still meet a compiler that disagrees. The effect is then not drawn, its inspector shows **Doesn't compile on this computer** with the error, and the export goes on without it. + +## Projects from someone else + +A project carries the code of its effects, so opening someone else's project runs code you didn't write. That code runs only on the GPU, with no access to your files, the network or the app's memory. + +## For agent authors + +The tools are `addEffect`, `setEffect`, `setEffectDefinition` and `getEffect`, and `removeModifier` takes an effect off. The full contract, the `effect(uv)` function, what it can read and its rules, is the MCP resource `openscreen://effects/contract`. diff --git a/website/i18n/TRANSLATING.md b/website/i18n/TRANSLATING.md index 1920df10de..a0f714330e 100644 --- a/website/i18n/TRANSLATING.md +++ b/website/i18n/TRANSLATING.md @@ -21,13 +21,13 @@ Replace `` with `fr`, `es`, `pt-BR`, `ja`, `zh-CN`, `zh-TW` or `de`. 3. **`i18n//docusaurus-plugin-content-docs/current.json`**: sidebar category and link labels. `version.label` is not shown. 4. **`i18n//docusaurus-plugin-content-docs/current/`**: one translated copy of each doc, same file name and folder: - `intro.md`, `installation.md`, `quick-start.md`, `faq.md` - - `recording.md`, `media-library.md`, `editing-timeline.md`, `captions.md`, `ai-editing.md`, `export.md`, `cli.md` + - `recording.md`, `media-library.md`, `editing-timeline.md`, `captions.md`, `ai-editing.md`, `effects.md`, `export.md`, `cli.md` - `guides/product-demo-video.md` Start from a copy of `website/docs/`. In the front matter, translate `title`, `description`, `sidebar_label` and `keywords`. Leave `id`, `slug` and `sidebar_position` as they are. 5. **`i18n//docusaurus-plugin-content-pages/`**: one translated copy of each `website/src/pages/**/*.mdx`, same path (`alternatives/screen-studio.mdx`, `screen-recorder-linux.mdx`…). Translate `title`, `description` and `keywords` in the front matter. Prices, plan limits and "checked September 2026" dates are facts: copy them as they are, in your language's number and currency format. Keep the Sources list's URLs, and translate the trademark note. -**All 12 docs of a locale land together.** The docs link to each other with relative paths (`./captions.md`), and Docusaurus only resolves those between files in the same folder: with some docs translated and others not, the build fails on broken links. A doc missing from a finished locale would also be published under `//docs/` in English, marked as your language. +**All 13 docs of a locale land together.** The docs link to each other with relative paths (`./captions.md`), and Docusaurus only resolves those between files in the same folder: with some docs translated and others not, the build fails on broken links. A doc missing from a finished locale would also be published under `//docs/` in English, marked as your language. ## Keep every heading's English anchor diff --git a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md index 39c5936703..e1f6e6be53 100644 --- a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ Die Anmeldeoptionen für ChatGPT und GitHub Copilot wurden **in 1.8.0 entfernt** ## Den Agenten nutzen {#using-the-agent} -Beschreibe die Änderung in Alltagssprache, etwa „schneide die Stille im Intro heraus“ oder „zoome hinein, wenn ich das Terminal öffne“. Der Agent arbeitet mit echten, rückgängig machbaren Operationen auf der Zeitleiste, nicht mit einem neuen Rendering: Er kann Schnitte, Zooms, Geschwindigkeitsbereiche, Annotationen (Text, Pfeile und Unschärfen, die private Details auf dem Bildschirm verbergen), importierte Audiospuren samt Ein- und Ausblendung und Full-Camera-Segmente hinzufügen und anpassen, automatische Zooms auf deine Klicks setzen, Start- und Endpunkte und den Zuschnitt eines Clips ändern, Material einer anderen Quelle platzieren, Clips duplizieren, umsortieren oder entfernen und das Transkript lesen, um herauszufinden, was du meinst. Außerdem stellt er das Aussehen des Videos ein: Hintergrund, Innenabstand, abgerundete Ecken, Schatten, Geräterahmen, Format, das Kamera-Layout, den Cursor und die Gesamtlautstärke. Er kann außerdem die [Untertitel](./captions.md) einschalten, gestalten und platzieren und sie selbst in eine andere Sprache übersetzen. +Beschreibe die Änderung in Alltagssprache, etwa „schneide die Stille im Intro heraus“ oder „zoome hinein, wenn ich das Terminal öffne“. Der Agent arbeitet mit echten, rückgängig machbaren Operationen auf der Zeitleiste, nicht mit einem neuen Rendering: Er kann Schnitte, Zooms, Geschwindigkeitsbereiche, Annotationen (Text, Pfeile und Unschärfen, die private Details auf dem Bildschirm verbergen), importierte Audiospuren samt Ein- und Ausblendung und Full-Camera-Segmente hinzufügen und anpassen, automatische Zooms auf deine Klicks setzen, Start- und Endpunkte und den Zuschnitt eines Clips ändern, Material einer anderen Quelle platzieren, Clips duplizieren, umsortieren oder entfernen und das Transkript lesen, um herauszufinden, was du meinst. Außerdem stellt er das Aussehen des Videos ein: Hintergrund, Innenabstand, abgerundete Ecken, Schatten, Geräterahmen, Format, das Kamera-Layout, den Cursor und die Gesamtlautstärke. Er kann außerdem die [Untertitel](./captions.md) einschalten, gestalten und platzieren und sie selbst in eine andere Sprache übersetzen. Und er kann einen visuellen Effekt schreiben, den die Einstellungen nicht bieten: siehe [Eigene Effekte](./effects.md). Der Bereich drumherum: diff --git a/website/i18n/de/docusaurus-plugin-content-docs/current/editing-timeline.md b/website/i18n/de/docusaurus-plugin-content-docs/current/editing-timeline.md index 6c6fefdf3f..a4428c3382 100644 --- a/website/i18n/de/docusaurus-plugin-content-docs/current/editing-timeline.md +++ b/website/i18n/de/docusaurus-plugin-content-docs/current/editing-timeline.md @@ -78,6 +78,10 @@ Eine Reihe von Vorgabe-Schaltflächen (0.5×, 1×, 1.5×, 2×, 4×) und ein frei Ein Abschnitt, in dem die Webcam das Bild füllt, statt in ihrem Layout-Rahmen zu sitzen. Praktisch für ein Intro mit dir vor der Kamera mitten in einer Bildschirmaufnahme. Nur sinnvoll, wenn die Aufnahme eine Webcam-Spur hat. +### Effektbereiche {#effect-regions} + +Ein visueller Effekt, den ein KI-Agent für das Projekt geschrieben hat. Seine Reihe erscheint unter den Full-Camera-Segmenten, sobald das Projekt einen enthält: siehe [Eigene Effekte](./effects.md#in-the-editor). + ### Annotationen {#annotations} Vier Arten, wählbar in der Zeile **Type** im Inspektor. Beim Umschalten bleiben die Zeitspanne des Bereichs und seine Stelle im Bild erhalten. diff --git a/website/i18n/de/docusaurus-plugin-content-docs/current/effects.md b/website/i18n/de/docusaurus-plugin-content-docs/current/effects.md new file mode 100644 index 0000000000..c16fa7f4e9 --- /dev/null +++ b/website/i18n/de/docusaurus-plugin-content-docs/current/effects.md @@ -0,0 +1,64 @@ +--- +id: effects +title: Eigene Effekte +description: "Bitte einen KI-Agenten um einen visuellen Effekt, den die Einstellungen nicht bieten. Er schreibt einen GPU-Shader, den OpenScreen in der Vorschau und in jedem Export zeichnet." +keywords: + - eigene Videoeffekte + - Shader-Effekte + - WGSL + - KI-Videoeffekte + - GPU-Effekte +--- + +# Eigene Effekte + +Wenn die Einstellungen den Look nicht haben, den du suchst, bitte einen Agenten darum: eine Welle bei jedem Klick, eine Vignette, die atmet, eine Abblende ins Schwarze in den letzten Sekunden. Der Agent schreibt den Effekt als kleinen GPU-Shader, und OpenScreen zeichnet ihn in der Vorschau und in jedem Export, MP4 wie GIF, unter Windows, macOS und Linux. + +Effekte kommen von einem Agenten: dem eingebauten Chat oder einem MCP-Client wie Claude Code oder Codex (siehe [KI-Bearbeitung](./ai-editing.md)). Sobald ein Effekt im Projekt ist, stellst du ihn selbst im Editor ein. + +## Einen Effekt anfragen {#asking-for-an-effect} + +Beschreibe, was du sehen willst, und wann: „eine Welle bei jedem Klick“, „Abblende ins Schwarze über die letzten zwei Sekunden, Untertitel inklusive“. Der Agent schreibt den Effekt, legt ihn auf die Zeitleiste und kann sich das Ergebnis mit `getFrames` ansehen, ohne das Video zu exportieren. + +Ein Effekt weiß, wo der Cursor ist und wo jeder Klick gelandet ist, wo die Aufnahme und die Webcam nach Zooms und Layout im Video sitzen, und welcher Zeitpunkt gerade ist. So kann ein Lichthof der Maus folgen, und eine Welle kann von jedem Klick ausgehen, wohin der Zoom ihn auch verschoben hat. + +## Im Editor {#in-the-editor} + +Effekte bekommen eine eigene Reihe auf der Zeitleiste, unter den Full-Camera-Segmenten, sobald das Projekt einen enthält. Jeder Block trägt den Namen des Effekts. Verschiebe ihn, ändere seine Größe, kopiere und füge ihn ein oder lösche ihn wie jeden anderen Bereich, und `Ctrl/Cmd + Z` macht die Änderung rückgängig. Es läuft immer nur ein Effekt: Einfügen auf einen anderen Effekt wird abgelehnt. + +Klicke auf einen Block, um seinen Inspektor zu öffnen: + +- **Ein Regler für jede Einstellung**, die der Agent dem Effekt gegeben hat: ein Schieberegler, ein Schalter, eine Farbe oder ein Punkt. +- **Gilt für**, also wo der Effekt gezeichnet wird: + - **Bild**, die Voreinstellung: die Aufnahme, die Webcam, der Cursor und der Hintergrund, unter deinen Annotationen und Untertiteln, die lesbar bleiben. + - **Texte**: nur die Annotationen und Untertitel. + - **Beides**: das ganze Bild, Text inklusive, für eine Abblende ins Schwarze, die auch die Untertitel bedeckt. +- **Kosten**: **Leicht**, **Mittel**, **Schwer** oder **Sehr schwer** (siehe [unten](#cost)). +- **Code**: der Shader, schreibgeschützt, mit **Code kopieren**. Um ihn zu ändern, bitte den Agenten darum. + +Ein kopierter Effekt, der in ein anderes Projekt eingefügt wird, bringt seinen Code mit. + +## Kosten {#cost} + +Die Kosten werden aus dem Code geschätzt: Gezählt wird, wie oft der Effekt das Bild pro Pixel liest. Derselbe Code bekommt auf jedem Computer dieselbe Stufe. Nichts wird wegen seiner Kosten abgelehnt: Je schwerer der Effekt, desto kürzer der Abschnitt, der zu ihm passt. + +| Stufe | Typische Effekte | Passt für | +|---|---|---| +| **Leicht** | Vignette, Farbkorrektur, Welle beim Klick | das ganze Video | +| **Mittel** | chromatische Aberration, leichte Unschärfe, Glitch | einige Dutzend Sekunden | +| **Schwer** | starke Unschärfe, Bokeh | einen Übergang, einen Höhepunkt | +| **Sehr schwer** | Raymarching, Simulationen | wenige Sekunden: Der Export bleibt exakt, aber langsamer, und die Vorschau kann auf einer kleinen GPU langsamer werden | + +## Wenn ein Effekt nicht kompiliert {#if-an-effect-doesnt-compile} + +Der Code eines Agenten wird kompiliert, bevor der Effekt angenommen wird. Code, der scheitert, wird mit dem Fehler des Compilers abgelehnt, und das Projekt ändert sich nicht. + +Ein Projekt, das auf einem anderen Computer geöffnet wird, kann trotzdem auf einen Compiler treffen, der anderer Meinung ist. Der Effekt wird dann nicht gezeichnet, sein Inspektor zeigt **Lässt sich auf diesem Computer nicht kompilieren** mit dem Fehler, und der Export läuft ohne ihn weiter. + +## Projekte von anderen {#projects-from-someone-else} + +Ein Projekt trägt den Code seiner Effekte: Öffnest du das Projekt eines anderen, führst du also Code aus, den du nicht geschrieben hast. Dieser Code läuft nur auf der GPU, ohne Zugriff auf deine Dateien, das Netzwerk oder den Speicher der App. + +## Für Agent-Entwickler {#for-agent-authors} + +Die Werkzeuge sind `addEffect`, `setEffect`, `setEffectDefinition` und `getEffect`, und `removeModifier` entfernt einen Effekt. Der vollständige Vertrag, also die Funktion `effect(uv)`, was sie lesen kann und ihre Regeln, ist die MCP-Ressource `openscreen://effects/contract`. diff --git a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md index aa9d101cf9..991cc24e1e 100644 --- a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ Las opciones de inicio de sesión con ChatGPT y GitHub Copilot se **eliminaron e ## Usar el agente {#using-the-agent} -Describe la edición con tus propias palabras: "corta el tiempo muerto de la introducción", "haz zoom cuando abro la terminal". El agente trabaja con operaciones reales de la línea de tiempo, que se pueden deshacer, y no con un nuevo renderizado: puede agregar y ajustar recortes, zooms, regiones de velocidad, anotaciones (texto, flechas y desenfoques que ocultan datos privados en pantalla), pistas de audio importadas con sus fundidos y segmentos de cámara a pantalla completa, agregar zooms automáticos en tus clics, editar los puntos de entrada y salida y el recorte de un clip, colocar el material de otra fuente, duplicar, reordenar o quitar clips, y leer la transcripción para encontrar aquello a lo que te refieres. También ajusta el aspecto del vídeo: fondo, margen, esquinas redondeadas, sombra, marco de dispositivo, formato, la disposición de la cámara, el cursor y el volumen general. También puede activar los [subtítulos](./captions.md), darles estilo y colocarlos, y traducirlos él mismo a otro idioma. +Describe la edición con tus propias palabras: "corta el tiempo muerto de la introducción", "haz zoom cuando abro la terminal". El agente trabaja con operaciones reales de la línea de tiempo, que se pueden deshacer, y no con un nuevo renderizado: puede agregar y ajustar recortes, zooms, regiones de velocidad, anotaciones (texto, flechas y desenfoques que ocultan datos privados en pantalla), pistas de audio importadas con sus fundidos y segmentos de cámara a pantalla completa, agregar zooms automáticos en tus clics, editar los puntos de entrada y salida y el recorte de un clip, colocar el material de otra fuente, duplicar, reordenar o quitar clips, y leer la transcripción para encontrar aquello a lo que te refieres. También ajusta el aspecto del vídeo: fondo, margen, esquinas redondeadas, sombra, marco de dispositivo, formato, la disposición de la cámara, el cursor y el volumen general. También puede activar los [subtítulos](./captions.md), darles estilo y colocarlos, y traducirlos él mismo a otro idioma. Y puede escribir un efecto visual que los ajustes no ofrecen: consulta [Efectos personalizados](./effects.md). El panel que lo rodea: diff --git a/website/i18n/es/docusaurus-plugin-content-docs/current/editing-timeline.md b/website/i18n/es/docusaurus-plugin-content-docs/current/editing-timeline.md index d065f84dc4..86cd7e6fea 100644 --- a/website/i18n/es/docusaurus-plugin-content-docs/current/editing-timeline.md +++ b/website/i18n/es/docusaurus-plugin-content-docs/current/editing-timeline.md @@ -78,6 +78,10 @@ Una fila de botones predefinidos (0.5×, 1×, 1.5×, 2×, 4×) y un campo numér Un tramo en el que la cámara web llena el cuadro en lugar de ocupar su recuadro de la disposición: útil para una introducción hablando a cámara en medio de una grabación de pantalla. Solo tiene sentido cuando la grabación tiene una pista de cámara web. +### Regiones de efecto {#effect-regions} + +Un efecto visual que un agente de IA escribió para el proyecto. Su fila aparece bajo los tramos de cámara a pantalla completa en cuanto el proyecto tiene uno: consulta [Efectos personalizados](./effects.md#in-the-editor). + ### Anotaciones {#annotations} Cuatro tipos, que se eligen en la fila **Tipo** del inspector. Cambiar de tipo conserva el tramo de la región y su lugar en la imagen. diff --git a/website/i18n/es/docusaurus-plugin-content-docs/current/effects.md b/website/i18n/es/docusaurus-plugin-content-docs/current/effects.md new file mode 100644 index 0000000000..02936bb40b --- /dev/null +++ b/website/i18n/es/docusaurus-plugin-content-docs/current/effects.md @@ -0,0 +1,64 @@ +--- +id: effects +title: Efectos personalizados +description: "Pide a un agente de IA un efecto visual que los ajustes no ofrecen. Escribe un shader de GPU que OpenScreen dibuja en la vista previa y en cada exportación." +keywords: + - efectos de vídeo personalizados + - efectos shader + - WGSL + - efectos de vídeo con IA + - efectos GPU +--- + +# Efectos personalizados + +Cuando los ajustes no tienen el aspecto que buscas, pídeselo a un agente: una onda en cada clic, una viñeta que respira, un fundido a negro en los últimos segundos. El agente escribe el efecto como un pequeño shader de GPU, y OpenScreen lo dibuja en la vista previa y en cada exportación, MP4 y GIF, en Windows, macOS y Linux. + +Los efectos vienen de un agente: el chat integrado o un cliente MCP como Claude Code o Codex (consulta [Edición con IA](./ai-editing.md)). Una vez que el efecto está en el proyecto, lo ajustas tú mismo en el editor. + +## Pedir un efecto {#asking-for-an-effect} + +Describe lo que quieres ver, y cuándo: «una onda en cada clic», «fundido a negro en los dos últimos segundos, subtítulos incluidos». El agente escribe el efecto, lo coloca en la línea de tiempo y puede ver el resultado con `getFrames` sin exportar el vídeo. + +Un efecto sabe dónde está el cursor y dónde cayó cada clic, dónde están la grabación y la cámara web en el vídeo tras los zooms y la disposición, y en qué momento está. Así, un halo puede seguir al ratón, y una onda puede salir de cada clic, allí donde el zoom lo haya llevado. + +## En el editor {#in-the-editor} + +Los efectos tienen su propia fila en la línea de tiempo, bajo los tramos de cámara a pantalla completa, en cuanto el proyecto tiene uno. Cada bloque lleva el nombre del efecto. Muévelo, cambia su tamaño, cópialo y pégalo o elimínalo como cualquier otra región, y `Ctrl/Cmd + Z` deshace el cambio. Solo se reproduce un efecto a la vez: pegar sobre otro efecto se rechaza. + +Haz clic en un bloque para abrir su inspector: + +- **Un control para cada ajuste** que el agente dio al efecto: un deslizador, un interruptor, un color o un punto. +- **Se aplica a**, dónde se dibuja el efecto: + - **Imagen**, por defecto: la grabación, la cámara web, el cursor y el fondo, bajo tus anotaciones y subtítulos, que siguen legibles. + - **Textos**: solo las anotaciones y los subtítulos. + - **Ambos**: el cuadro entero, texto incluido, para un fundido a negro que también cubra los subtítulos. +- **Coste**: **Ligero**, **Medio**, **Pesado** o **Muy pesado** (consulta [más abajo](#cost)). +- **Código**: el shader, de solo lectura, con **Copiar código**. Para cambiarlo, pídeselo al agente. + +Un efecto copiado y pegado en otro proyecto se lleva su código. + +## Coste {#cost} + +El coste se estima a partir del código, contando cuántas veces lee el efecto la imagen por cada píxel. El mismo código recibe el mismo nivel en todos los ordenadores. Nada se rechaza por su coste: cuanto más pesado es el efecto, más corto es el tramo que le conviene. + +| Nivel | Efectos típicos | Conviene para | +|---|---|---| +| **Ligero** | viñeta, etalonaje, onda al hacer clic | todo el vídeo | +| **Medio** | aberración cromática, desenfoque pequeño, glitch | unas decenas de segundos | +| **Pesado** | desenfoque fuerte, bokeh | una transición, un momento destacado | +| **Muy pesado** | raymarching, simulaciones | unos segundos: la exportación sigue siendo exacta pero más lenta, y la vista previa puede ralentizarse en una GPU pequeña | + +## Si un efecto no compila {#if-an-effect-doesnt-compile} + +El código de un agente se compila antes de aceptar el efecto. El código que falla se rechaza con el error del compilador, y el proyecto no cambia. + +Un proyecto abierto en otro ordenador aún puede toparse con un compilador que no está de acuerdo. El efecto entonces no se dibuja, su inspector muestra **No compila en este ordenador** con el error, y la exportación continúa sin él. + +## Proyectos de otras personas {#projects-from-someone-else} + +Un proyecto lleva el código de sus efectos, así que abrir el proyecto de otra persona ejecuta código que no escribiste tú. Ese código solo se ejecuta en la GPU, sin acceso a tus archivos, a la red ni a la memoria de la app. + +## Para autores de agentes {#for-agent-authors} + +Las herramientas son `addEffect`, `setEffect`, `setEffectDefinition` y `getEffect`, y `removeModifier` quita un efecto. El contrato completo, es decir, la función `effect(uv)`, lo que puede leer y sus reglas, es el recurso MCP `openscreen://effects/contract`. diff --git a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md index 22f5e25c32..e3fc85198b 100644 --- a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ Les options de connexion ChatGPT et GitHub Copilot ont été **supprimées dans ## Utiliser l'agent {#using-the-agent} -Décrivez la modification en langage courant : « coupe les temps morts de l'intro », « zoome quand j'ouvre le terminal ». L'agent travaille avec de vraies opérations de timeline annulables, pas avec un nouveau rendu : il peut ajouter et ajuster des coupes, des zooms, des régions de vitesse, des annotations (texte, flèches et flous qui masquent une information privée à l'écran), des pistes audio importées avec leurs fondus, et des segments Caméra plein écran, ajouter des zooms automatiques sur vos clics, modifier les points d'entrée et de sortie et le recadrage d'un clip, placer les images d'une autre source, dupliquer, réordonner ou retirer des clips, et lire la transcription pour trouver ce dont vous parlez. Il règle aussi l'apparence de la vidéo : fond, marge, coins arrondis, ombre, cadre d'appareil, format, disposition de la caméra, curseur et volume général. Il peut aussi activer les [sous-titres](./captions.md), les styliser et les placer, et les traduire lui-même dans une autre langue. +Décrivez la modification en langage courant : « coupe les temps morts de l'intro », « zoome quand j'ouvre le terminal ». L'agent travaille avec de vraies opérations de timeline annulables, pas avec un nouveau rendu : il peut ajouter et ajuster des coupes, des zooms, des régions de vitesse, des annotations (texte, flèches et flous qui masquent une information privée à l'écran), des pistes audio importées avec leurs fondus, et des segments Caméra plein écran, ajouter des zooms automatiques sur vos clics, modifier les points d'entrée et de sortie et le recadrage d'un clip, placer les images d'une autre source, dupliquer, réordonner ou retirer des clips, et lire la transcription pour trouver ce dont vous parlez. Il règle aussi l'apparence de la vidéo : fond, marge, coins arrondis, ombre, cadre d'appareil, format, disposition de la caméra, curseur et volume général. Il peut aussi activer les [sous-titres](./captions.md), les styliser et les placer, et les traduire lui-même dans une autre langue. Et il peut écrire un effet visuel que les réglages n'offrent pas : voir [Effets personnalisés](./effects.md). Le panneau qui l'entoure : diff --git a/website/i18n/fr/docusaurus-plugin-content-docs/current/editing-timeline.md b/website/i18n/fr/docusaurus-plugin-content-docs/current/editing-timeline.md index 8b95ebdf04..5105538ab6 100644 --- a/website/i18n/fr/docusaurus-plugin-content-docs/current/editing-timeline.md +++ b/website/i18n/fr/docusaurus-plugin-content-docs/current/editing-timeline.md @@ -78,6 +78,10 @@ Une rangée de boutons de préréglage (0.5×, 1×, 1.5×, 2×, 4×) et un champ Un passage où la webcam remplit le cadre au lieu de rester dans l'emplacement que lui donne sa disposition : utile pour une introduction face caméra au milieu d'un enregistrement d'écran. N'a de sens que si l'enregistrement contient une piste webcam. +### Régions d'effet {#effect-regions} + +Un effet visuel qu'un agent IA a écrit pour le projet. Sa rangée apparaît sous les passages Caméra plein écran dès que le projet en contient un : voir [Effets personnalisés](./effects.md#in-the-editor). + ### Annotations {#annotations} Quatre types, à choisir dans la rangée **Type** de l'inspecteur. Changer de type conserve la plage de la région et sa place à l'image. diff --git a/website/i18n/fr/docusaurus-plugin-content-docs/current/effects.md b/website/i18n/fr/docusaurus-plugin-content-docs/current/effects.md new file mode 100644 index 0000000000..8d3d6b67e5 --- /dev/null +++ b/website/i18n/fr/docusaurus-plugin-content-docs/current/effects.md @@ -0,0 +1,64 @@ +--- +id: effects +title: Effets personnalisés +description: "Demandez à un agent IA un effet visuel que les réglages n'offrent pas. Il écrit un shader GPU qu'OpenScreen dessine dans l'aperçu et dans chaque export." +keywords: + - effets vidéo personnalisés + - effets shader + - WGSL + - effets vidéo IA + - effets GPU +--- + +# Effets personnalisés + +Quand les réglages n'ont pas le rendu que vous cherchez, demandez-le à un agent : une onde à chaque clic, un vignettage qui respire, un fondu au noir sur les dernières secondes. L'agent écrit l'effet sous la forme d'un petit shader GPU, et OpenScreen le dessine dans l'aperçu et dans chaque export, MP4 comme GIF, sous Windows, macOS et Linux. + +Les effets viennent d'un agent : le chat intégré, ou un client MCP comme Claude Code ou Codex (voir [Montage par IA](./ai-editing.md)). Une fois l'effet dans le projet, vous le réglez vous-même dans l'éditeur. + +## Demander un effet {#asking-for-an-effect} + +Décrivez ce que vous voulez voir, et à quel moment : « une onde à chaque clic », « un fondu au noir sur les deux dernières secondes, sous-titres compris ». L'agent écrit l'effet, le place sur la timeline, et peut regarder le résultat avec `getFrames` sans exporter la vidéo. + +Un effet sait où est le curseur et où chaque clic a eu lieu, où se trouvent l'enregistrement et la webcam dans la vidéo après les zooms et la disposition, et quel est le moment. Un halo peut donc suivre la souris, et une onde partir de chaque clic, où que le zoom l'ait déplacé. + +## Dans l'éditeur {#in-the-editor} + +Les effets ont leur propre rangée dans la timeline, sous les passages Caméra plein écran, dès que le projet en contient un. Chaque bloc porte le nom de l'effet. Déplacez-le, redimensionnez-le, copiez-collez-le ou supprimez-le comme n'importe quelle autre région, et `Ctrl/Cmd + Z` annule la modification. Un seul effet joue à la fois : un collage sur un autre effet est refusé. + +Cliquez sur un bloc pour ouvrir son inspecteur : + +- **Un contrôle pour chaque réglage** que l'agent a donné à l'effet : un curseur, un interrupteur, une couleur ou un point. +- **S'applique à**, là où l'effet est dessiné : + - **Image**, par défaut : l'enregistrement, la webcam, le curseur et le fond, sous vos annotations et vos sous-titres, qui restent lisibles. + - **Textes** : les annotations et les sous-titres seulement. + - **Les deux** : l'image entière, textes compris, pour un fondu au noir qui couvre aussi les sous-titres. +- **Coût** : **Léger**, **Moyen**, **Lourd** ou **Très lourd** (voir [plus bas](#cost)). +- **Code** : le shader, en lecture seule, avec **Copier le code**. Pour le modifier, demandez-le à l'agent. + +Un effet copié puis collé dans un autre projet emporte son code avec lui. + +## Coût {#cost} + +Le coût est estimé à partir du code, en comptant combien de fois l'effet lit l'image pour chaque pixel. Le même code reçoit le même palier sur tous les ordinateurs. Rien n'est refusé pour son coût : plus l'effet est lourd, plus le passage qui lui convient est court. + +| Palier | Effets typiques | Convient à | +|---|---|---| +| **Léger** | vignettage, étalonnage, onde au clic | toute la vidéo | +| **Moyen** | aberration chromatique, petit flou, glitch | quelques dizaines de secondes | +| **Lourd** | gros flou, bokeh | une transition, un temps fort | +| **Très lourd** | raymarching, simulations | quelques secondes : l'export reste exact mais plus lent, et l'aperçu peut ralentir sur un petit GPU | + +## Si un effet ne compile pas {#if-an-effect-doesnt-compile} + +Le code d'un agent est compilé avant que l'effet soit accepté. Un code qui échoue est refusé avec l'erreur du compilateur, et le projet ne change pas. + +Un projet ouvert sur un autre ordinateur peut quand même tomber sur un compilateur qui n'est pas d'accord. L'effet n'est alors pas dessiné, son inspecteur affiche **Ne compile pas sur cet ordinateur** avec l'erreur, et l'export se poursuit sans lui. + +## Les projets des autres {#projects-from-someone-else} + +Un projet emporte le code de ses effets : ouvrir le projet de quelqu'un d'autre exécute donc du code que vous n'avez pas écrit. Ce code ne tourne que sur le GPU, sans accès à vos fichiers, au réseau ni à la mémoire de l'app. + +## Pour les auteurs d'agents {#for-agent-authors} + +Les outils sont `addEffect`, `setEffect`, `setEffectDefinition` et `getEffect`, et `removeModifier` retire un effet. Le contrat complet, c'est-à-dire la fonction `effect(uv)`, ce qu'elle peut lire et ses règles, est la ressource MCP `openscreen://effects/contract`. diff --git a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md index 833fdbd9af..6ec2fc3af8 100644 --- a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ ChatGPT と GitHub Copilot のサインインは、**1.8.0 で削除されまし ## エージェントを使う {#using-the-agent} -行いたい編集を、普段の言葉で伝えます（「イントロの無音部分をカットして」「ターミナルを開いたところでズームして」など）。エージェントは再レンダリングではなく、元に戻せる実際のタイムライン操作で作業します。トリム、ズーム、再生速度の範囲、注釈（テキスト、矢印、画面上の個人情報を隠すぼかし）、取り込んだオーディオトラックとそのフェード、フルスクリーンカメラのセグメントの追加と調整、クリック位置への自動ズームの追加、クリップのイン点とアウト点やクロップの編集、別のソースの映像の配置、クリップの複製・並べ替え・削除ができ、文字起こしを読んで、あなたが指している箇所を見つけます。背景、余白、角の丸み、影、デバイスフレーム、フォーマット、カメラのレイアウト、カーソル、全体の音量といった動画の見た目も設定できます。また、[字幕](./captions.md)をオンにし、スタイルと位置を設定し、エージェント自身で別の言語に翻訳することもできます。 +行いたい編集を、普段の言葉で伝えます（「イントロの無音部分をカットして」「ターミナルを開いたところでズームして」など）。エージェントは再レンダリングではなく、元に戻せる実際のタイムライン操作で作業します。トリム、ズーム、再生速度の範囲、注釈（テキスト、矢印、画面上の個人情報を隠すぼかし）、取り込んだオーディオトラックとそのフェード、フルスクリーンカメラのセグメントの追加と調整、クリック位置への自動ズームの追加、クリップのイン点とアウト点やクロップの編集、別のソースの映像の配置、クリップの複製・並べ替え・削除ができ、文字起こしを読んで、あなたが指している箇所を見つけます。背景、余白、角の丸み、影、デバイスフレーム、フォーマット、カメラのレイアウト、カーソル、全体の音量といった動画の見た目も設定できます。また、[字幕](./captions.md)をオンにし、スタイルと位置を設定し、エージェント自身で別の言語に翻訳することもできます。さらに、設定にない視覚エフェクトを書くこともできます。[カスタムエフェクト](./effects.md)を参照してください。 エージェントを囲むパネル： diff --git a/website/i18n/ja/docusaurus-plugin-content-docs/current/editing-timeline.md b/website/i18n/ja/docusaurus-plugin-content-docs/current/editing-timeline.md index 21c517724a..7ba1db717d 100644 --- a/website/i18n/ja/docusaurus-plugin-content-docs/current/editing-timeline.md +++ b/website/i18n/ja/docusaurus-plugin-content-docs/current/editing-timeline.md @@ -78,6 +78,10 @@ keywords: ウェブカメラがレイアウトの枠に収まらず、フレーム全体を占める区間です。画面録画の途中に、顔出しで話すイントロを挟みたいときに便利です。録画にウェブカメラのトラックがある場合にのみ意味があります。 +### エフェクトの範囲 {#effect-regions} + +AI エージェントがプロジェクトのために書いた視覚エフェクトです。プロジェクトにエフェクトが入ると、フルスクリーンカメラの区間の下にその行が現れます。[カスタムエフェクト](./effects.md#in-the-editor)を参照してください。 + ### 注釈 {#annotations} 注釈には 4 つの種類があり、インスペクターの**種類**の行で選びます。種類を切り替えても、範囲の区間と画面上の位置は保たれます。 diff --git a/website/i18n/ja/docusaurus-plugin-content-docs/current/effects.md b/website/i18n/ja/docusaurus-plugin-content-docs/current/effects.md new file mode 100644 index 0000000000..c565e5a63e --- /dev/null +++ b/website/i18n/ja/docusaurus-plugin-content-docs/current/effects.md @@ -0,0 +1,64 @@ +--- +id: effects +title: カスタムエフェクト +description: "設定にない視覚エフェクトを AI エージェントに頼めます。エージェントが GPU シェーダーを書き、OpenScreen がそれをプレビューとすべての書き出しで描画します。" +keywords: + - カスタム動画エフェクト + - シェーダーエフェクト + - WGSL + - AI 動画エフェクト + - GPU エフェクト +--- + +# カスタムエフェクト + +設定に欲しい見た目がないときは、エージェントに頼みましょう。クリックごとに広がる波紋、呼吸するように明暗が変わるビネット、最後の数秒の黒へのフェードなどです。エージェントはエフェクトを小さな GPU シェーダーとして書き、OpenScreen はそれをプレビューと、MP4 と GIF のすべての書き出しで、Windows、macOS、Linux のどれでも描画します。 + +エフェクトはエージェントから生まれます。内蔵のチャットか、Claude Code や Codex などの MCP クライアントです（[AI 編集](./ai-editing.md)を参照）。エフェクトがプロジェクトに入ったら、エディターで自分で調整できます。 + +## エフェクトを頼む {#asking-for-an-effect} + +見たいものと、そのタイミングを説明します。たとえば「クリックごとに波紋」「最後の 2 秒で黒にフェード、字幕も含めて」。エージェントはエフェクトを書いてタイムラインに配置し、動画を書き出さずに `getFrames` で結果を確認できます。 + +エフェクトは、カーソルの位置と各クリックの位置、ズームとレイアウトを経た動画の中で録画とウェブカメラがどこにあるか、そして時刻を知っています。そのため、ハローをマウスに追従させたり、ズームでどこに移動していても各クリックの位置から波紋を広げたりできます。 + +## エディターで {#in-the-editor} + +プロジェクトにエフェクトが入ると、タイムラインのフルスクリーンカメラの区間の下に、エフェクト専用の行が現れます。各ブロックにはエフェクトの名前が表示されます。ほかの範囲と同じように移動、サイズ変更、コピーと貼り付け、削除ができ、`Ctrl/Cmd + Z` で変更を元に戻せます。同時に再生されるエフェクトは 1 つだけです。別のエフェクトの上への貼り付けは拒否されます。 + +ブロックをクリックするとインスペクターが開きます。 + +- **各設定のコントロール**：エージェントがエフェクトに与えた設定ごとに、スライダー、スイッチ、色、点のいずれかが表示されます。 +- **適用先**：エフェクトを描画する場所です。 + - **画像**（既定）：録画、ウェブカメラ、カーソル、背景。注釈と字幕はその上に重なり、読みやすいままです。 + - **テキスト**：注釈と字幕だけ。 + - **両方**：テキストを含むフレーム全体。字幕も覆う黒へのフェードに使えます。 +- **負荷**：**軽い**、**中**、**重い**、**非常に重い**のいずれか（[下記](#cost)を参照）。 +- **コード**：シェーダーを読み取り専用で表示し、**コードをコピー**できます。変更するにはエージェントに頼みます。 + +コピーしたエフェクトを別のプロジェクトに貼り付けると、そのコードも一緒に移ります。 + +## 負荷 {#cost} + +負荷はコードから見積もられます。エフェクトがピクセルごとに画像を何回読み取るかを数えます。同じコードは、どのコンピューターでも同じ段階になります。負荷を理由に拒否されることはありません。重いエフェクトほど、向いている区間は短くなります。 + +| 段階 | 代表的なエフェクト | 向いている長さ | +|---|---|---| +| **軽い** | ビネット、カラーグレーディング、クリックの波紋 | 動画全体 | +| **中** | 色収差、弱いぼかし、グリッチ | 数十秒 | +| **重い** | 強いぼかし、ボケ | トランジション、見せ場 | +| **非常に重い** | レイマーチング、シミュレーション | 数秒。書き出しは正確なままですが遅くなり、小さな GPU ではプレビューが遅くなることがあります | + +## エフェクトがコンパイルできない場合 {#if-an-effect-doesnt-compile} + +エージェントのコードは、エフェクトが受け入れられる前にコンパイルされます。失敗したコードはコンパイラーのエラーとともに拒否され、プロジェクトは変わりません。 + +それでも、別のコンピューターで開いたプロジェクトが、判断の異なるコンパイラーに出会うことはあります。その場合、エフェクトは描画されず、インスペクターにエラーとともに**このコンピューターではコンパイルできません**と表示され、書き出しはそのエフェクトなしで続きます。 + +## ほかの人のプロジェクト {#projects-from-someone-else} + +プロジェクトにはエフェクトのコードが含まれるため、ほかの人のプロジェクトを開くと、自分が書いていないコードが実行されます。このコードは GPU 上でのみ動作し、あなたのファイル、ネットワーク、アプリのメモリにはアクセスできません。 + +## エージェント開発者向け {#for-agent-authors} + +ツールは `addEffect`、`setEffect`、`setEffectDefinition`、`getEffect` で、`removeModifier` でエフェクトを外します。完全な仕様、つまり `effect(uv)` 関数、読み取れる値、そのルールは、MCP リソース `openscreen://effects/contract` にあります。 diff --git a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md index 0d2c10ed9b..3e5ca9c4a2 100644 --- a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ As opções de login do ChatGPT e do GitHub Copilot foram **removidas na 1.8.0** ## Como usar o agente {#using-the-agent} -Descreva a edição em linguagem natural — "corte o tempo morto da introdução", "dê zoom quando eu abrir o terminal". O agente trabalha com operações reais da linha do tempo, que podem ser desfeitas, e não com uma nova renderização: ele pode adicionar e ajustar recortes, zooms, regiões de velocidade, anotações (texto, setas e desfoques que escondem dados privados na tela), faixas de áudio importadas com seus fades e segmentos de Câmera em Tela Cheia, adicionar zooms automáticos nos seus cliques, editar os pontos de entrada/saída e o corte de um clipe, posicionar as imagens de outra fonte, duplicar, reordenar ou remover clipes e ler a transcrição para encontrar o trecho a que você se refere. Ele também ajusta a aparência do vídeo: fundo, margem, cantos arredondados, sombra, moldura de dispositivo, formato, o layout da câmera, o cursor e o volume geral. Ele também pode ativar as [legendas](./captions.md), estilizá-las e posicioná-las, e traduzi-las ele mesmo para outro idioma. +Descreva a edição em linguagem natural — "corte o tempo morto da introdução", "dê zoom quando eu abrir o terminal". O agente trabalha com operações reais da linha do tempo, que podem ser desfeitas, e não com uma nova renderização: ele pode adicionar e ajustar recortes, zooms, regiões de velocidade, anotações (texto, setas e desfoques que escondem dados privados na tela), faixas de áudio importadas com seus fades e segmentos de Câmera em Tela Cheia, adicionar zooms automáticos nos seus cliques, editar os pontos de entrada/saída e o corte de um clipe, posicionar as imagens de outra fonte, duplicar, reordenar ou remover clipes e ler a transcrição para encontrar o trecho a que você se refere. Ele também ajusta a aparência do vídeo: fundo, margem, cantos arredondados, sombra, moldura de dispositivo, formato, o layout da câmera, o cursor e o volume geral. Ele também pode ativar as [legendas](./captions.md), estilizá-las e posicioná-las, e traduzi-las ele mesmo para outro idioma. E ele pode escrever um efeito visual que as configurações não oferecem: veja [Efeitos personalizados](./effects.md). O painel em volta dele: diff --git a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/editing-timeline.md b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/editing-timeline.md index 0571478e08..34c4375278 100644 --- a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/editing-timeline.md +++ b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/editing-timeline.md @@ -78,6 +78,10 @@ Uma fileira de botões de predefinição (0.5×, 1×, 1.5×, 2×, 4×) e um camp Um trecho em que a webcam preenche o quadro em vez de ficar na caixa do layout — útil para uma introdução com você falando para a câmera no meio de uma gravação de tela. Só faz sentido quando a gravação tem uma faixa de webcam. +### Regiões de efeito {#effect-regions} + +Um efeito visual que um agente de IA escreveu para o projeto. A fileira dele aparece abaixo dos trechos de Câmera em Tela Cheia assim que o projeto tem um: veja [Efeitos personalizados](./effects.md#in-the-editor). + ### Anotações {#annotations} Quatro tipos, escolhidos na linha **Tipo** do inspetor. Trocar o tipo mantém o trecho da região e o lugar dela na imagem. diff --git a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/effects.md b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/effects.md new file mode 100644 index 0000000000..720cf00d76 --- /dev/null +++ b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/effects.md @@ -0,0 +1,64 @@ +--- +id: effects +title: Efeitos personalizados +description: "Peça a um agente de IA um efeito visual que as configurações não oferecem. Ele escreve um shader de GPU que o OpenScreen desenha na pré-visualização e em cada exportação." +keywords: + - efeitos de vídeo personalizados + - efeitos shader + - WGSL + - efeitos de vídeo com IA + - efeitos GPU +--- + +# Efeitos personalizados + +Quando as configurações não têm o visual que você procura, peça a um agente: uma onda a cada clique, uma vinheta que respira, um fade para preto nos últimos segundos. O agente escreve o efeito como um pequeno shader de GPU, e o OpenScreen o desenha na pré-visualização e em cada exportação, MP4 e GIF, no Windows, no macOS e no Linux. + +Os efeitos vêm de um agente: o chat integrado ou um cliente MCP como o Claude Code ou o Codex (veja [Edição com IA](./ai-editing.md)). Depois que o efeito está no projeto, você mesmo o ajusta no editor. + +## Pedir um efeito {#asking-for-an-effect} + +Descreva o que você quer ver, e quando: "uma onda a cada clique", "fade para preto nos dois últimos segundos, com as legendas". O agente escreve o efeito, coloca-o na linha do tempo e pode ver o resultado com `getFrames` sem exportar o vídeo. + +Um efeito sabe onde está o cursor e onde caiu cada clique, onde ficam a gravação e a webcam no vídeo depois dos zooms e do layout, e em que momento está. Assim, um halo pode seguir o mouse, e uma onda pode partir de cada clique, onde quer que o zoom o tenha levado. + +## No editor {#in-the-editor} + +Os efeitos ganham uma fileira própria na linha do tempo, abaixo dos trechos de Câmera em Tela Cheia, assim que o projeto tem um. Cada bloco leva o nome do efeito. Mova, redimensione, copie e cole ou exclua o bloco como qualquer outra região, e `Ctrl/Cmd + Z` desfaz a alteração. Só um efeito toca por vez: colar sobre outro efeito é recusado. + +Clique em um bloco para abrir o inspetor dele: + +- **Um controle para cada ajuste** que o agente deu ao efeito: um controle deslizante, um interruptor, uma cor ou um ponto. +- **Aplica-se a**, onde o efeito é desenhado: + - **Imagem**, o padrão: a gravação, a webcam, o cursor e o fundo, abaixo das suas anotações e legendas, que continuam legíveis. + - **Textos**: só as anotações e as legendas. + - **Ambos**: o quadro inteiro, com o texto, para um fade para preto que também cubra as legendas. +- **Custo**: **Leve**, **Médio**, **Pesado** ou **Muito pesado** (veja [abaixo](#cost)). +- **Código**: o shader, somente leitura, com **Copiar código**. Para mudá-lo, peça ao agente. + +Um efeito copiado e colado em outro projeto leva o seu código junto. + +## Custo {#cost} + +O custo é estimado a partir do código, contando quantas vezes o efeito lê a imagem para cada pixel. O mesmo código recebe o mesmo nível em todos os computadores. Nada é recusado pelo custo: quanto mais pesado o efeito, mais curto o trecho que combina com ele. + +| Nível | Efeitos típicos | Combina com | +|---|---|---| +| **Leve** | vinheta, correção de cor, onda ao clicar | o vídeo inteiro | +| **Médio** | aberração cromática, desfoque pequeno, glitch | algumas dezenas de segundos | +| **Pesado** | desfoque forte, bokeh | uma transição, um momento de destaque | +| **Muito pesado** | raymarching, simulações | alguns segundos: a exportação continua exata, mas mais lenta, e a pré-visualização pode ficar lenta em uma GPU pequena | + +## Se um efeito não compila {#if-an-effect-doesnt-compile} + +O código de um agente é compilado antes de o efeito ser aceito. Um código que falha é recusado com o erro do compilador, e o projeto não muda. + +Um projeto aberto em outro computador ainda pode encontrar um compilador que discorda. O efeito então não é desenhado, o inspetor dele mostra **Não compila neste computador** com o erro, e a exportação continua sem ele. + +## Projetos de outras pessoas {#projects-from-someone-else} + +Um projeto leva o código dos seus efeitos, então abrir o projeto de outra pessoa executa código que você não escreveu. Esse código roda só na GPU, sem acesso aos seus arquivos, à rede nem à memória do app. + +## Para autores de agentes {#for-agent-authors} + +As ferramentas são `addEffect`, `setEffect`, `setEffectDefinition` e `getEffect`, e `removeModifier` remove um efeito. O contrato completo, ou seja, a função `effect(uv)`, o que ela pode ler e as suas regras, é o recurso MCP `openscreen://effects/contract`. diff --git a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md index 346d5336a5..3e885fdd88 100644 --- a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ ChatGPT 和 GitHub Copilot 登录选项已**在 1.8.0 中移除**。它们的工 ## 使用 AI 代理 {#using-the-agent} -用日常语言描述你想要的编辑，例如“剪掉开头的空白”“我打开终端时放大画面”。AI 代理通过真实、可撤销的时间轴操作来完成编辑，而不是重新渲染：它可以添加和调整删减区间、缩放、速度区间、标注（文字、箭头，以及遮住屏幕上隐私信息的模糊）、导入的音轨及其淡入淡出和全屏摄像头区间，在你的点击处添加自动缩放，编辑片段的入点/出点和裁剪，放入另一个素材的画面，复制、重新排序或移除片段，还会读取转录文本，找到你所指的内容。它还能设置视频的外观：背景、边距、圆角、阴影、设备边框、画面比例、摄像头布局、光标和总音量。它还可以开启[字幕](./captions.md)，设置字幕的样式和位置，并亲自把字幕翻译成另一种语言。 +用日常语言描述你想要的编辑，例如“剪掉开头的空白”“我打开终端时放大画面”。AI 代理通过真实、可撤销的时间轴操作来完成编辑，而不是重新渲染：它可以添加和调整删减区间、缩放、速度区间、标注（文字、箭头，以及遮住屏幕上隐私信息的模糊）、导入的音轨及其淡入淡出和全屏摄像头区间，在你的点击处添加自动缩放，编辑片段的入点/出点和裁剪，放入另一个素材的画面，复制、重新排序或移除片段，还会读取转录文本，找到你所指的内容。它还能设置视频的外观：背景、边距、圆角、阴影、设备边框、画面比例、摄像头布局、光标和总音量。它还可以开启[字幕](./captions.md)，设置字幕的样式和位置，并亲自把字幕翻译成另一种语言。它还能写出设置里没有的视觉效果：参见[自定义效果](./effects.md)。 它周围的面板包括： diff --git a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/editing-timeline.md b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/editing-timeline.md index 09120cc935..ea1c5bdf1e 100644 --- a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/editing-timeline.md +++ b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/editing-timeline.md @@ -78,6 +78,10 @@ keywords: 在这段时间里，摄像头画面会占满整个画面，而不是待在布局框里，适合在屏幕录制中间插入一段真人出镜的介绍。仅当录制内容带有摄像头轨道时才有意义。 +### 效果区间 {#effect-regions} + +AI 代理为项目写的视觉效果。项目里一有效果，它所在的一行就会出现在全屏摄像头区间下方：参见[自定义效果](./effects.md#in-the-editor)。 + ### 标注 {#annotations} 共有四种类型，在检查器的**类型**一行中选择。切换类型时，区间的时间范围和它在画面中的位置都会保留。 diff --git a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/effects.md b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/effects.md new file mode 100644 index 0000000000..16d072779d --- /dev/null +++ b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/effects.md @@ -0,0 +1,64 @@ +--- +id: effects +title: 自定义效果 +description: "让 AI 代理做出设置里没有的视觉效果。它会写一个 GPU 着色器，OpenScreen 在预览和每次导出中绘制它。" +keywords: + - 自定义视频效果 + - 着色器效果 + - WGSL + - AI 视频效果 + - GPU 效果 +--- + +# 自定义效果 + +如果设置里没有你想要的效果，就让代理来做：每次点击时泛起的波纹、像呼吸一样明暗变化的暗角、最后几秒淡出到黑色。代理把效果写成一个小小的 GPU 着色器，OpenScreen 会在预览以及每次导出（MP4 和 GIF）中绘制它，Windows、macOS 和 Linux 上都是如此。 + +效果来自代理：内置聊天，或者 Claude Code、Codex 等 MCP 客户端（参见 [AI 编辑](./ai-editing.md)）。效果进入项目后，你可以在编辑器里自己调整。 + +## 请求一个效果 {#asking-for-an-effect} + +描述你想看到什么，以及在什么时候出现，例如“每次点击都有波纹”“最后两秒淡出到黑色，字幕也一起”。代理会写出效果、把它放到时间轴上，并且可以用 `getFrames` 查看结果，无需导出视频。 + +效果知道光标在哪里、每次点击落在哪里，知道经过缩放和布局之后录屏和摄像头在视频中的位置，也知道当前时间。因此，光晕可以跟随鼠标，波纹可以从每次点击的位置泛起，无论缩放把它移到了哪里。 + +## 在编辑器中 {#in-the-editor} + +项目里一有效果，时间轴上就会在全屏摄像头区间下方出现效果专用的一行。每个区块都显示效果的名称。可以像其他区间一样移动、调整长度、复制粘贴或删除它，`Ctrl/Cmd + Z` 可以撤销修改。同一时间只播放一个效果：粘贴到另一个效果上会被拒绝。 + +点击区块即可打开它的检查器： + +- **每个设置对应一个控件**：代理为效果提供的每个设置，会显示为滑块、开关、颜色或点。 +- **作用于**，即效果绘制在哪里： + - **画面**（默认）：录屏、摄像头、光标和背景，位于你的标注和字幕之下，文字保持清晰可读。 + - **文字**：只作用于标注和字幕。 + - **两者**：整个画面，包括文字，例如连字幕一起覆盖的淡出到黑色。 +- **开销**：**轻**、**中**、**重**或**很重**（参见[下文](#cost)）。 +- **代码**：只读的着色器代码，附带**复制代码**按钮。要修改它，请让代理来改。 + +把复制的效果粘贴到另一个项目时，它的代码也会一起带过去。 + +## 开销 {#cost} + +开销根据代码估算：统计效果为每个像素读取画面的次数。同一份代码在任何电脑上都得到相同的级别。不会因为开销而拒绝任何效果：效果越重，适合它的片段就越短。 + +| 级别 | 典型效果 | 适合 | +|---|---|---| +| **轻** | 暗角、调色、点击波纹 | 整个视频 | +| **中** | 色差、轻微模糊、故障效果 | 几十秒 | +| **重** | 强模糊、散景 | 一个转场、一个高光时刻 | +| **很重** | 光线步进、模拟 | 几秒钟：导出依然准确但更慢，在性能较弱的 GPU 上预览可能会变慢 | + +## 效果无法编译时 {#if-an-effect-doesnt-compile} + +代理的代码会在效果被接受之前编译。编译失败的代码会连同编译器的错误一起被拒绝，项目保持不变。 + +在另一台电脑上打开的项目，仍可能遇到判断不同的编译器。这时效果不会被绘制，它的检查器会显示**无法在这台电脑上编译**以及错误信息，导出则会在没有该效果的情况下继续。 + +## 他人的项目 {#projects-from-someone-else} + +项目会带着其中效果的代码，所以打开别人的项目会运行不是你写的代码。这段代码只在 GPU 上运行，无法访问你的文件、网络或应用的内存。 + +## 面向代理开发者 {#for-agent-authors} + +相关工具是 `addEffect`、`setEffect`、`setEffectDefinition` 和 `getEffect`，用 `removeModifier` 移除效果。完整的约定，也就是 `effect(uv)` 函数、它能读取的内容及其规则，在 MCP 资源 `openscreen://effects/contract` 中。 diff --git a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md index 1ccdabde9a..efe33494df 100644 --- a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ ChatGPT 與 GitHub Copilot 的登入選項已**在 1.8.0 中移除**。它們的 ## 使用代理 {#using-the-agent} -用一般的語言描述你要的剪輯，例如「剪掉開頭的空檔」、「我打開終端機時放大畫面」。代理會透過真實、可復原的時間軸操作來完成工作，而不是重新算繪：它可以新增與調整剪輯、縮放、速度區域、標註（文字、箭頭，以及遮住畫面上隱私資訊的模糊）、匯入的音軌及其淡入淡出與全螢幕攝影機片段，在你的點擊處加入自動縮放，編輯片段的入點與出點和裁切，放入另一個來源的畫面，複製、重新排序或移除片段，也會讀取逐字稿來找出你指的是哪一段。它也能設定影片的外觀：背景、邊距、圓角、陰影、裝置外框、畫面比例、攝影機版面、游標與整體音量。它也能開啟[字幕](./captions.md)，設定字幕的樣式與位置，並自行將字幕翻譯成另一種語言。 +用一般的語言描述你要的剪輯，例如「剪掉開頭的空檔」、「我打開終端機時放大畫面」。代理會透過真實、可復原的時間軸操作來完成工作，而不是重新算繪：它可以新增與調整剪輯、縮放、速度區域、標註（文字、箭頭，以及遮住畫面上隱私資訊的模糊）、匯入的音軌及其淡入淡出與全螢幕攝影機片段，在你的點擊處加入自動縮放，編輯片段的入點與出點和裁切，放入另一個來源的畫面，複製、重新排序或移除片段，也會讀取逐字稿來找出你指的是哪一段。它也能設定影片的外觀：背景、邊距、圓角、陰影、裝置外框、畫面比例、攝影機版面、游標與整體音量。它也能開啟[字幕](./captions.md)，設定字幕的樣式與位置，並自行將字幕翻譯成另一種語言。它也能寫出設定裡沒有的視覺效果：請參閱[自訂效果](./effects.md)。 它周圍的面板包括： diff --git a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/editing-timeline.md b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/editing-timeline.md index ec9ef423f0..331af98b9b 100644 --- a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/editing-timeline.md +++ b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/editing-timeline.md @@ -78,6 +78,10 @@ keywords: 在這段範圍內，網路攝影機會佔滿整個畫面，而不是待在版面配置的框框裡，適合在螢幕錄影中間穿插一段真人出鏡的開場。只有在錄影含有網路攝影機軌道時才有意義。 +### 效果區域 {#effect-regions} + +AI 代理為專案寫的視覺效果。專案裡一有效果，它所在的那一列就會出現在全螢幕攝影機區域下方：請參閱[自訂效果](./effects.md#in-the-editor)。 + ### 標註 {#annotations} 共有四種類型，在檢查器的**類型**一列中選擇。切換類型時會保留區域的時間範圍，以及它在畫面中的位置。 diff --git a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/effects.md b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/effects.md new file mode 100644 index 0000000000..e3f2cb5e17 --- /dev/null +++ b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/effects.md @@ -0,0 +1,64 @@ +--- +id: effects +title: 自訂效果 +description: "請 AI 代理做出設定裡沒有的視覺效果。它會寫一個 GPU 著色器，OpenScreen 會在預覽和每次匯出中繪製它。" +keywords: + - 自訂影片效果 + - 著色器效果 + - WGSL + - AI 影片效果 + - GPU 效果 +--- + +# 自訂效果 + +如果設定裡沒有你想要的效果，就請代理來做：每次點擊時泛起的波紋、像呼吸一樣明暗變化的暗角、最後幾秒淡出成黑色。代理會把效果寫成一個小小的 GPU 著色器，OpenScreen 會在預覽以及每次匯出（MP4 和 GIF）中繪製它，Windows、macOS 和 Linux 上都是如此。 + +效果來自代理：內建聊天，或是 Claude Code、Codex 等 MCP 用戶端（請參閱 [AI 剪輯](./ai-editing.md)）。效果進入專案後，你可以在編輯器裡自行調整。 + +## 請求一個效果 {#asking-for-an-effect} + +描述你想看到什麼，以及在什麼時候出現，例如「每次點擊都有波紋」、「最後兩秒淡出成黑色，字幕也一起」。代理會寫出效果、把它放到時間軸上，並且可以用 `getFrames` 查看結果，不必匯出影片。 + +效果知道游標在哪裡、每次點擊落在哪裡，知道經過縮放和版面配置之後，螢幕錄影和攝影機在影片中的位置，也知道目前的時間。因此，光暈可以跟著滑鼠移動，波紋可以從每次點擊的位置泛起，無論縮放把它移到了哪裡。 + +## 在編輯器中 {#in-the-editor} + +專案裡一有效果，時間軸上就會在全螢幕攝影機區域下方出現效果專用的一列。每個區塊都會顯示效果的名稱。你可以像其他區域一樣移動、調整長度、複製貼上或刪除它，`Ctrl/Cmd + Z` 可以復原修改。同一時間只會播放一個效果：貼到另一個效果上會被拒絕。 + +點擊區塊即可開啟它的檢查器： + +- **每個設定對應一個控制項**：代理為效果提供的每個設定，會顯示為滑桿、開關、顏色或點。 +- **作用於**，也就是效果繪製在哪裡： + - **畫面**（預設）：螢幕錄影、攝影機、游標和背景，位於你的標註和字幕之下，文字保持清楚可讀。 + - **文字**：只作用於標註和字幕。 + - **兩者**：整個畫面，包括文字，例如連字幕一起覆蓋的淡出成黑色。 +- **開銷**：**輕**、**中**、**重**或**很重**（請參閱[下文](#cost)）。 +- **程式碼**：唯讀的著色器程式碼，附有**複製程式碼**按鈕。要修改它，請代理來改。 + +把複製的效果貼到另一個專案時，它的程式碼也會一起帶過去。 + +## 開銷 {#cost} + +開銷是根據程式碼估算的：計算效果為每個像素讀取畫面的次數。同一份程式碼在任何電腦上都會得到相同的等級。不會因為開銷而拒絕任何效果：效果越重，適合它的片段就越短。 + +| 等級 | 典型效果 | 適合 | +|---|---|---| +| **輕** | 暗角、調色、點擊波紋 | 整部影片 | +| **中** | 色差、輕微模糊、故障效果 | 幾十秒 | +| **重** | 強烈模糊、散景 | 一個轉場、一個精彩時刻 | +| **很重** | 光線步進、模擬 | 幾秒鐘：匯出依然準確但較慢，在效能較弱的 GPU 上預覽可能會變慢 | + +## 效果無法編譯時 {#if-an-effect-doesnt-compile} + +代理的程式碼會在效果被接受之前編譯。編譯失敗的程式碼會連同編譯器的錯誤一起被拒絕，專案保持不變。 + +在另一台電腦上開啟的專案，仍可能遇到判斷不同的編譯器。這時效果不會被繪製，它的檢查器會顯示**無法在這台電腦上編譯**以及錯誤訊息，匯出則會在沒有該效果的情況下繼續。 + +## 他人的專案 {#projects-from-someone-else} + +專案會帶著其中效果的程式碼，所以開啟別人的專案會執行不是你寫的程式碼。這段程式碼只在 GPU 上執行，無法存取你的檔案、網路或應用程式的記憶體。 + +## 給代理開發者 {#for-agent-authors} + +相關工具是 `addEffect`、`setEffect`、`setEffectDefinition` 和 `getEffect`，用 `removeModifier` 移除效果。完整的規範，也就是 `effect(uv)` 函式、它能讀取的內容及其規則，在 MCP 資源 `openscreen://effects/contract` 中。 diff --git a/website/sidebars.ts b/website/sidebars.ts index c69ccbea7b..b66c0d03c4 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -18,6 +18,7 @@ const sidebars: SidebarsConfig = { "editing-timeline", "captions", "ai-editing", + "effects", "export", "cli", ], diff --git a/workbench/lib/editorial.ts b/workbench/lib/editorial.ts index 9f4f28150a..5c7c989abc 100644 --- a/workbench/lib/editorial.ts +++ b/workbench/lib/editorial.ts @@ -64,6 +64,7 @@ export function regionFamilies(document: AxcutDocument): RegionFamily[] { { kind: "annotation", regions: document.annotations as unknown as Region[] }, { kind: "speed", regions: legacyRegions(document, "speedRegions") }, { kind: "cameraFullscreen", regions: legacyRegions(document, "cameraFullscreenRegions") }, + { kind: "shaderEffect", regions: (document.shaderEffectRanges ?? []) as unknown as Region[] }, ]; } @@ -537,7 +538,14 @@ export function zoomIssues(document: AxcutDocument, options: ZoomHygieneOptions // ─── 6. did it do THAT, and nothing else ──────────────────────────────────── -export type EditFamily = "clip" | "trim" | "zoom" | "annotation" | "speed" | "cameraFullscreen"; +export type EditFamily = + | "clip" + | "trim" + | "zoom" + | "annotation" + | "speed" + | "cameraFullscreen" + | "shaderEffect"; export interface FamilyDelta { family: EditFamily; diff --git a/workbench/lib/oracles.ts b/workbench/lib/oracles.ts index e25f3350da..505ab180b6 100644 --- a/workbench/lib/oracles.ts +++ b/workbench/lib/oracles.ts @@ -201,6 +201,33 @@ export function documentInvariants(document: AxcutDocument): Violation[] { } } + // 3b. Shader effects: every region names a definition the document holds, and its values + // stay inside their parameter's bounds (`storedShaderEffectValues` clamps on write). + const effectsById = new Map((document.shaderEffects ?? []).map((e) => [e.id, e])); + for (const range of document.shaderEffectRanges ?? []) { + const effect = effectsById.get(range.effectId); + if (!effect) { + violations.push({ + rule: "effects.definition-exists", + detail: `effect region ${range.id} names missing effect ${range.effectId}`, + }); + continue; + } + for (const param of effect.params) { + const value = range.values[param.name]; + if ( + param.type === "number" && + typeof value === "number" && + (value < param.min || value > param.max) + ) { + violations.push({ + rule: "effects.value-in-bounds", + detail: `effect region ${range.id}: ${param.name}=${value} outside ${param.min}-${param.max}`, + }); + } + } + } + // 4. Ids stay unique per family. `replaceTimeline` used to mint `clip_1..N` // and `trim_1..N` by hand (DSL-8) instead of going through `createId`; // it now reuses the id of any clip an interval reproduces exactly and diff --git a/workbench/scenarios/effect-ripple-on-click.scn.ts b/workbench/scenarios/effect-ripple-on-click.scn.ts new file mode 100644 index 0000000000..651c3c4225 --- /dev/null +++ b/workbench/scenarios/effect-ripple-on-click.scn.ts @@ -0,0 +1,158 @@ +// ponytail: the first scenario of agent-written shader effects (docs/shader-effects.md, E.7). +// The prompt names a visual effect no setting offers, so the one right move is to WRITE one: +// addEffect, with WGSL whose wave starts from `ctx.clicks`, over a region that covers the clicks. +// The effect does not need the cursor track: the compositor hands it every click already placed +// in output uv (contract C.2). Reading the track is therefore neither required nor penalised. +// +// Offline there is no compositor: the code is not compiled, the result says `validated: false`, +// and the checks read the code as text. A live run inside the app compiles it for real. + +import { cursorTelemetry, singleClip } from "../lib/fixtures"; +import { defineScenario, fail, pass } from "../lib/scenario"; + +/** Three clicks, one per dwell, at its middle: 3 s, 9 s and 17 s. */ +const DWELLS = [ + { atSec: 3, holdSec: 1.2, cx: 0.3, cy: 0.4 }, + { atSec: 9, holdSec: 1, cx: 0.72, cy: 0.55 }, + { atSec: 17, holdSec: 0.9, cx: 0.5, cy: 0.8 }, +]; +const DURATION_SEC = 24.703979; +/** How long after a click its ripple must still be inside a region. */ +const RIPPLE_SEC = 0.5; + +const RIPPLE = [ + "fn ripple(uv: vec2, click: vec4) -> vec2 {", + " if (click.w < 0.5 || click.z > params.duration) { return vec2(0.0); }", + " let t = click.z / params.duration;", + " let d = (uv - click.xy) * vec2(ctx.aspect, 1.0);", + " let r = length(d);", + " let x = params.radius * (1.0 - pow(1.0 - t, 3.0)) - r;", + " let env = smoothstep(-0.008, 0.0, x) * exp(-max(x, 0.0) / 0.07);", + " let wave = sin(x / 0.03 * 6.2831853) * env * (1.0 - t) * (1.0 - t);", + " let dir = select(vec2(0.0), d / r, r > 1e-5);", + " return dir * wave * 0.01 / vec2(ctx.aspect, 1.0);", + "}", + "fn effect(uv: vec2) -> vec4 {", + " var offset = vec2(0.0);", + " for (var i = 0; i < 4; i++) { offset += ripple(uv, ctx.clicks[i]); }", + " return frame(uv - offset);", + "}", +].join("\n"); + +export default defineScenario({ + id: "effect-ripple-on-click", + title: "Effet shader — un ripple à chaque clic de souris", + tags: ["effects", "dsl"], + prompt: "Add a ripple effect on every mouse click.", + document: () => singleClip(), + cursorTelemetry: () => cursorTelemetry({ dwells: DWELLS, durationSec: DURATION_SEC }), + gate: 0.6, + reps: 3, + + behaviour: [ + { + id: "beh.writes-an-effect", + weight: 3, + check: (c) => + c.calls("addEffect").length > 0 + ? pass() + : fail("aucun addEffect : aucun réglage ne fait de ripple, il fallait l'écrire"), + }, + ], + + dsl: [ + { + id: "dsl.effect.reads-the-clicks", + weight: 3, + check: (c) => + (c.after.shaderEffects ?? []).some((e) => e.code.includes("ctx.clicks")) + ? pass() + : fail("aucun effet ne lit ctx.clicks : l'onde ne peut pas partir du point du clic"), + }, + { + id: "dsl.effect.covers-the-clicks", + weight: 3, + check: (c) => { + // Only a range of an effect that reads the clicks makes a ripple there. + const readsClicks = new Set( + (c.after.shaderEffects ?? []) + .filter((e) => e.code.includes("ctx.clicks")) + .map((e) => e.id), + ); + const ranges = (c.after.shaderEffectRanges ?? []).filter((r) => + readsClicks.has(r.effectId), + ); + const missed = DWELLS.filter( + (d) => + !ranges.some( + (r) => r.startMs <= d.atSec * 1000 && r.endMs >= (d.atSec + RIPPLE_SEC) * 1000, + ), + ); + return missed.length === 0 + ? pass() + : fail( + `clics hors de toute région d'un effet qui lit ctx.clicks : ${missed.map((d) => `${d.atSec} s`).join(", ")}`, + ); + }, + }, + { + id: "dsl.effect.image-by-default", + weight: 2, + check: (c) => { + const elsewhere = (c.after.shaderEffectRanges ?? []).filter((r) => r.target !== "image"); + return elsewhere.length === 0 + ? pass() + : fail(`cible ${elsewhere[0].target} alors que rien ne demandait les textes`); + }, + }, + { + id: "dsl.turn.completed", + weight: 2, + check: (c) => + c.run.ok ? pass() : fail(`${c.classifyFailure()} : ${(c.run.error ?? "").slice(0, 200)}`), + }, + ], + + demoScript: [ + { kind: "tools", calls: [{ name: "getCurrentDocument", args: {} }] }, + { + kind: "tools", + calls: [ + { + name: "addEffect", + args: { + startSec: 0, + endSec: DURATION_SEC, + effect: { + name: "Ripple on click", + description: "A water ripple spreads from every click and bends the picture.", + code: RIPPLE, + params: [ + { + type: "number", + name: "radius", + label: "Radius", + default: 0.2, + min: 0.05, + max: 0.5, + }, + { + type: "number", + name: "duration", + label: "Duration", + default: 0.9, + min: 0.3, + max: 2, + }, + ], + }, + }, + }, + ], + }, + { + kind: "text", + text: "I added a ripple that spreads from every click, across the whole video.", + }, + ], +}); diff --git a/workbench/scenarios/registry.ts b/workbench/scenarios/registry.ts index bacd869d93..629107fcdc 100644 --- a/workbench/scenarios/registry.ts +++ b/workbench/scenarios/registry.ts @@ -10,6 +10,7 @@ import cursorQuestion, { cursorBlind } from "./cursor-question.scn"; import cutSilencesClean from "./cut-silences-clean.scn"; import describeProject from "./describe-project.scn"; import describeZooms, { describeZoomsMigrated } from "./describe-zooms.scn"; +import effectRippleOnClick from "./effect-ripple-on-click.scn"; import noInventedBounds from "./no-invented-bounds.scn"; import outOfScopeStyling from "./out-of-scope-styling.scn"; import realWizardEnhance, { @@ -52,6 +53,8 @@ export const SCENARIOS: Scenario[] = [ realCutSilences, realZooms, realZoomGrounding, + // Effects the agent writes: no setting offers them, so the move is to write one. + effectRippleOnClick, // Controls: reachable, and expected green. targetRightClip, removeOneModifier,